Подкасты в FRID: публичное обнаружение podcast_dht — подписанные карточки, tombstone и поиск (MR-F3) #3

Merged
ab merged 9 commits from sanapach/frid:podcasts/discovery into main 2026-11-20 13:51:54 +00:00
4 changed files with 987 additions and 212 deletions
Showing only changes of commit c0def74115 - Show all commits

No files matched your search

+276 -91
View File
@@ -124,6 +124,11 @@ pub const MAX_PODCAST_DHT_REQUESTS_PER_PEER: usize = 8;
/// Maximum distinct `(podcast_id, publisher)` records one lookup keeps;
/// records of further owners are ignored.
pub const MAX_PODCAST_LOOKUP_RECORDS: usize = 512;
/// Most new `(podcast_id, publisher)` owners one response (or the local
/// cache) may add to a lookup: a `1 / K` share of
/// [`MAX_PODCAST_LOOKUP_RECORDS`], so the first responders cannot fill the
/// lookup and crowd out later ones.
pub const MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE: usize = MAX_PODCAST_LOOKUP_RECORDS / K;
/// A node stores a placement only when fewer than this many of its known
/// overlay contacts are closer to the key than itself (`2 * K`: the `K`
/// closest with slack for differing routing views).
@@ -141,6 +146,9 @@ const SERVE_TIMEOUT: Duration = Duration::from_secs(10);
/// A peer that failed to answer (network error or timeout) is skipped by
/// lookups, publication and bootstrap for this long.
const FAILED_PEER_BACKOFF: Duration = Duration::from_secs(30);
/// A contact that failed this many times in a row (each after its backoff)
/// is removed from the routing table and the persisted peers.
const MAX_CONSECUTIVE_FAILURES: u32 = 3;
const MAX_TICKET_BYTES: usize = 16 * 1024;
const MAX_ERROR_BYTES: usize = 1024;
@@ -354,6 +362,7 @@ pub struct PodcastDht {
routing: Mutex<RoutingTable>,
unsupported: Mutex<HashMap<EndpointId, u64>>,
failed: Mutex<HashMap<EndpointId, u64>>,
failures: Mutex<HashMap<EndpointId, u32>>,
publish_gate: Semaphore,
serve_limit: Arc<Semaphore>,
inflight: Arc<Mutex<HashMap<EndpointId, usize>>>,
@@ -401,6 +410,7 @@ impl PodcastDht {
routing: Mutex::new(routing),
unsupported: Mutex::new(HashMap::new()),
failed: Mutex::new(HashMap::new()),
failures: Mutex::new(HashMap::new()),
publish_gate: Semaphore::new(1),
serve_limit: Arc::new(Semaphore::new(MAX_CONCURRENT_PODCAST_DHT_REQUESTS)),
inflight: Arc::new(Mutex::new(HashMap::new())),
@@ -481,9 +491,12 @@ impl PodcastDht {
/// A peer whose manifest lacks [`PODCAST_DHT_ID`] at
/// [`PODCAST_DHT_PROTOCOL_VERSION`] is removed from the overlay routing
/// table and not contacted for [`UNSUPPORTED_PEER_BACKOFF`]. Peers only
/// enter the routing table after answering on [`PODCAST_DHT_ALPN`] or
/// being returned by an overlay peer, so a peer without the overlay is
/// never queried for records or asked to store them.
/// enter the routing table after answering a request on
/// [`PODCAST_DHT_ALPN`] or sending one as an authenticated requester, so
/// a peer without the overlay is never asked to store records and never
/// counts for replica responsibility. A contact another peer returns is
/// only a lookup candidate until it answers itself; a candidate that does
/// not answer is backed off like any failed peer.
pub async fn note_peer_capabilities(
&self,
peer: EndpointId,
@@ -500,10 +513,11 @@ impl PodcastDht {
}
/// Discovers overlay peers among the routing table and the music DHT
/// contacts with one `FindNode` each. Music-DHT contacts whose capability
/// manifest lacks the overlay ([`Self::note_peer_capabilities`]) are
/// skipped; others are dialed on [`PODCAST_DHT_ALPN`] and only become
/// overlay contacts when they answer.
/// contacts with one `FindNode` each, then asks the not yet known contacts
/// they return once more. Music-DHT contacts whose capability manifest
/// lacks the overlay ([`Self::note_peer_capabilities`]) are skipped;
/// others are dialed on [`PODCAST_DHT_ALPN`]. A peer only becomes an
/// overlay contact when it answers.
pub async fn bootstrap(&self) -> Result<()> {
let known_feature: HashSet<EndpointId> = lock(&self.routing)
.contacts()
@@ -517,25 +531,19 @@ impl PodcastDht {
.into_iter()
.filter(|contact| !known_feature.contains(&contact.peer_id)),
);
let mut seen = HashSet::new();
let mut seen: HashSet<EndpointId> = known_feature;
seen.insert(self.own_contact.peer_id);
let mut queried = HashSet::new();
candidates.retain(|contact| {
contact.peer_id != self.own_contact.peer_id
&& !self.is_unsupported(&contact.peer_id)
&& !self.is_backed_off(&contact.peer_id)
&& seen.insert(contact.peer_id)
&& queried.insert(contact.peer_id)
});
seen.extend(queried);
candidates.truncate(MAX_BOOTSTRAP_PEERS);
let target = *self.own_contact.node_id.as_bytes();
let responses = stream::iter(candidates.into_iter().map(|contact| async move {
let response = self
.exchange(&contact, PodcastDhtOperation::FindNode { target })
.await;
(contact, response)
}))
.buffer_unordered(BOOTSTRAP_CONCURRENCY)
.collect::<Vec<_>>()
.await;
for (contact, response) in responses {
let mut discovered = Vec::new();
for (contact, response) in self.find_own_node(candidates).await {
let response = match response {
Ok(response) => response,
Err(error) => {
@@ -544,11 +552,46 @@ impl PodcastDht {
}
};
self.learn_contact(contact).await?;
self.learn_response_nodes(response.nodes).await;
for node in response.nodes {
if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id)).is_ok()
&& !self.is_unsupported(&node.peer_id)
&& !self.is_backed_off(&node.peer_id)
&& seen.insert(node.peer_id)
{
discovered.push(node);
}
}
}
// Returned contacts are unverified: they join only by answering.
discovered.truncate(MAX_BOOTSTRAP_PEERS);
for (contact, response) in self.find_own_node(discovered).await {
match response {
Ok(_) => self.learn_contact(contact).await?,
Err(error) => {
debug!(peer = %contact.peer_id, %error, "podcast DHT contact did not answer");
}
}
}
Ok(())
}
/// One `FindNode` for this node's id to each contact.
async fn find_own_node(
&self,
contacts: Vec<NodeContact>,
) -> Vec<(NodeContact, Result<PodcastDhtResponse>)> {
let target = *self.own_contact.node_id.as_bytes();
stream::iter(contacts.into_iter().map(|contact| async move {
let response = self
.exchange(&contact, PodcastDhtOperation::FindNode { target })
.await;
(contact, response)
}))
.buffer_unordered(BOOTSTRAP_CONCURRENCY)
.collect::<Vec<_>>()
.await
}
/// Publishes (or refreshes) this device's card of a podcast.
///
/// Refused with [`MusicDhtError::PodcastPublication`] unless
@@ -892,7 +935,10 @@ impl PodcastDht {
}
/// Stores records on the `K` closest overlay nodes of each of their keys
/// (including this node when it is among them).
/// (including this node when it is among them). When a remote node
/// refuses a batch (`stored = 0`, for example because it does not count
/// itself responsible for the keys) or fails, its records are offered
/// once to the next closest nodes of the same keys, one per refusal.
async fn push(&self, records: Vec<SignedPodcastRecord>) -> Result<PodcastDhtPublishStats> {
let mut stats = PodcastDhtPublishStats {
records: records.len(),
@@ -902,69 +948,105 @@ impl PodcastDht {
return Ok(stats);
}
let mut local = Vec::new();
let mut remote: HashMap<EndpointId, (NodeContact, Vec<PodcastStoreEntry>)> = HashMap::new();
for record in records {
let mut plan = StorePlan::new();
let mut fallbacks: HashMap<(usize, PodcastDhtKey), Vec<NodeContact>> = HashMap::new();
for (index, record) in records.iter().enumerate() {
let mut local_keys = Vec::new();
let mut remote_keys: HashMap<EndpointId, (NodeContact, Vec<PodcastDhtKey>)> =
HashMap::new();
for key in record.dht_keys() {
stats.keys += 1;
let (store_local, contacts) = self.replica_targets(&key);
if store_local {
let targets = self.replica_targets(&key);
if targets.local {
local_keys.push(key);
}
for contact in contacts {
remote_keys
.entry(contact.peer_id)
.or_insert_with(|| (contact, Vec::new()))
.1
.push(key);
for contact in targets.primary {
plan.add(contact, index, key);
}
fallbacks.insert((index, key), targets.fallback);
}
if !local_keys.is_empty() {
local.push((local_keys, record.clone()));
}
for (peer, (contact, keys)) in remote_keys {
remote
.entry(peer)
.or_insert_with(|| (contact, Vec::new()))
.1
.push(PodcastStoreEntry {
keys,
record: record.clone(),
});
}
}
if !local.is_empty() {
self.database.store(local, now_ms()).await?;
stats.local_replica = true;
}
let sends = stream::iter(remote.into_values().map(|(contact, entries)| async move {
let mut accepted = false;
for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) {
let response = self
.exchange(
&contact,
PodcastDhtOperation::StoreBatch {
records: batch.to_vec(),
let (mut accepted, refused) = self.send_store_plan(&records, plan).await;
let retry = retry_plan(refused, &mut fallbacks);
if !retry.is_empty() {
accepted.extend(self.send_store_plan(&records, retry).await.0);
}
stats.remote_nodes = accepted.len();
Ok(stats)
}
/// Sends a store plan. Returns the nodes that stored at least one
/// placement and the `(record index, key)` placements offered in batches
/// that were refused or failed.
async fn send_store_plan(
&self,
records: &[SignedPodcastRecord],
plan: StorePlan,
) -> (HashSet<EndpointId>, Vec<(usize, PodcastDhtKey)>) {
let sends = stream::iter(plan.targets.into_values().map(|(contact, entries)| async move {
let entries: Vec<(usize, PodcastStoreEntry)> = entries
.into_iter()
.map(|(index, keys)| {
(
index,
PodcastStoreEntry {
keys,
record: records[index].clone(),
},
)
.await;
match response {
Ok(response) => accepted |= response.stored > 0,
Err(error) => {
debug!(peer = %contact.peer_id, %error, "podcast DHT store failed");
break;
})
.collect();
let mut accepted = false;
let mut refused = Vec::new();
let mut failed = false;
for batch in entries.chunks(MAX_PODCAST_STORE_RECORDS) {
let stored = if failed {
false
} else {
let response = self
.exchange(
&contact,
PodcastDhtOperation::StoreBatch {
records: batch.iter().map(|(_, entry)| entry.clone()).collect(),
},
)
.await;
match response {
Ok(response) => response.stored > 0,
Err(error) => {
debug!(peer = %contact.peer_id, %error, "podcast DHT store failed");
failed = true;
false
}
}
};
if stored {
accepted = true;
} else {
for (index, entry) in batch {
refused.extend(entry.keys.iter().map(|key| (*index, *key)));
}
}
}
accepted
(contact.peer_id, accepted, refused)
}))
.buffer_unordered(PUBLISH_CONCURRENCY)
.collect::<Vec<_>>()
.await;
stats.remote_nodes = sends.into_iter().filter(|accepted| *accepted).count();
Ok(stats)
let mut accepted = HashSet::new();
let mut refused = Vec::new();
for (peer, stored, keys) in sends {
if stored {
accepted.insert(peer);
}
refused.extend(keys);
}
(accepted, refused)
}
/// Iterative lookup of `key`: queries the `K` closest known overlay
@@ -1013,16 +1095,23 @@ impl PodcastDht {
filter_tokens: filter_tokens.clone(),
};
let mut futures: FuturesUnordered<_> = batch
.iter()
.into_iter()
.map(|contact| {
let operation = operation.clone();
async move { self.exchange(contact, operation).await }
async move {
let response = self.exchange(&contact, operation).await;
(contact, response)
}
})
.collect();
while let Ok(Some(response)) = tokio::time::timeout_at(deadline, futures.next()).await {
while let Ok(Some((contact, response))) =
tokio::time::timeout_at(deadline, futures.next()).await
{
let Ok(response) = response else {
continue;
};
// It answered on the overlay: now it is a verified contact.
self.learn_contact(contact).await?;
merge_records(
&mut winners,
response.records,
@@ -1031,12 +1120,12 @@ impl PodcastDht {
&filter_tokens,
);
for node in response.nodes {
// Returned contacts are only candidates until they answer.
if validate_contact(&node, &self.network_id, Some(self.own_contact.peer_id))
.is_ok()
&& !self.is_unsupported(&node.peer_id)
&& known.insert(node.peer_id)
{
self.learn_contact(node.clone()).await?;
candidates.push(node);
}
}
@@ -1098,9 +1187,11 @@ impl PodcastDht {
PodcastDhtOperation::StoreBatch { records } => {
let entries =
validate_store_batch(records, &self.network_id, authenticated_peer, now_ms())?;
// Only verified contacts that are not backed off count.
let nodes: Vec<NodeId> = lock(&self.routing)
.contacts()
.into_iter()
.filter(|contact| !self.is_backed_off(&contact.peer_id))
.map(|contact| contact.node_id)
.collect();
let entries: Vec<_> = entries
@@ -1137,10 +1228,21 @@ impl PodcastDht {
match &result {
Ok(_) => {
lock(&self.failed).remove(&contact.peer_id);
lock(&self.failures).remove(&contact.peer_id);
}
Err(MusicDhtError::Network(_) | MusicDhtError::Timeout) => {
let until = now_ms().saturating_add(FAILED_PEER_BACKOFF.as_millis() as u64);
lock(&self.failed).insert(contact.peer_id, until);
let failures = {
let mut failures = lock(&self.failures);
let count = failures.entry(contact.peer_id).or_default();
*count += 1;
*count
};
if failures >= MAX_CONSECUTIVE_FAILURES {
lock(&self.failures).remove(&contact.peer_id);
self.forget_contact(contact.peer_id).await;
}
}
Err(_) => {}
}
@@ -1180,6 +1282,16 @@ impl PodcastDht {
Ok(response)
}
/// Removes a contact that keeps failing from routing and the cache.
async fn forget_contact(&self, peer: EndpointId) {
if lock(&self.routing).remove(&peer) {
debug!(%peer, "podcast DHT contact dropped after repeated failures");
}
if let Err(error) = self.database.remove_peer(peer).await {
debug!(%peer, %error, "podcast DHT contact removal failed");
}
}
fn is_unsupported(&self, peer: &EndpointId) -> bool {
still_listed(&self.unsupported, peer)
}
@@ -1188,21 +1300,15 @@ impl PodcastDht {
still_listed(&self.failed, peer)
}
fn replica_targets(&self, key: &PodcastDhtKey) -> (bool, Vec<NodeContact>) {
let mut contacts: Vec<_> = lock(&self.routing)
/// Replica targets of `key` among the contacts that are not backed off
/// ([`replica_targets`]).
fn replica_targets(&self, key: &PodcastDhtKey) -> ReplicaTargets {
let contacts: Vec<_> = lock(&self.routing)
.contacts()
.into_iter()
.filter(|contact| !self.is_backed_off(&contact.peer_id))
.collect();
contacts.push(self.fresh_own_contact());
contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes()));
contacts.dedup_by_key(|contact| contact.peer_id);
contacts.truncate(K);
let local = contacts
.iter()
.any(|contact| contact.peer_id == self.own_contact.peer_id);
contacts.retain(|contact| contact.peer_id != self.own_contact.peer_id);
(local, contacts)
replica_targets(self.fresh_own_contact(), contacts, key)
}
fn closest_contacts(&self, target: &[u8; 32], count: usize) -> Vec<NodeContact> {
@@ -1220,17 +1326,6 @@ impl PodcastDht {
contact
}
async fn learn_response_nodes(&self, nodes: Vec<NodeContact>) {
for contact in nodes {
if self.is_unsupported(&contact.peer_id) {
continue;
}
if let Err(error) = self.learn_contact(contact).await {
debug!(%error, "invalid podcast DHT contact ignored");
}
}
}
async fn learn_contact(&self, mut contact: NodeContact) -> Result<()> {
validate_contact(&contact, &self.network_id, Some(self.own_contact.peer_id))?;
contact.node_id = NodeId::from_endpoint(&contact.peer_id);
@@ -1309,8 +1404,12 @@ fn validate_filter_tokens(tokens: &[String]) -> Result<()> {
Ok(())
}
/// Adds verified records found under `key` to the per-owner winners, keeping
/// at most [`MAX_PODCAST_LOOKUP_RECORDS`] owners.
/// Adds verified records of one response (or the local cache) found under
/// `key` to the per-owner winners. A newer record of a known owner always
/// replaces its winner; the response may add at most
/// [`MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE`] new owners, and the lookup
/// keeps at most [`MAX_PODCAST_LOOKUP_RECORDS`]. So the order in which
/// responders answer does not let the first ones take every slot.
fn merge_records(
winners: &mut HashMap<(PodcastId, EndpointId), SignedPodcastRecord>,
records: Vec<SignedPodcastRecord>,
@@ -1319,6 +1418,7 @@ fn merge_records(
filter_tokens: &[String],
) {
let now = now_ms();
let mut added = 0usize;
for record in records {
if record.verify_at(network_id, now).is_err() || !record.belongs_under(key) {
continue;
@@ -1332,14 +1432,99 @@ fn merge_records(
let owner = (record.podcast_id().clone(), record.publisher());
match winners.get(&owner) {
Some(current) if compare_records(current, &record).is_ge() => {}
None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS => {}
_ => {
Some(_) => {
winners.insert(owner, record);
}
None if winners.len() >= MAX_PODCAST_LOOKUP_RECORDS
|| added >= MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE => {}
None => {
winners.insert(owner, record);
added += 1;
}
}
}
}
/// Replica nodes of one key, from [`replica_targets`].
#[derive(Debug)]
struct ReplicaTargets {
/// Whether this node is among the `K` closest.
local: bool,
/// The other nodes among the `K` closest.
primary: Vec<NodeContact>,
/// The next up to `K` nodes, farthest first (popped closest first).
fallback: Vec<NodeContact>,
}
/// The `K` closest nodes of `key` among `contacts` and `own` (this node), and
/// the next `K` as fallbacks.
fn replica_targets(
own: NodeContact,
mut contacts: Vec<NodeContact>,
key: &PodcastDhtKey,
) -> ReplicaTargets {
let own_peer = own.peer_id;
contacts.push(own);
contacts.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes()));
contacts.dedup_by_key(|contact| contact.peer_id);
let mut fallback = contacts.split_off(K.min(contacts.len()));
fallback.retain(|contact| contact.peer_id != own_peer);
fallback.truncate(K);
fallback.reverse();
let local = contacts.iter().any(|contact| contact.peer_id == own_peer);
contacts.retain(|contact| contact.peer_id != own_peer);
ReplicaTargets {
local,
primary: contacts,
fallback,
}
}
/// One fallback node per refused `(record index, key)` placement: the
/// closest one not yet offered that placement.
fn retry_plan(
refused: Vec<(usize, PodcastDhtKey)>,
fallbacks: &mut HashMap<(usize, PodcastDhtKey), Vec<NodeContact>>,
) -> StorePlan {
let mut retry = StorePlan::new();
for (index, key) in refused {
if let Some(contact) = fallbacks
.get_mut(&(index, key))
.and_then(|fallback| fallback.pop())
{
retry.add(contact, index, key);
}
}
retry
}
/// Placements to offer per remote node: record index and keys.
struct StorePlan {
targets: HashMap<EndpointId, (NodeContact, BTreeMap<usize, Vec<PodcastDhtKey>>)>,
}
impl StorePlan {
fn new() -> Self {
Self {
targets: HashMap::new(),
}
}
fn add(&mut self, contact: NodeContact, index: usize, key: PodcastDhtKey) {
self.targets
.entry(contact.peer_id)
.or_insert_with(|| (contact, BTreeMap::new()))
.1
.entry(index)
.or_default()
.push(key);
}
fn is_empty(&self) -> bool {
self.targets.is_empty()
}
}
/// One in-flight inbound request of a peer; released on drop.
struct PeerSlot {
inflight: Arc<Mutex<HashMap<EndpointId, usize>>>,
+219 -43
View File
@@ -17,33 +17,53 @@ use crate::error::{MusicDhtError, Result};
use crate::podcast::PodcastId;
use crate::routing::{NodeContact, NodeId};
use super::MAX_PODCAST_DHT_RECORDS;
use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records};
/// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1
/// (pre-release) stored one payload per key; its replica table is dropped on
/// open because the cache is replaceable. Own publications are kept.
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2;
/// Maximum records (cards and tombstones) indexed under one key. A full key
/// admits no new `(podcast_id, publisher)` until a record expires: existing
/// records are never evicted for newcomers, and their owners keep refreshing
/// them in place.
pub const MAX_PODCAST_RECORDS_PER_KEY: usize = 256;
/// Maximum tombstones newly indexed under one key. A tombstone is admitted
/// under a text key only in place of a record of the same
/// `(podcast_id, publisher)` already indexed there, so this budget applies to
/// the podcast-id key, where tombstones of unknown cards land.
pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = 32;
/// Maximum records (cards and tombstones) indexed under one key. It equals
/// [`MAX_PODCAST_DHT_RECORDS`], so one `FindValue` response carries every
/// record of a key: nothing admitted is unreachable.
///
/// A full key admits a newcomer by evicting a random placement from its
/// younger half (by first indexing time). The older half is never evicted
/// for newcomers, so cards that were there before a flood stay, and the
/// younger half stays contestable, so nobody who stored first holds the key
/// for good.
pub const MAX_PODCAST_RECORDS_PER_KEY: usize = MAX_PODCAST_DHT_RECORDS;
/// Maximum tombstones indexed under one key, including tombstones that
/// replaced their owner's card in place. Over budget, a random tombstone of
/// another owner from the younger half of the key's tombstones is evicted.
/// It equals [`MAX_PODCAST_RESPONSE_TOMBSTONES`], so every tombstone of a key
/// is served.
pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = MAX_PODCAST_RESPONSE_TOMBSTONES;
/// Maximum tombstones in one `FindValue` response; cards fill the rest.
pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32;
/// Maximum `(podcast_id, publisher)` records one publisher device keeps in a
/// replica.
/// replica. A publisher over it is refused (nothing is evicted for it).
pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256;
/// Maximum `(podcast_id, publisher)` records in a replica cache.
/// Maximum `(podcast_id, publisher)` records in a replica cache. A full cache
/// admits a new record by evicting random records from the younger half of
/// the cache (by first indexing time), like a full key.
pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096;
/// Maximum total encoded size of the records in a replica cache.
/// Maximum total encoded size of the records in a replica cache, enforced
/// like [`MAX_PODCAST_CACHE_RECORDS`].
pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024;
/// Maximum persisted overlay contacts; the least recently seen are dropped.
pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024;
/// Maximum records evicted from the cache to admit one record; when that is
/// not enough room, the record is refused and nothing is evicted.
const MAX_EVICTIONS_PER_RECORD: usize = 16;
const _: () = assert!(
MAX_PODCAST_RECORDS_PER_KEY <= MAX_PODCAST_DHT_RECORDS
&& MAX_PODCAST_TOMBSTONES_PER_KEY <= MAX_PODCAST_RESPONSE_TOMBSTONES
&& MAX_PODCAST_RESPONSE_TOMBSTONES <= MAX_PODCAST_DHT_RECORDS,
"every record admitted under a key must fit one FindValue response"
);
const SCHEMA: &str = "
CREATE TABLE IF NOT EXISTS podcast_dht_records (
@@ -227,7 +247,8 @@ impl PodcastDhtDatabase {
/// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale
/// replicas elsewhere can be overruled) and cards containing every token
/// in `filter_tokens` up to `limit` records in all. Records indexed longest
/// come first. Cards precede tombstones.
/// come first. Cards precede tombstones. With the default limits and
/// `limit` = [`MAX_PODCAST_DHT_RECORDS`], every record of the key fits.
pub async fn records_by_key(
&self,
key: PodcastDhtKey,
@@ -475,10 +496,10 @@ fn store_one(
};
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
if replace
&& !admits_record(
&& !make_room_for_record(
conn,
limits,
&publisher,
(&podcast_id, &publisher),
existing.is_none(),
existing_bytes,
payload.len(),
@@ -536,6 +557,14 @@ fn store_one(
clamp_i64(record.expires_at_ms()),
],
)?;
// A card converted to a tombstone in place counts against the
// tombstone budget of each key it stays under, like a new one.
if record.is_tombstone() {
for key in &indexed {
let key = PodcastDhtKey::from_bytes(*key);
make_room_under_key(conn, limits, &key, true, (&podcast_id, &publisher), now, 0)?;
}
}
}
// New placements. A tombstone only takes a new place under the podcast-id
@@ -549,7 +578,15 @@ fn store_one(
if record.is_tombstone() && *key != id_key {
continue;
}
if !admits_placement(conn, limits, key, record.is_tombstone(), now)? {
if !make_room_under_key(
conn,
limits,
key,
record.is_tombstone(),
(&podcast_id, &publisher),
now,
1,
)? {
continue;
}
conn.execute(
@@ -596,58 +633,197 @@ fn store_one(
Ok(written)
}
/// Whether the cache may hold the new version of a record: per-publisher and
/// global record counts for a new owner, and the global byte budget.
fn admits_record(
/// Makes room in the cache for the new version of a record of `owner`.
///
/// * A publisher at [`StoreLimits::per_publisher`] gets no new record.
/// * When the record count (for a new owner) or the byte budget would be
/// exceeded, random records of other owners from the younger half of the
/// cache (by first indexing time) are evicted, at most
/// [`MAX_EVICTIONS_PER_RECORD`]. If that is not enough, nothing is evicted
/// and the record is refused.
fn make_room_for_record(
conn: &Connection,
limits: &StoreLimits,
publisher: &str,
owner: (&str, &str),
new_owner: bool,
previous_bytes: usize,
new_bytes: usize,
now: i64,
) -> rusqlite::Result<bool> {
if new_bytes > limits.bytes {
return Ok(false);
}
if new_owner {
let own: i64 = conn.query_row(
"SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2",
params![owner.1, now],
|row| row.get(0),
)?;
if own.max(0) as usize >= limits.per_publisher {
return Ok(false);
}
}
let (records, bytes): (i64, i64) = conn.query_row(
"SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records
WHERE expires_at_ms > ?1",
[now],
|row| Ok((row.get(0)?, row.get(1)?)),
)?;
let bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes);
if bytes.saturating_add(new_bytes) > limits.bytes {
return Ok(false);
}
if !new_owner {
let mut records = records.max(0) as usize;
let mut bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes);
let fits = |records: usize, bytes: usize| {
bytes.saturating_add(new_bytes) <= limits.bytes && (!new_owner || records < limits.records)
};
if fits(records, bytes) {
return Ok(true);
}
if records.max(0) as usize >= limits.records {
// Candidates: the younger half of the other owners' records, shuffled.
let others = records.saturating_sub(usize::from(!new_owner));
let mut statement = conn.prepare(
"SELECT podcast_id, publisher, payload_bytes FROM (
SELECT r.podcast_id, r.publisher, r.payload_bytes,
(SELECT MIN(k.first_stored_at_ms) FROM podcast_dht_keys k
WHERE k.podcast_id = r.podcast_id AND k.publisher = r.publisher) AS since
FROM podcast_dht_records r
WHERE r.expires_at_ms > ?1 AND NOT (r.podcast_id = ?2 AND r.publisher = ?3)
ORDER BY since DESC, random()
LIMIT ?4)
ORDER BY random() LIMIT ?5",
)?;
let candidates = statement
.query_map(
params![
now,
owner.0,
owner.1,
others.div_ceil(2) as i64,
MAX_EVICTIONS_PER_RECORD as i64
],
|row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, i64>(2)?,
))
},
)?
.collect::<rusqlite::Result<Vec<_>>>()?;
let mut evict = Vec::new();
for (podcast_id, publisher, payload_bytes) in candidates {
if fits(records, bytes) {
break;
}
records = records.saturating_sub(1);
bytes = bytes.saturating_sub(payload_bytes.max(0) as usize);
evict.push((podcast_id, publisher));
}
if !fits(records, bytes) {
return Ok(false);
}
let own: i64 = conn.query_row(
"SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2",
params![publisher, now],
|row| row.get(0),
)?;
Ok((own.max(0) as usize) < limits.per_publisher)
for (podcast_id, publisher) in evict {
delete_owner(conn, &podcast_id, &publisher)?;
}
Ok(true)
}
/// Whether `key` has room for one more placement. Existing placements are
/// never evicted.
fn admits_placement(
/// Makes room under `key` for `incoming` more placements (0 when an existing
/// placement of `owner` was converted to a tombstone in place) and returns
/// whether the key has room.
///
/// Over [`StoreLimits::per_key`], or over [`StoreLimits::tombstones_per_key`]
/// for a tombstone, a random placement of another owner from the younger
/// half of the key (by first indexing time; only tombstones for the
/// tombstone budget) is evicted. The older half is never evicted for a
/// newcomer.
fn make_room_under_key(
conn: &Connection,
limits: &StoreLimits,
key: &PodcastDhtKey,
tombstone: bool,
owner: (&str, &str),
now: i64,
incoming: usize,
) -> rusqlite::Result<bool> {
let (total, tombstones): (i64, i64) = conn.query_row(
"SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys
WHERE dht_key = ?1 AND expires_at_ms > ?2",
params![key.as_bytes().as_slice(), now],
|row| Ok((row.get(0)?, row.get(1)?)),
for _ in 0..=MAX_EVICTIONS_PER_RECORD {
let (total, tombstones): (i64, i64) = conn.query_row(
"SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys
WHERE dht_key = ?1 AND expires_at_ms > ?2",
params![key.as_bytes().as_slice(), now],
|row| Ok((row.get(0)?, row.get(1)?)),
)?;
let over_tombstones =
tombstone && tombstones.max(0) as usize + incoming > limits.tombstones_per_key;
let over_total = total.max(0) as usize + incoming > limits.per_key;
if !over_tombstones && !over_total {
return Ok(true);
}
let Some((podcast_id, publisher)) =
young_placement(conn, key, over_tombstones, owner, now)?
else {
return Ok(false);
};
conn.execute(
"DELETE FROM podcast_dht_keys
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
params![key.as_bytes().as_slice(), podcast_id, publisher],
)?;
let placed: bool = conn.query_row(
"SELECT EXISTS (SELECT 1 FROM podcast_dht_keys
WHERE podcast_id = ?1 AND publisher = ?2)",
params![podcast_id, publisher],
|row| row.get(0),
)?;
if !placed {
delete_owner(conn, &podcast_id, &publisher)?;
}
}
Ok(false)
}
/// A random unexpired placement of another owner than `owner` from the
/// younger half of `key` (only tombstones with `tombstones_only`).
fn young_placement(
conn: &Connection,
key: &PodcastDhtKey,
tombstones_only: bool,
owner: (&str, &str),
now: i64,
) -> rusqlite::Result<Option<(String, String)>> {
let filter = "dht_key = ?1 AND expires_at_ms > ?2 AND (?3 = 0 OR tombstone = 1)
AND NOT (podcast_id = ?4 AND publisher = ?5)";
let count: i64 = conn.query_row(
&format!("SELECT COUNT(*) FROM podcast_dht_keys WHERE {filter}"),
params![
key.as_bytes().as_slice(),
now,
tombstones_only,
owner.0,
owner.1
],
|row| row.get(0),
)?;
Ok((total.max(0) as usize) < limits.per_key
&& (!tombstone || (tombstones.max(0) as usize) < limits.tombstones_per_key))
if count <= 0 {
return Ok(None);
}
conn.query_row(
&format!(
"SELECT podcast_id, publisher FROM (
SELECT podcast_id, publisher FROM podcast_dht_keys WHERE {filter}
ORDER BY first_stored_at_ms DESC, random() LIMIT ?6)
ORDER BY random() LIMIT 1"
),
params![
key.as_bytes().as_slice(),
now,
tombstones_only,
owner.0,
owner.1,
(count as usize).div_ceil(2) as i64
],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.optional()
}
fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> {
+402 -78
View File
@@ -1,6 +1,8 @@
//! Unit tests of the podcast-discovery overlay: records, ordering, cache and
//! search aggregation. Multi-node behavior is in `tests/podcast_dht.rs`.
use std::collections::HashSet;
use federation_net::{NetworkId, SecretKey};
use crate::routing::NodeId;
@@ -584,50 +586,239 @@ async fn open_db_with(
.unwrap()
}
fn rust_card(index: usize, issued: u64) -> SignedPodcastRecord {
card_with(
spec(&format!("show-{index}.xml"), "Rust Radio"),
&numbered_secret(index),
1,
issued,
)
}
async fn store_all(
db: &store::PodcastDhtDatabase,
entries: Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>,
now: u64,
) -> usize {
let mut written = 0;
for chunk in entries.chunks(MAX_PODCAST_STORE_RECORDS) {
written += db.store(chunk.to_vec(), now).await.unwrap();
}
written
}
async fn count_records(temp: &tempfile::TempDir) -> i64 {
rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3"))
.unwrap()
.query_row("SELECT COUNT(*) FROM podcast_dht_records", [], |row| {
row.get(0)
})
.unwrap()
}
#[tokio::test]
async fn cache_full_key_keeps_its_records_and_refuses_newcomers() {
async fn cache_full_key_admits_newcomers_from_its_younger_half() {
let temp = tempfile::tempdir().unwrap();
let db = open_db(&temp).await;
let token_key = PodcastDhtKey::token(&network(), "rust");
let mut entries = Vec::new();
for index in 0..=MAX_PODCAST_RECORDS_PER_KEY {
// Later publishers expire later; that buys them nothing.
let record = card_with(
spec("show.xml", "Rust Radio"),
&numbered_secret(index),
let rust = PodcastDhtKey::token(&network(), "rust");
let half = MAX_PODCAST_RECORDS_PER_KEY / 2;
let older: Vec<_> = (0..half).map(|index| rust_card(index, NOW)).collect();
let younger: Vec<_> = (half..MAX_PODCAST_RECORDS_PER_KEY)
.map(|index| rust_card(index, NOW))
.collect();
let keyed = |records: &[SignedPodcastRecord]| -> Vec<_> {
records
.iter()
.map(|record| (vec![rust], record.clone()))
.collect()
};
assert_eq!(store_all(&db, keyed(&older), NOW).await, half);
assert_eq!(store_all(&db, keyed(&younger), NOW + 1).await, half);
// Newcomers keep getting in; each takes a random place of the younger
// half, the older half stays.
let mut newcomers = Vec::new();
for index in 0..half {
let newcomer = rust_card(1_000 + index, NOW);
assert_eq!(
db.store(vec![(vec![rust], newcomer.clone())], NOW + 2 + index as u64)
.await
.unwrap(),
1,
NOW + index as u64,
"a full key admits a newcomer"
);
entries.push((vec![token_key], record));
newcomers.push(newcomer);
let cached = db
.records_by_key(rust, Vec::new(), NOW + 2, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap();
assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY);
assert!(older.iter().all(|record| cached.contains(record)));
assert!(cached.contains(newcomers.last().unwrap()));
}
let first = entries[0].1.clone();
let last = entries[MAX_PODCAST_RECORDS_PER_KEY].1.clone();
// The first publisher arrived first; the rest a moment later.
db.store(entries[..1].to_vec(), NOW).await.unwrap();
for chunk in entries[1..].chunks(64) {
db.store(chunk.to_vec(), NOW + 1).await.unwrap();
}
let cached = db
.records_by_key(token_key, Vec::new(), NOW, usize::MAX)
.await
.unwrap();
assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY);
assert!(cached.contains(&first));
assert!(!cached.contains(&last));
assert_eq!(cached[0], first, "records indexed longest come first");
// The first publisher keeps refreshing its place.
// Owners refresh in place without evicting anybody.
let refresh = card_with(
spec("show.xml", "Rust Radio"),
spec("show-0.xml", "Rust Radio"),
&numbered_secret(0),
1,
NOW + 1_000,
);
assert_eq!(
db.store(vec![(vec![token_key], refresh)], NOW)
db.store(vec![(vec![rust], refresh.clone())], NOW + 1_000)
.await
.unwrap(),
1
);
let cached = db
.records_by_key(rust, Vec::new(), NOW + 1_000, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap();
assert_eq!(cached.len(), MAX_PODCAST_RECORDS_PER_KEY);
assert!(cached.contains(&refresh));
}
#[tokio::test]
async fn sybils_planted_first_cannot_lock_a_late_card_out_of_a_key() {
let temp = tempfile::tempdir().unwrap();
let db = open_db(&temp).await;
let rust = PodcastDhtKey::token(&network(), "rust");
// 128 throwaway identities, each with its own podcast titled like the
// victim's, fill the key first.
let sybils: Vec<_> = (0..MAX_PODCAST_DHT_RECORDS)
.map(|index| (vec![rust], rust_card(index, NOW)))
.collect();
store_all(&db, sybils, NOW).await;
let victim = card(&secret(1), 1);
assert_eq!(
db.store(vec![entry(&victim)], NOW + 1).await.unwrap(),
victim.dht_keys().len()
);
let tokens = vec!["rust".to_string(), "radio".to_string()];
let served = db
.records_by_key(rust, tokens.clone(), NOW + 1, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap();
assert!(served.len() <= MAX_PODCAST_DHT_RECORDS);
assert!(served.contains(&victim), "everything admitted is served");
let hits = rank_hits(aggregate_hits(served), "Rust Radio", &tokens);
assert!(
hits.iter()
.any(|hit| hit.podcast_id == *victim.podcast_id())
);
}
#[tokio::test]
async fn a_card_back_after_expiring_gets_into_a_cache_full_of_sybils() {
let limits = store::StoreLimits {
records: 16,
..store::StoreLimits::default()
};
let temp = tempfile::tempdir().unwrap();
let db = open_db_with(&temp, limits).await;
let victim = card(&secret(1), 1);
let id_key = PodcastDhtKey::podcast(&network(), victim.podcast_id());
db.store(vec![entry(&victim)], NOW).await.unwrap();
let sybils = |issued: u64| -> Vec<_> {
(0..16)
.map(|index| {
let record = rust_card(index, issued);
(record.dht_keys(), record)
})
.collect()
};
store_all(&db, sybils(NOW), NOW + 1).await;
assert_eq!(count_records(&temp).await, 16);
// The victim's device is offline longer than a card lives; the sybils
// keep refreshing theirs in place.
let later = NOW + PODCAST_CARD_TTL.as_millis() as u64 + 3_600_000;
store_all(&db, sybils(later), later).await;
assert!(
db.records_by_key(id_key, Vec::new(), later, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap()
.is_empty()
);
let back = card_with(spec("show.xml", "Rust Radio"), &secret(1), 1, later + 1);
assert!(db.store(vec![entry(&back)], later + 1).await.unwrap() > 0);
assert_eq!(
db.records_by_key(id_key, Vec::new(), later + 1, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap(),
vec![back]
);
assert_eq!(count_records(&temp).await, 16);
}
#[tokio::test]
async fn sybil_tombstones_planted_first_cannot_hide_a_revocation() {
let temp = tempfile::tempdir().unwrap();
let db = open_db(&temp).await;
let victim_card = card(&secret(1), 1);
let id_key = PodcastDhtKey::podcast(&network(), victim_card.podcast_id());
let rust = PodcastDhtKey::token(&network(), "rust");
// 64 sybils publish a card of the victim's podcast, then convert it to
// a tombstone in place, under the id key and the text keys.
for index in 0..64 {
let key = numbered_secret(index);
let sybil_card = card_with(spec("show.xml", "Rust Radio"), &key, 1, NOW);
db.store(vec![entry(&sybil_card)], NOW).await.unwrap();
let tombstone = PodcastTombstone::new(
network(),
key.public(),
sybil_card.podcast_id().clone(),
2,
NOW,
sybil_card.dht_keys(),
);
let tombstone =
SignedPodcastRecord::sign_tombstone(tombstone, |bytes| key.sign(bytes)).unwrap();
assert!(db.store(vec![entry(&tombstone)], NOW).await.unwrap() > 0);
}
let conn = rusqlite::Connection::open(temp.path().join("podcast-dht.sqlite3")).unwrap();
let tombstones = |key: &PodcastDhtKey| -> usize {
conn.query_row(
"SELECT COUNT(*) FROM podcast_dht_keys WHERE dht_key = ?1 AND tombstone = 1",
[key.as_bytes().as_slice()],
|row| row.get::<_, i64>(0),
)
.unwrap() as usize
};
for key in [id_key, rust] {
assert_eq!(
tombstones(&key),
MAX_PODCAST_TOMBSTONES_PER_KEY,
"in-place conversions count against the tombstone budget"
);
}
// The honest publisher publishes later, then unpublishes.
assert!(db.store(vec![entry(&victim_card)], NOW + 1).await.unwrap() > 0);
let victim_tombstone = tombstone(&secret(1), 2, NOW + 2);
assert!(
db.store(vec![entry(&victim_tombstone)], NOW + 2)
.await
.unwrap()
> 0
);
for key in [id_key, rust] {
assert!(tombstones(&key) <= MAX_PODCAST_TOMBSTONES_PER_KEY);
let served = db
.records_by_key(key, vec![], NOW + 2, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap();
assert!(
served.contains(&victim_tombstone),
"the revocation is served"
);
// It overrules a stale replica still holding the card.
let mut all = served;
all.push(victim_card.clone());
assert!(aggregate_hits(all).iter().all(|hit| {
hit.podcast_id != *victim_card.podcast_id()
|| !hit.publishers.contains(&secret(1).public())
}));
}
}
#[tokio::test]
@@ -663,8 +854,8 @@ async fn sybil_tombstones_cannot_suppress_a_card() {
written += db.store(chunk.to_vec(), NOW).await.unwrap();
}
assert_eq!(
written, MAX_PODCAST_TOMBSTONES_PER_KEY,
"only the id key's tombstone budget is used"
written, 256,
"each sybil tombstone takes a young sybil tombstone's place under the id key"
);
let by_id = db
@@ -726,36 +917,67 @@ async fn cache_enforces_publisher_record_and_byte_quotas() {
let temp = tempfile::tempdir().unwrap();
let db = open_db_with(&temp, limits).await;
let show = |index: usize, key: &SecretKey| {
entry(&card_with(
card_with(
spec(&format!("show-{index}.xml"), "Rust Radio"),
key,
1,
NOW,
))
)
};
let heavy = secret(1);
let stored: Vec<usize> = {
let mut out = Vec::new();
for index in 0..4 {
out.push(db.store(vec![show(index, &heavy)], NOW).await.unwrap());
}
out
};
assert!(stored[..3].iter().all(|written| *written > 0));
for index in 0..3 {
let record = show(index, &heavy);
assert!(
db.store(vec![entry(&record)], NOW + index as u64)
.await
.unwrap()
> 0
);
}
assert_eq!(
stored[3], 0,
db.store(vec![entry(&show(3, &heavy))], NOW + 3)
.await
.unwrap(),
0,
"the fourth podcast of one publisher is refused"
);
// A refresh of an admitted record is always accepted.
let refresh = card_with(spec("show-0.xml", "Rust Radio"), &heavy, 1, NOW + 1);
assert!(db.store(vec![entry(&refresh)], NOW).await.unwrap() > 0);
assert!(db.store(vec![show(10, &secret(2))], NOW).await.unwrap() > 0);
assert!(db.store(vec![show(11, &secret(3))], NOW).await.unwrap() > 0);
assert_eq!(
db.store(vec![show(12, &secret(4))], NOW).await.unwrap(),
0,
"the cache holds at most five records"
assert!(db.store(vec![entry(&refresh)], NOW + 4).await.unwrap() > 0);
assert!(
db.store(vec![entry(&show(10, &secret(2)))], NOW + 10)
.await
.unwrap()
> 0
);
assert!(
db.store(vec![entry(&show(11, &secret(3)))], NOW + 11)
.await
.unwrap()
> 0
);
assert_eq!(count_records(&temp).await, 5);
// The cache is full: every newcomer still gets in by evicting a random
// record of the younger half; the two oldest records always stay.
let rust = PodcastDhtKey::token(&network(), "rust");
for index in 0..6 {
let newcomer = show(20 + index, &secret(10 + index as u8));
assert!(
db.store(vec![entry(&newcomer)], NOW + 20 + index as u64)
.await
.unwrap()
> 0
);
assert_eq!(count_records(&temp).await, 5);
let cached = db
.records_by_key(rust, Vec::new(), NOW + 30, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap();
assert!(cached.contains(&refresh));
assert!(cached.contains(&show(1, &heavy)));
assert!(cached.contains(&newcomer));
}
let one = card(&secret(1), 1);
let limits = store::StoreLimits {
@@ -764,25 +986,34 @@ async fn cache_enforces_publisher_record_and_byte_quotas() {
};
let temp = tempfile::tempdir().unwrap();
let db = open_db_with(&temp, limits).await;
assert!(
db.store(vec![entry(&card(&secret(1), 1))], NOW)
.await
.unwrap()
> 0
);
assert!(
db.store(vec![entry(&card(&secret(2), 1))], NOW)
.await
.unwrap()
> 0
);
let id_key = PodcastDhtKey::podcast(&network(), one.podcast_id());
for (index, now) in [(1, NOW), (2, NOW + 1), (3, NOW + 2)] {
assert!(
db.store(vec![entry(&card(&secret(index), 1))], now)
.await
.unwrap()
> 0
);
}
// The byte budget holds two cards: the younger half of the two others
// (the second card) made room for the third.
assert_eq!(
db.store(vec![entry(&card(&secret(3), 1))], NOW)
db.records_by_key(id_key, Vec::new(), NOW + 2, MAX_PODCAST_DHT_RECORDS)
.await
.unwrap(),
0,
"the byte budget is full"
.unwrap()
.into_iter()
.map(|record| record.publisher())
.collect::<HashSet<_>>(),
HashSet::from([secret(1).public(), secret(3).public()])
);
// A record larger than the whole budget evicts nothing.
let limits = store::StoreLimits {
bytes: one.encoded_len() - 1,
..store::StoreLimits::default()
};
let temp = tempfile::tempdir().unwrap();
let db = open_db_with(&temp, limits).await;
assert_eq!(db.store(vec![entry(&one)], NOW).await.unwrap(), 0);
}
#[tokio::test]
@@ -931,6 +1162,71 @@ fn replicas_only_accept_keys_they_are_close_to() {
assert!(is_responsible(&own, &near[1..], &key));
}
fn contact_of(key: &SecretKey) -> NodeContact {
let peer = key.public();
NodeContact {
node_id: NodeId::from_endpoint(&peer),
peer_id: peer,
ticket: String::new(),
last_seen_ms: NOW,
}
}
#[test]
fn refused_placements_fall_back_to_the_next_closest_nodes() {
let key = PodcastDhtKey::token(&network(), "rust");
let contacts: Vec<NodeContact> = (0..3 * K)
.map(|index| contact_of(&numbered_secret(index)))
.collect();
let mut by_distance = contacts.clone();
by_distance.sort_by_key(|contact| distance(contact.node_id.as_bytes(), key.as_bytes()));
// This node is the farthest one: it is no replica and no fallback.
let own = by_distance.pop().unwrap();
let others: Vec<NodeContact> = by_distance.clone();
let targets = replica_targets(own.clone(), others.clone(), &key);
assert!(!targets.local);
let ids = |contacts: &[NodeContact]| -> Vec<EndpointId> {
contacts.iter().map(|contact| contact.peer_id).collect()
};
assert_eq!(ids(&targets.primary), ids(&by_distance[..K]));
let mut fallback = targets.fallback.clone();
fallback.reverse();
assert_eq!(ids(&fallback), ids(&by_distance[K..2 * K]));
// Two refusals of one placement go to the two next closest nodes;
// another record's refusal has its own fallbacks.
let mut fallbacks = HashMap::from([
((0, key), targets.fallback.clone()),
((1, key), targets.fallback),
]);
let retry = retry_plan(vec![(0, key), (0, key), (1, key)], &mut fallbacks);
let first = by_distance[K].peer_id;
let second = by_distance[K + 1].peer_id;
assert_eq!(retry.targets.len(), 2);
assert_eq!(
retry.targets[&first].1.keys().copied().collect::<Vec<_>>(),
vec![0, 1]
);
assert_eq!(
retry.targets[&second].1.keys().copied().collect::<Vec<_>>(),
vec![0]
);
assert_eq!(fallbacks[&(0, key)].len(), K - 2);
// With this node among the closest it keeps a local replica and is never
// its own fallback.
let targets = replica_targets(by_distance[0].clone(), others, &key);
assert!(targets.local);
assert_eq!(targets.primary.len(), K - 1);
assert!(
targets
.fallback
.iter()
.chain(&targets.primary)
.all(|contact| contact.peer_id != by_distance[0].peer_id)
);
}
#[test]
fn inbound_requests_are_limited_per_peer() {
let inflight = Arc::new(Mutex::new(HashMap::new()));
@@ -946,21 +1242,50 @@ fn inbound_requests_are_limited_per_peer() {
}
#[test]
fn lookups_keep_a_bounded_number_of_owners() {
fn lookups_give_each_response_a_bounded_share_of_owners() {
let mut winners = HashMap::new();
let id_key = PodcastDhtKey::podcast(&network(), card(&secret(1), 1).podcast_id());
let records: Vec<_> = (0..=MAX_PODCAST_LOOKUP_RECORDS)
.map(|index| {
card_with(
spec("show.xml", "Rust Radio"),
&numbered_secret(index),
1,
now_ms(),
)
})
.collect();
let first = records[0].clone();
merge_records(&mut winners, records, &network(), &id_key, &[]);
let response = |first: usize| -> Vec<SignedPodcastRecord> {
(first..first + MAX_PODCAST_DHT_RECORDS)
.map(|index| {
card_with(
spec("show.xml", "Rust Radio"),
&numbered_secret(index),
1,
now_ms(),
)
})
.collect()
};
// Four attacker responders answer first, each with 128 sybil owners.
for responder in 0..4 {
merge_records(
&mut winners,
response(10_000 + responder * MAX_PODCAST_DHT_RECORDS),
&network(),
&id_key,
&[],
);
}
assert_eq!(winners.len(), 4 * MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE);
// An honest replica answering later still contributes its owners.
let honest = response(0);
let victim = honest[0].clone();
merge_records(&mut winners, honest, &network(), &id_key, &[]);
let owner = (victim.podcast_id().clone(), victim.publisher());
assert_eq!(winners[&owner], victim);
assert_eq!(winners.len(), 5 * MAX_PODCAST_LOOKUP_OWNERS_PER_RESPONSE);
// The lookup as a whole stays bounded.
for responder in 0..8 {
merge_records(
&mut winners,
response(20_000 + responder * MAX_PODCAST_DHT_RECORDS),
&network(),
&id_key,
&[],
);
}
assert_eq!(winners.len(), MAX_PODCAST_LOOKUP_RECORDS);
// A newer record of a known owner still replaces its winner.
let newer = card_with(
@@ -970,7 +1295,6 @@ fn lookups_keep_a_bounded_number_of_owners() {
now_ms(),
);
merge_records(&mut winners, vec![newer.clone()], &network(), &id_key, &[]);
let owner = (first.podcast_id().clone(), first.publisher());
assert_eq!(winners[&owner], newer);
}
+90
View File
@@ -18,6 +18,8 @@
//! and is never asked for records or to store them.
//! * `sync_shares` stays fail-safe for revocation and holds back URLs that
//! look private (single node).
//! * Contacts a peer returns are never trusted: fabricated, unreachable
//! contacts do not enter the routing table (two nodes).
use std::path::Path;
use std::sync::Arc;
@@ -566,3 +568,91 @@ async fn single_node_scenario() {
a.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn fabricated_contacts_never_join_the_overlay() {
tokio::time::timeout(TEST_TIMEOUT, fabricated_contacts_scenario())
.await
.unwrap();
}
/// K contacts whose node ids are genuine hashes of their endpoint ids but
/// whose tickets carry no address, as a malicious peer could return them.
fn fabricated_contacts(network: NetworkId) -> Vec<NodeContact> {
(0..music_dht::K)
.map(|index| {
let peer = federation_net::SecretKey::from_bytes(&[index as u8 + 101; 32]).public();
let ticket = federation_net::PeerTicket {
ticket_version: federation_net::TICKET_VERSION,
protocol_version: 1,
network_id: network,
schema_id: federation_net::SchemaId::from_bytes([0; 32]),
endpoint_addr: federation_net::EndpointAddr::new(peer),
};
NodeContact {
node_id: NodeId::from_endpoint(&peer),
peer_id: peer,
ticket: ticket.to_string(),
last_seen_ms: u64::MAX,
}
})
.collect()
}
async fn fabricated_contacts_scenario() {
let temp = tempfile::tempdir().unwrap();
let network = NetworkId::from_name("podcast-dht-fabricated-contacts");
let a = Node::start(&temp.path().join("a"), network).await;
// M speaks the overlay ALPN but answers every request with fabricated
// contacts.
let (m, m_events) = start_service(&temp.path().join("m"), network, true).await;
let mut acceptor = m.stream_acceptor(PODCAST_DHT_ALPN).unwrap();
let fabricated = fabricated_contacts(network);
let response = PodcastDhtResponse {
version: PODCAST_DHT_PROTOCOL_VERSION,
ok: true,
error: None,
nodes: fabricated.clone(),
records: Vec::new(),
stored: 0,
};
let payload = postcard::to_stdvec(&response).unwrap();
let server = tokio::spawn(async move {
while let Some(mut stream) = acceptor.accept().await {
let mut request = Vec::new();
let _ = AsyncReadExt::read_to_end(
&mut (&mut stream.recv).take(MAX_PODCAST_DHT_REQUEST_BYTES as u64),
&mut request,
)
.await;
let _ = stream.send.write_all(&payload).await;
let _ = stream.send.finish();
let _ = tokio::time::timeout(Duration::from_secs(2), stream.send.stopped()).await;
}
});
m.connect(a.service.ticket().await.unwrap()).await.unwrap();
wait_for("music DHT contact", async || {
(!a.service.known_peers().is_empty()).then_some(())
})
.await;
// Bootstrap learns M, which answered, and none of the contacts it made
// up: they are tried once and never answer.
a.dht.bootstrap().await.unwrap();
assert_eq!(a.dht.known_peers(), 1);
// A lookup treats them as candidates only.
let podcast_id = PodcastId::from_feed_url(&feed("nothing.xml"));
assert!(a.dht.lookup_podcast(&podcast_id).await.unwrap().is_none());
assert_eq!(a.dht.known_peers(), 1);
a.dht.bootstrap().await.unwrap();
assert_eq!(
a.dht.known_peers(),
1,
"fabricated contacts never count for replica responsibility"
);
server.abort();
a.stop().await;
m_events.abort();
m.shutdown().await.unwrap();
}