Compare commits
23
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e47348b21d | ||
|
|
8af4d2cbbc | ||
|
|
84c5eb571c | ||
|
|
987063d616 | ||
|
|
3046c3f924 | ||
|
|
ddea053e9d | ||
|
|
9c82f6d916 | ||
|
|
912527a671 | ||
|
|
c0def74115 | ||
|
|
71e3f1fa0a | ||
|
|
5a4d418d2b | ||
|
|
81ccd924d0 | ||
|
|
fef155d6ef | ||
|
|
2fcb5f1822 | ||
|
|
d27155b257 | ||
|
|
d2530dca6c | ||
|
|
b8d8b532f2 | ||
|
|
06ac3691d8 | ||
|
|
9709f5e2b5 | ||
|
|
e627be51d4 | ||
|
|
49ae6423f6 | ||
|
|
e22c470ea7 | ||
|
|
35115ecc98 |
No files matched your search
@@ -0,0 +1,3 @@
|
||||
[registries.gitea]
|
||||
index = "sparse+https://gt.hexor.cy/api/packages/Furumi/cargo/"
|
||||
credential-provider = "cargo:token"
|
||||
@@ -2,6 +2,8 @@ name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "**"
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
@@ -20,6 +22,7 @@ jobs:
|
||||
toolchain: "1.97"
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: python3 -m unittest discover -s scripts -p 'test_*.py'
|
||||
- run: cargo fmt --all -- --check
|
||||
- run: cargo check --workspace --all-targets --all-features
|
||||
- run: cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
name: Publish crates
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v[0-9]*"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: crates-io-publish
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_NET_RETRY: 10
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.97"
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: python3 -m unittest discover -s scripts -p 'test_*.py'
|
||||
- run: cargo fmt --all -- --check
|
||||
- run: cargo check --workspace --all-targets --all-features --locked
|
||||
- run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
|
||||
- run: cargo test --workspace --locked
|
||||
- name: Verify release packages once
|
||||
run: python3 scripts/publish_crates.py --dry-run
|
||||
|
||||
publish-crates-io:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.97"
|
||||
- name: Publish to crates.io without rebuilding
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
run: python3 scripts/publish_crates.py --no-verify
|
||||
|
||||
publish-gitea:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.97"
|
||||
- name: Publish mirror to Gitea without rebuilding
|
||||
env:
|
||||
CARGO_REGISTRIES_GITEA_TOKEN: ${{ secrets.CARGO_REGISTRIES_GITEA_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$CARGO_REGISTRIES_GITEA_TOKEN" ]; then
|
||||
echo "::error::Add the CARGO_REGISTRIES_GITEA_TOKEN repository Actions secret, then rerun this workflow."
|
||||
exit 1
|
||||
fi
|
||||
export CARGO_REGISTRIES_GITEA_TOKEN="Bearer $CARGO_REGISTRIES_GITEA_TOKEN"
|
||||
python3 scripts/publish_crates.py --registry gitea --no-verify
|
||||
+61
-3
@@ -12,6 +12,7 @@ application
|
||||
furumi-library
|
||||
├── local catalog, import, migrations
|
||||
├── playlists, likes, history, similarity persistence
|
||||
├── podcast store (podcasts.sqlite3)
|
||||
│
|
||||
music-dht
|
||||
├── Kademlia routing and iterative lookup
|
||||
@@ -41,9 +42,30 @@ metadata import, artwork resolution, playlists, likes, listening history, and
|
||||
similarity index storage. Connections enable foreign keys, WAL, and a bounded
|
||||
busy timeout so independently running clients can safely share the database.
|
||||
|
||||
Podcasts use a separate store, `furumi_library::podcasts::PodcastStore`, in
|
||||
its own `podcasts.sqlite3` file next to the library database. It implements
|
||||
the podcast contract of `music_dht::podcast` for SQLite clients: the episode
|
||||
catalog refreshed from RSS, the subscription, share and progress registers,
|
||||
grow-only listening sessions, feed aliases and local-only fetch state.
|
||||
Local actions, received operations and snapshot sections share one apply
|
||||
path built on the `music-dht` materializers and merge functions, so the store
|
||||
converges independently of delivery order. Migrations follow
|
||||
`PRAGMA user_version`; the prototype podcast database is imported and its
|
||||
tables are kept as `legacy_*`; newer or foreign files are refused unchanged.
|
||||
`TrackItem.podcast` marks episode queue items, which music-only code must
|
||||
skip. An episode queue item is identified by its episode id
|
||||
(`TrackItem::key`, a typed source identity: library row, federated owner and
|
||||
item id, or episode id), never by the numeric id: episodes use the fixed
|
||||
sentinel id, `release_id` and artist id -1, which carries no identity and
|
||||
only keeps code written for library tracks (`id >= 0`) away from them.
|
||||
Received playback items are converted with `TrackItem::from_playback_track`,
|
||||
which never copies the wire ids (0 for episodes and private placeholders). See the "SQLite reference store" section
|
||||
of [PODCASTS.md](PODCASTS.md).
|
||||
|
||||
Filesystem scanning schedules, playback, federation lifecycle, and UI state
|
||||
remain application responsibilities. Database and metadata work must stay off
|
||||
interactive and async reactor threads.
|
||||
interactive and async reactor threads. RSS fetching and refresh scheduling are
|
||||
application work as well.
|
||||
|
||||
## Transport: federation-net
|
||||
|
||||
@@ -210,8 +232,44 @@ refused, and their playback commands are ignored while the queue contains
|
||||
episodes. Jam snapshots replace episodes with opaque placeholders. Progress,
|
||||
subscription and share registers order by a Lamport generation before
|
||||
wall-clock `hlc_ms`, so a device with a fast clock cannot undo an observed
|
||||
seek, restart, unsubscribe or share decision. Public podcast discovery is a future
|
||||
separate overlay; the music DHT schema and `federation-net` are unchanged.
|
||||
seek, restart, unsubscribe or share decision. The music DHT schema and
|
||||
`federation-net` are unchanged.
|
||||
|
||||
Public podcast discovery is a third schema-independent Kademlia overlay,
|
||||
`music_dht::podcast_dht` (ALPN `furumi-fd/podcast-dht/1`, version 1,
|
||||
capability `podcast_dht`), built like the similarity-routing overlay. A
|
||||
device publishes a card only for podcasts whose merged registers satisfy
|
||||
`effective_share` (SQLite clients read the merged registers from
|
||||
`PodcastStore`); `sync_shares` reconciles publications declaratively and
|
||||
nothing is sent for a podcast the device never published. Own cards persisted
|
||||
by an earlier run are neither republished nor served until the first
|
||||
`sync_shares` of the process, so a stale cache cannot revive a share revoked
|
||||
elsewhere while the device was offline. A card is the
|
||||
public projection of a podcast (feed URL and channel metadata, no personal
|
||||
state), signed by the device's iroh identity over a domain-separated
|
||||
canonical encoding. Only the origin URL the id hashes from is bound to the
|
||||
podcast; a moved `feed_url` is an unverified publisher claim that search never
|
||||
offers as the podcast's feed. Cards live under the podcast-id key and one key
|
||||
per normalized title/author token; unpublishing writes a signed tombstone of
|
||||
the next revision that supersedes lower and equal revisions on replicas that
|
||||
hold it, for the keys it lists, and outlives every older card. Only a record's
|
||||
publisher may store it; replicas keep one payload per publisher and podcast,
|
||||
admit placements only for keys they are close to (counting only contacts that
|
||||
answered or authenticated, never ones another peer suggested), within per-key,
|
||||
per-publisher and total budgets. A key holds no more records than one response
|
||||
carries; a full key or cache admits a newcomer by evicting a random record from
|
||||
its younger half, so the older half survives floods and capacity is never
|
||||
first-come-forever (over their budget, tombstones only displace other tombstones). Records use the music DHT's
|
||||
active/tombstone TTLs and republish cadence, are verified on every receive and
|
||||
cache read, and are kept in a separate, replaceable SQLite cache. Search
|
||||
filters on every replica by all query tokens, merges cards of one podcast
|
||||
across publisher devices, and returns typed `podcast` hits that clients merge
|
||||
with music results; publisher counts are not a trust signal. The publisher key
|
||||
is the device's transport identity, so a card is linkable to the device's
|
||||
network address and music-DHT records. Applications advertise the
|
||||
`podcast_dht` capability only when they run the overlay; capability data only
|
||||
helps skip peers, it never authorizes. See the "Public discovery" section of
|
||||
[PODCASTS.md](PODCASTS.md).
|
||||
|
||||
Music discovery identifies an owner and content; it does not force all domain
|
||||
traffic through DHT messages. `music-dht` exposes the underlying authenticated
|
||||
|
||||
+80
-1
@@ -1,6 +1,54 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased — podcast contract
|
||||
## v0.6.0 — 2026-11-20
|
||||
|
||||
Releases `music-dht 0.6.0` and `furumi-library 0.3.0`.
|
||||
`federation-net` remains at `0.3.1`.
|
||||
|
||||
- Tag-driven CI verifies release packages once, then publishes to crates.io
|
||||
and the Furumi Gitea Cargo mirror in parallel jobs without recompiling.
|
||||
Already published versions are skipped independently in each registry.
|
||||
Authentication uses `CARGO_REGISTRY_TOKEN` and `CARGO_REGISTRIES_GITEA_TOKEN`.
|
||||
|
||||
### Podcast discovery (`podcast_dht`)
|
||||
|
||||
- `music-dht 0.6.0`: new `music_dht::podcast_dht` overlay for public podcast
|
||||
discovery on its own ALPN `furumi-fd/podcast-dht/1` (version 1) and its own
|
||||
SQLite cache (schema 2): signed `PodcastCard`s and `PodcastTombstone`s
|
||||
(`furumi-podcast-card:v1` / `furumi-podcast-card-tombstone:v1` canonical
|
||||
signing input), monotonic revisions with a total tie-break, text and id
|
||||
keys, `PodcastDht::{publish, unpublish, sync_shares, republish, search,
|
||||
lookup_podcast}` gated by `effective_share`, typed search hits
|
||||
(`PodcastSearchHit`, `ClaimedFeedUrl`, `FederatedSearchHit`,
|
||||
`SearchHitKind`, `merge_search_results`) and the advisory
|
||||
`feed_url_privacy_warning`.
|
||||
- Replicas accept records only from their publisher, store one payload per
|
||||
`(podcast_id, publisher)`, admit placements only for keys they are close to
|
||||
(counting verified contacts only) and within per-key, per-publisher and
|
||||
total budgets. A key never holds more than one response carries; full keys
|
||||
and caches admit newcomers by evicting random records from their younger
|
||||
half. Lookups give each response a bounded share of owners. Search offers
|
||||
only the URL the podcast id hashes from; moved-feed URLs are reported as
|
||||
unverified claims.
|
||||
- `sync_shares` is fail-safe for revocation (per-podcast `failed`) and holds
|
||||
back cards with privacy warnings until confirmed (`held_back`,
|
||||
`PodcastShareState::privacy_confirmed`).
|
||||
- Own cards persisted by an earlier run are neither republished (by
|
||||
`maintenance` or `republish`) nor served to lookups until the first
|
||||
`sync_shares` of the process, so a share revoked on another trusted device
|
||||
while this one was offline is not re-sent on startup. That first call also
|
||||
re-sends unchanged cards; tombstones are always re-sent.
|
||||
- `music-dht 0.6.0`: capability manifests can advertise `podcast_dht = 1`
|
||||
(`capabilities::PODCAST_DHT_ID`, `CapabilityManifest::with_podcast_dht`,
|
||||
`CapabilityManifest::protocol_version`,
|
||||
`CapabilityManifest::supports_podcast_dht`). It is opt-in:
|
||||
`CapabilityManifest::frid()` does not include it.
|
||||
- `music-dht 0.6.0`: new `MusicDhtError::PodcastPublication` variant (the enum
|
||||
is `#[non_exhaustive]`).
|
||||
- The music DHT (`music-dht-v5`), `federation-net` and device-sync protocol v4
|
||||
are unchanged. See "Public discovery (podcast_dht)" in `PODCASTS.md`.
|
||||
|
||||
### Podcast contract and storage
|
||||
|
||||
- `music-dht 0.6.0` (breaking): new `music_dht::podcast` module with the shared
|
||||
podcast contract: canonical `FeedUrl`, `PodcastId`/`EpisodeId` derivation
|
||||
@@ -18,6 +66,37 @@
|
||||
Jam redaction. New struct fields and enum variants break exhaustive
|
||||
construction and matching in clients.
|
||||
- `furumi-library 0.3.0`: follows `music-dht 0.6.0`, whose types it re-exposes.
|
||||
- `furumi-library 0.3.0`: new `furumi_library::podcasts::PodcastStore`, the
|
||||
SQLite reference store of the podcast contract in its own
|
||||
`podcasts.sqlite3`: `user_version` migrations that import the prototype
|
||||
podcast database (tables kept as `legacy_*`, unimportable rows in
|
||||
`legacy_quarantine`) and refuse newer or foreign files unchanged; podcasts,
|
||||
feed aliases, seasons, episodes, subscription/share/progress registers,
|
||||
listening sessions and local-only fetch state; local actions that return
|
||||
the `podcast_*` operations for the client's sync log; `apply_remote_op`,
|
||||
bounded snapshot export and merge; RSS refresh ingestion with stable
|
||||
episode ids; UI queries. Enables the `music-dht` `podcast-feed` feature.
|
||||
- `furumi-library 0.3.0` (breaking): `TrackItem.podcast:
|
||||
Option<PodcastEpisodeRef>` and `TrackItem::podcast_episode`/`is_podcast`.
|
||||
Exhaustive `TrackItem` literals in clients need the new field; music-only
|
||||
code must skip podcast items. Podcast items use the fixed sentinel
|
||||
`TrackItem::NOT_A_LIBRARY_ROW` (-1) as `id`, `release_id` and artist id,
|
||||
so existing `id < 0` / `release_id < 0` checks already treat them as
|
||||
non-library items; the sentinel carries no identity. New
|
||||
`TrackItem::key` / `TrackItemKey` (`lib:<id>`, `fed:<owner>:<item_id>`,
|
||||
`ep:<episode id>`), the source identity: episodes must be keyed by it (or
|
||||
`TrackItem.podcast`), never by `id`; content-first music matching stays
|
||||
as it is. New `TrackItem::from_playback_track` / `PlaybackItem` convert
|
||||
received playback items without copying wire ids (0 for episodes and Jam
|
||||
private placeholders). The library database is unchanged.
|
||||
- `furumi-library 0.3.0`: prototype import collapses rows of one GUID-less
|
||||
item under several enclosure URLs, and the first refresh re-keys imported
|
||||
GUID-less episodes without personal state to the ids the current feed
|
||||
gives, so they match devices that never had the prototype; when another
|
||||
episode (for example one created by a remote operation) already holds
|
||||
that id, the imported copy is deleted instead of kept as a duplicate.
|
||||
Fetch-state
|
||||
writes are atomic across threads and processes.
|
||||
- See `PODCASTS.md` for the normative contract.
|
||||
|
||||
## v0.4.1 — 2026-09-02
|
||||
|
||||
Generated
+1
@@ -942,6 +942,7 @@ dependencies = [
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
|
||||
+520
-6
@@ -10,8 +10,9 @@ live in `music_dht::podcast` (pure types and functions) and
|
||||
`music_dht::device_sync` (wire DTOs and gating helpers). Clients must use
|
||||
them instead of local copies.
|
||||
|
||||
Public discovery (signed podcast cards in a separate DHT overlay) is not part
|
||||
of this version of the contract; it will be specified with the overlay.
|
||||
Public discovery of podcasts a user explicitly shared uses signed podcast
|
||||
cards in a separate DHT overlay, `music_dht::podcast_dht`; see
|
||||
[Public discovery (podcast_dht)](#public-discovery-podcast_dht).
|
||||
|
||||
## Responsibilities
|
||||
|
||||
@@ -19,7 +20,8 @@ of this version of the contract; it will be specified with the overlay.
|
||||
| --- | --- |
|
||||
| `music_dht::podcast` | ids, URL canonicalization, no-GUID rule, metadata DTOs and bounds, merge functions, RSS/Atom parser (feature `podcast-feed`) |
|
||||
| `music_dht::device_sync` | sync operation kinds, snapshot sections, `PlaybackSource`, version gating, Jam redaction |
|
||||
| Client storage (`furumi-library`, web PostgreSQL) | persistence and migrations implementing the rules below |
|
||||
| `music_dht::podcast_dht` | public podcast cards and tombstones, their signatures, replication, cache and search, privacy warnings for feed URLs |
|
||||
| Client storage (`furumi_library::podcasts::PodcastStore`, web PostgreSQL) | persistence and migrations implementing the rules below (SQLite reference: [SQLite reference store](#sqlite-reference-store)) |
|
||||
| Client runtime | HTTP fetching, conditional requests, scheduling, audio buffering and decoding, UI |
|
||||
|
||||
Audio is always fetched from the publisher. It is never imported into the
|
||||
@@ -32,7 +34,7 @@ offered for P2P transfer. RSS refreshes fetch metadata only.
|
||||
| --- | --- | --- | --- |
|
||||
| Subscription state, channel metadata | `podcast_subscription_set` + snapshot | never | |
|
||||
| Feed aliases | `podcast_feed_aliases_added`, `podcast_subscription_set` + snapshot | never | |
|
||||
| Share decision | `podcast_share_set` + snapshot | only the published card (future overlay) | |
|
||||
| Share decision | `podcast_share_set` + snapshot | only the signed card while `effective_share` holds (`podcast_dht`) | |
|
||||
| Episode progress | `podcast_progress_set` + snapshot | never | |
|
||||
| Listening sessions, play counts | `podcast_listen_recorded` + snapshot | never | |
|
||||
| Episode catalog | each device parses RSS itself; `PodcastEpisodeRef` travels with progress and queue items | never | |
|
||||
@@ -498,6 +500,371 @@ Jam participants are outside the trusted-device group.
|
||||
private item. States that drop, duplicate or inject private items are
|
||||
rejected. Participants may still pause, seek and reorder music.
|
||||
|
||||
## Public discovery (podcast_dht)
|
||||
|
||||
`music_dht::podcast_dht` publishes podcasts the user explicitly shared so
|
||||
that peers outside the user's trusted-device group can find and subscribe to
|
||||
them. It is a separate, schema-independent Kademlia overlay built like
|
||||
`similarity_dht`: its own ALPN, version, routing contacts and SQLite cache.
|
||||
The music DHT (`music-dht-v5`) and `federation-net` are unchanged. Every
|
||||
device of a user publishes under its own identity; search merges the cards
|
||||
of one podcast by `podcast_id`.
|
||||
|
||||
| Item | Value |
|
||||
| --- | --- |
|
||||
| ALPN | `furumi-fd/podcast-dht/1` (schema-independent stream protocol) |
|
||||
| Wire and record version | `PODCAST_DHT_PROTOCOL_VERSION = 1` |
|
||||
| Capability manifest | `podcast_dht = 1`, opt-in (`CapabilityManifest::with_podcast_dht`, `supports_podcast_dht`) |
|
||||
| Cache | separate SQLite file, `PRAGMA user_version = 2`; a newer schema is refused, the pre-release version 1 replica table is dropped |
|
||||
| Card TTL | `ACTIVE_RECORD_TTL` of the music DHT (12 h) |
|
||||
| Tombstone TTL | `TOMBSTONE_TTL` of the music DHT (24 h) |
|
||||
| Republish | `DEFAULT_REPUBLISH_INTERVAL` (10 min), expiry sweep `DEFAULT_EXPIRE_INTERVAL` (1 min) |
|
||||
| Clock skew | `issued_at_ms` at most 5 min in the future |
|
||||
|
||||
### Card
|
||||
|
||||
A card is the public projection of one podcast by one publisher device. It
|
||||
is built from the subscription with `PodcastCardSpec::from_subscription`,
|
||||
which reads only the podcast id, the feed URLs and the channel metadata. It
|
||||
never contains progress, listening history, play counts, subscription or
|
||||
share registers (`generation`, `hlc_ms`, `op_id`), device names, local paths,
|
||||
HTTP validators or episodes.
|
||||
|
||||
| Field | Bound |
|
||||
| --- | --- |
|
||||
| `version`, `network_id`, `publisher` | 1, the federation network, the device's iroh endpoint key |
|
||||
| `podcast_id` | must equal `PodcastId::from_feed_url(id_feed_url or feed_url)` |
|
||||
| `feed_url` | canonical `FeedUrl` the publisher fetches the feed from |
|
||||
| `id_feed_url` | the URL the id was derived from, present only when it differs from `feed_url` (feed moved) |
|
||||
| `title` | 1..300 characters, no control characters |
|
||||
| `author` | optional, ≤ 300 characters, no control characters |
|
||||
| `description` | optional, ≤ 1 000 characters, only `\n`/`\t` controls |
|
||||
| `image_url`, `link` | optional absolute `http(s)` URLs ≤ 2 048 bytes |
|
||||
| `language` | optional, ≤ 35 bytes of ASCII letters, digits and `-` |
|
||||
| `categories` | ≤ 8, each 1..64 characters, no control characters |
|
||||
| `episode_count`, `latest_episode_at_ms` | optional; the time is ≥ 0 |
|
||||
| `revision` | ≥ 1, monotonic per `(podcast_id, publisher)` |
|
||||
| `issued_at_ms`, `expires_at_ms` | `0 < issued < expires ≤ issued + 12 h` |
|
||||
|
||||
`PodcastCardSpec::bounded` truncates publisher data to these bounds. The
|
||||
whole signed record is at most 24 KiB postcard-encoded; the field bounds keep
|
||||
every valid card below it.
|
||||
|
||||
Only the **origin URL** (`PodcastCard::origin_feed_url`: `id_feed_url`, or
|
||||
`feed_url` when the feed has not moved) is bound to the podcast, because the
|
||||
id is its hash. When `id_feed_url` is present, `feed_url` is an unverified
|
||||
claim of the publisher that the feed moved: anyone can sign a card that
|
||||
carries an existing podcast's origin URL and an arbitrary `feed_url`. Search
|
||||
therefore never offers a claimed URL as the podcast's feed (see Search), and
|
||||
clients must not fetch it as that podcast or record it as an alias unless the
|
||||
feed at the origin URL itself redirects there.
|
||||
|
||||
### Tombstone
|
||||
|
||||
`(version, network_id, publisher, podcast_id, revision, issued_at_ms,
|
||||
expires_at_ms, index_keys)` with `expires_at_ms ≤ issued_at_ms + 24 h` and
|
||||
`index_keys` the sorted, distinct text keys of the retracted card (at most
|
||||
32, without the podcast-id key). It carries key hashes, not the title. A
|
||||
tombstone supersedes every card of the same `(podcast_id, publisher)` with a
|
||||
lower or equal revision and is itself retained and served until it expires.
|
||||
Because the tombstone TTL exceeds the card TTL, every card issued before a
|
||||
tombstone expires before it.
|
||||
|
||||
`index_keys` is self-asserted, so it never creates a placement by itself: a
|
||||
replica indexes a tombstone under a text key only in place of a record of the
|
||||
same `(podcast_id, publisher)` it already holds there (see Replica storage).
|
||||
|
||||
### Signing input
|
||||
|
||||
Both records are signed with the publisher's Ed25519 iroh identity over a
|
||||
canonical, language-independent encoding: the domain string, a `0x00` byte,
|
||||
then the fields in the order listed. Integers are little-endian (`u16`,
|
||||
`u32`, `u64`, `i64`); strings are a `u32` byte length followed by UTF-8;
|
||||
`Option` is `0x00`, or `0x01` followed by the value; lists are a `u32` count
|
||||
followed by the items; `network_id`, `publisher` and keys are their raw 32
|
||||
bytes.
|
||||
|
||||
* Card, domain `furumi-podcast-card:v1`: `version u16, network_id, publisher,
|
||||
podcast_id str, feed_url str, id_feed_url opt str, title str, author opt
|
||||
str, description opt str, image_url opt str, link opt str, language opt
|
||||
str, categories list of str, episode_count opt u32, latest_episode_at_ms
|
||||
opt i64, revision u64, issued_at_ms u64, expires_at_ms u64`.
|
||||
* Tombstone, domain `furumi-podcast-card-tombstone:v1`: `version u16,
|
||||
network_id, publisher, podcast_id str, revision u64, issued_at_ms u64,
|
||||
expires_at_ms u64, index_keys list of 32-byte keys`.
|
||||
|
||||
Unit tests pin BLAKE3 vectors of both encodings. On the wire, records are
|
||||
postcard `SignedPodcastRecord { body: Card | Tombstone, signature }`.
|
||||
|
||||
### Keys and placement
|
||||
|
||||
`key = BLAKE3("furumi-podcast-dht:key:v1" || 0x00 || network_id || kind ||
|
||||
material)` with kind `0x01` and the podcast id string for the id key, or
|
||||
kind `0x02` and a token for a text key. Tokens are the distinct
|
||||
`normalize_name` tokens of the title, then of the author, skipping tokens
|
||||
longer than 64 bytes, at most 32. A card belongs under the id key and its
|
||||
text keys (at most 33); a tombstone under the id key and its `index_keys`.
|
||||
The publisher sends each record to the `K` (8) overlay nodes closest to each
|
||||
of its keys, one `StoreBatch` per node listing the keys that node should hold.
|
||||
|
||||
### Ordering
|
||||
|
||||
For one `(podcast_id, publisher)` the greater record wins: higher
|
||||
`revision`; at an equal revision a tombstone beats a card; then later
|
||||
`issued_at_ms` (a republish); then later `expires_at_ms`; then the
|
||||
lexicographically greater signing input. This is a total order, so every
|
||||
replica and every lookup converge whatever the delivery order. A replica
|
||||
keeps only the winning record of each `(podcast_id, publisher)`; a replayed
|
||||
old revision is ignored. These fields only order one publisher's own records:
|
||||
they never rank or select records of different publishers.
|
||||
|
||||
Revisions: a changed card gets `max(previous + 1, now_ms)` (time-seeded, so
|
||||
revisions stay monotonic even if the local cache is lost); an unchanged card
|
||||
is re-signed with the same revision and a fresh lifetime on every republish;
|
||||
unpublishing writes a tombstone with `previous + 1`; publishing again later
|
||||
gets `max(tombstone + 1, now_ms)`.
|
||||
|
||||
### Wire protocol
|
||||
|
||||
One request per bidirectional stream on `furumi-fd/podcast-dht/1`: the
|
||||
requester writes one postcard `PodcastDhtRequest` and finishes its send side;
|
||||
the responder writes one postcard `PodcastDhtResponse` and finishes. Messages
|
||||
are not length-prefixed; the stream end delimits them. Requests above
|
||||
512 KiB and responses above 4 MiB are refused.
|
||||
|
||||
```text
|
||||
PodcastDhtRequest { version: u16, requester: NodeContact, operation: PodcastDhtOperation }
|
||||
PodcastDhtOperation = 0: FindNode { target: [u8; 32] }
|
||||
| 1: FindValue { key: PodcastDhtKey, filter_tokens: Vec<String> }
|
||||
| 2: StoreBatch { records: Vec<PodcastStoreEntry> }
|
||||
PodcastStoreEntry { keys: Vec<PodcastDhtKey>, record: SignedPodcastRecord }
|
||||
PodcastDhtResponse { version: u16, ok: bool, error: Option<String>,
|
||||
nodes: Vec<NodeContact>, records: Vec<SignedPodcastRecord>, stored: u32 }
|
||||
NodeContact { node_id: [u8; 32], peer_id: EndpointId, ticket: String, last_seen_ms: u64 }
|
||||
PodcastRecordBody = 0: Card(PodcastCard) | 1: Tombstone(PodcastTombstone)
|
||||
```
|
||||
|
||||
Fields are in postcard order. `PodcastDhtKey` is 32 raw bytes.
|
||||
|
||||
* `version` must be 1 in both directions. `requester.peer_id` must equal the
|
||||
authenticated transport peer, `node_id` must be `NodeId::from_endpoint`
|
||||
of it and its ticket must name the same endpoint and network; otherwise the
|
||||
request is refused. The responder learns the requester as an overlay
|
||||
contact (it is authenticated).
|
||||
* `nodes` in a response are unverified suggestions. A requester treats them
|
||||
as lookup or bootstrap candidates only and adds one to its routing table
|
||||
(and persisted contacts) only after it answered a request itself.
|
||||
* `FindNode` returns the `K` closest overlay contacts (the responder
|
||||
included).
|
||||
* `FindValue` returns the records under `key` (see Replica storage) and the
|
||||
`K` closest contacts. `filter_tokens` (≤ 8, each 1..64 bytes) are
|
||||
normalized tokens every returned card must be indexed under; tombstones are
|
||||
returned regardless. Requesters verify every record again and drop records
|
||||
that do not belong under `key` or miss a filter token.
|
||||
* `StoreBatch` carries 1..16 entries, each with 1..33 distinct keys. Every
|
||||
record must be published by the requester itself (replicas never forward
|
||||
records), pass `verify_at` (bounds, network, lifetime, clock skew,
|
||||
signature, size) and belong under each listed key; otherwise the whole
|
||||
batch is refused. Keys the responder is not responsible for are skipped.
|
||||
`stored` counts the `(key, record)` placements written. A publisher
|
||||
offers the placements of a batch answered with `stored = 0` (or that
|
||||
failed) once to the next closest node of each key.
|
||||
* A refused request gets `ok = false` with an `error` of at most 1 KiB and
|
||||
empty `nodes`, `records` and `stored = 0`; an accepted one has `ok = true`
|
||||
and no `error`. Requesters reject responses that violate this or exceed
|
||||
`K` nodes or 128 records.
|
||||
|
||||
### Replica storage
|
||||
|
||||
* A replica stores one payload per `(podcast_id, publisher)` (the winning
|
||||
record) and indexes it under the keys it holds. A newer version replaces
|
||||
it in place; placements the new version no longer belongs under (an old
|
||||
title's tokens, keys a tombstone does not list) are dropped, so on one
|
||||
replica a tombstone supersedes the card under every key it held.
|
||||
* A node accepts a placement only when fewer than `2 * K` of its verified
|
||||
overlay contacts that are not backed off are closer to the key than
|
||||
itself. Contacts only enter the routing table by answering a request or
|
||||
by sending one, so a peer cannot make a replica refuse keys by returning
|
||||
made-up contacts. A contact that fails three times in a row (each after
|
||||
its 30 s backoff) is removed.
|
||||
* A key holds at most 128 records, at most 32 of them tombstones: exactly
|
||||
what one `FindValue` response carries, so every admitted record is served.
|
||||
A tombstone is indexed under a text key only in place of its owner's
|
||||
record there, and such an in-place conversion counts against the
|
||||
tombstone budget like a new tombstone.
|
||||
* Full key: a new placement evicts a random placement of another owner from
|
||||
the younger half of the key (by first indexing time; only tombstones when
|
||||
the tombstone budget is full). The older half is never evicted for a
|
||||
newcomer, so records placed before a flood keep their place and their
|
||||
publishers refresh them in place. Capacity is never first-come-forever:
|
||||
a newcomer always gets a place, though placements in the younger half can
|
||||
be evicted again by later newcomers.
|
||||
* A replica holds at most 256 records per publisher (a publisher over it is
|
||||
refused), 4 096 records and 32 MiB of payload. A full cache admits a new
|
||||
record by evicting random records of other owners from its younger half,
|
||||
at most 16; when that is not enough room, nothing is evicted and the record
|
||||
is refused. At most 1 024 overlay contacts are persisted.
|
||||
* Device identities cost nothing, so these rules bound flooding but cannot
|
||||
prevent it: an attacker with many identities can keep part of the younger
|
||||
half of a key or cache busy, and a newcomer's place there can be evicted
|
||||
again until its publisher's next republish (every 10 min) puts it back.
|
||||
Protocol 1 has no proof of work or other identity cost.
|
||||
* `FindValue` answers with at most 128 records: at most 32 tombstones and
|
||||
the cards indexed longest first. Every record is verified again on every
|
||||
read.
|
||||
|
||||
### Publication
|
||||
|
||||
* `PodcastDht::publish(subscription, share, card)` refuses
|
||||
(`MusicDhtError::PodcastPublication`) unless `effective_share` holds, so a
|
||||
new or resumed subscription is never published without a fresh share
|
||||
decision. The UI calls it only after asking the user about any privacy
|
||||
warning, so the call is the user's consent.
|
||||
* `PodcastDht::sync_shares(states)` is declarative and idempotent: podcasts
|
||||
for which `effective_share` holds are published (when changed, new or past
|
||||
half their lifetime), every other podcast this device published is
|
||||
unpublished. Clients call it after applying share or subscription
|
||||
operations, so unsharing or unsubscribing on any trusted device revokes
|
||||
every device's own card once the registers converge, and a stale device
|
||||
cannot republish (its registers lose to the newer share decision).
|
||||
* Revocation is fail-safe: each podcast is handled on its own. A shared
|
||||
podcast whose card cannot be built or signed is reported in
|
||||
`PodcastDhtSyncStats::failed` and treated as not shared (an earlier
|
||||
publication is tombstoned); it never prevents other podcasts from being
|
||||
unpublished.
|
||||
* A card whose `PodcastCardSpec::privacy_warnings` is not empty is held back
|
||||
(`PodcastDhtSyncStats::held_back`, with the warnings) unless
|
||||
`PodcastShareState::privacy_confirmed` is set or this device already
|
||||
publishes the same `feed_url` and `id_feed_url`. A feed move to a URL that
|
||||
looks private therefore asks again; an earlier publication of a held-back
|
||||
podcast stays as it was.
|
||||
* SQLite clients build each `PodcastShareState` from
|
||||
`PodcastStore::subscription` (the merged register with its alias set, which
|
||||
always contains the origin URL) and `PodcastStore::share`; its `is_shared`
|
||||
equals `PodcastSummary::shared`. A card for the URL the device now fetches
|
||||
sets `feed_url` from the catalog and `id_feed_url` to the origin URL. The
|
||||
store does not persist `privacy_confirmed`; the client keeps that
|
||||
confirmation. See [SQLite reference store](#sqlite-reference-store).
|
||||
* `PodcastDht::unpublish(podcast_id)` writes the tombstone. Nothing is ever
|
||||
sent for a podcast this device never published: no tombstone, no id.
|
||||
* `PodcastDht::republish` (run by `maintenance` every 10 minutes and after
|
||||
a bootstrap that found new peers) re-signs active cards and re-sends
|
||||
unexpired tombstones.
|
||||
* Own cards persisted by an earlier run are not sent again until the client
|
||||
calls `sync_shares` in this process: before that, `republish` sends only
|
||||
tombstones and cards published with `publish` in this process, and
|
||||
`FindValue` answers leave those cards out (they stay in the cache). A share
|
||||
revoked on another trusted device while this one was offline is therefore
|
||||
never re-sent on startup from the stale cache. Clients call `sync_shares`
|
||||
on start, with the registers they hold, and again after device sync
|
||||
applies newer operations; the first call also re-sends the unchanged cards
|
||||
as they are (`PodcastDhtSyncStats::unchanged`, counted in `push`).
|
||||
`sync_shares` itself still trusts the registers it is given: an unchanged
|
||||
card is only re-signed when it used half its lifetime, so clients that may
|
||||
hold stale registers on start lose nothing by reconciling again after
|
||||
device sync.
|
||||
* The privacy heuristics (`feed_url_privacy_warning`) are advisory: query
|
||||
parameter names containing words such as `token`, `key`, `auth`, `secret`,
|
||||
`sig`/`signature`, `uid`, `user`, `pass`, `session`, `access`, `private`;
|
||||
known member-feed hosts; and long high-entropy query values (≥ 16
|
||||
characters) or path segments (≥ 32 characters). They check `feed_url` and
|
||||
`id_feed_url`, not artwork or web-page links. False positives and misses
|
||||
are possible.
|
||||
|
||||
### Search
|
||||
|
||||
* `PodcastDht::search(query, limit)` normalizes the query
|
||||
(`podcast_query_tokens`: distinct tokens, at most 8), looks up the text
|
||||
keys of the three longest tokens and asks every replica to return only
|
||||
cards indexed under **all** query tokens (tombstones are always returned).
|
||||
Results are verified again, the winning record per publisher is kept,
|
||||
tombstoned publishers are dropped and cards are merged by `podcast_id`
|
||||
into `PodcastSearchHit { podcast_id, feed_url, card, publishers,
|
||||
claimed_feed_urls }`.
|
||||
* `PodcastDht::lookup_podcast(podcast_id)` reads the id key.
|
||||
* `feed_url` is the origin URL; it is the only URL a client subscribes with.
|
||||
`claimed_feed_urls` lists other `feed_url`s that publishers claim (at most
|
||||
8, with their publishers), for display only.
|
||||
* The representative `card` is a card whose `feed_url` is the origin URL
|
||||
when one exists, then the title most publishers agree on, then the
|
||||
smallest publisher key. Publisher-controlled times and revisions never
|
||||
break ties. Its metadata remains a publisher's claim.
|
||||
* `publishers` lists distinct publisher **devices**. It is not a subscriber
|
||||
count (one user may publish from several devices, and subscribers who did
|
||||
not share are invisible) and not a trust signal: device identities are
|
||||
free. No subscriber count is offered and ranking ignores it.
|
||||
* Ranking: exact normalized title first, then hits that have a card under
|
||||
the origin URL, then title and id.
|
||||
* `FederatedSearchHit` (`{"type": "music" | "podcast", "hit": ...}`),
|
||||
`SearchHitKind` (`artist`, `release`, `track`, `podcast`) and
|
||||
`merge_search_results` merge podcast hits with `SearchOutcome` music
|
||||
results; `SearchOutcome` itself is unchanged.
|
||||
* Lookups query the 8 closest overlay nodes (α = 3, at most 32 requests,
|
||||
15 s deadline) and return partial results when peers fail. They keep at
|
||||
most 512 `(podcast_id, publisher)` owners; each response (and the local
|
||||
cache) may add at most 64 new owners (`512 / K`), so the responders that
|
||||
answer first cannot fill a lookup and crowd out the rest. A newer record
|
||||
of a known owner always replaces its winner. A peer that fails a request
|
||||
is skipped for 30 s.
|
||||
|
||||
Subscribing from a hit creates the user's own private subscription with the
|
||||
hit's `feed_url` (the origin URL): resolve the id with `resolve_podcast_id`,
|
||||
which yields the hit's `podcast_id` unless the user already knows that URL,
|
||||
fetch the feed from that URL and follow its own HTTP redirects and
|
||||
`itunes:new-feed-url`, recording those as aliases as usual. A URL from
|
||||
`claimed_feed_urls` becomes an alias only when that fetch leads there. The new
|
||||
subscription is private (see `effective_share`), the publisher's availability
|
||||
is not needed afterwards, and the episode catalog comes from RSS.
|
||||
|
||||
### Limits
|
||||
|
||||
| Limit | Value |
|
||||
| --- | --- |
|
||||
| Request / response | 512 KiB / 4 MiB postcard |
|
||||
| Store batch | 1..16 entries, each 1..33 distinct keys, records of the requester only |
|
||||
| Records per `FindValue` response | 128, at most 32 of them tombstones |
|
||||
| Filter tokens | ≤ 8, each 1..64 bytes |
|
||||
| Records per key | 128, at most 32 of them tombstones; random eviction from the younger half |
|
||||
| Records per replica | 256 per publisher (refused beyond), 4 096 in all, 32 MiB of payload (random eviction from the younger half, at most 16 per record) |
|
||||
| Store responsibility | fewer than `2 * K` = 16 verified, not backed-off contacts closer to the key |
|
||||
| Records kept per lookup | 512 `(podcast_id, publisher)`, at most 64 new ones per response |
|
||||
| Persisted overlay contacts | 1 024 |
|
||||
| Inbound requests | 64 concurrent, 8 per peer (more are dropped); 10 s each |
|
||||
| Error message | ≤ 1 KiB |
|
||||
| Contact ticket | ≤ 16 KiB |
|
||||
|
||||
### Privacy guarantees
|
||||
|
||||
* Only podcasts with `effective_share` are published; never-shared podcasts
|
||||
produce no network traffic at all.
|
||||
* After a restart nothing is re-sent from the persisted cards until the
|
||||
client reconciled its registers with `sync_shares` (see Publication).
|
||||
* Cards carry the public projection only (see Card); tombstones carry key
|
||||
hashes only.
|
||||
* Records are signed by the publisher device and verified on every receive
|
||||
and on every cache read; a forwarder cannot alter or impersonate them,
|
||||
only the publisher can store them, and records from another network are
|
||||
rejected.
|
||||
* The card's `publisher` is the device's persistent iroh endpoint key, which
|
||||
is also its overlay and music-DHT identity. It is **linkable**: anyone can
|
||||
look the key up in the overlay (`FindNode` for
|
||||
`NodeId::from_endpoint(publisher)`) and obtain the device's contact ticket
|
||||
with its relay and direct IP addresses, because every overlay request
|
||||
carries the requester's ticket. The same key owns the device's music-DHT
|
||||
records, so a device's shared podcasts can be linked to its public music
|
||||
library and its network address. Subscribing from a card never contacts
|
||||
the publisher.
|
||||
* Devices of one user are not linked by the overlay, but publishing the same
|
||||
podcasts from several devices makes them correlatable.
|
||||
* Unpublishing is eventually consistent: replicas that missed the tombstone
|
||||
keep serving the old card until it expires (at most 12 h), including
|
||||
replicas that only held an older title's text keys.
|
||||
* Capability manifests are informational. A peer without `podcast_dht` never
|
||||
enters the overlay routing table (it cannot answer on the ALPN), and
|
||||
`PodcastDht::note_peer_capabilities` evicts a peer whose manifest lacks it
|
||||
and stops dialing it. Applications advertise the overlay only when they
|
||||
run it (`with_podcast_dht`).
|
||||
|
||||
## Feed parser
|
||||
|
||||
With the `podcast-feed` feature, `music_dht::podcast::feed` parses RSS 2.0,
|
||||
@@ -511,6 +878,150 @@ extracts `itunes:season`, `itunes:episode`, `itunes:episodeType`,
|
||||
Links are kept verbatim (trimmed only) because they are identity material.
|
||||
Text cleanup is linear in the input size. It performs no I/O.
|
||||
|
||||
## SQLite reference store
|
||||
|
||||
`furumi_library::podcasts::PodcastStore` (`furumi-library` 0.3) implements
|
||||
this contract for SQLite clients. Other stores (the web client's PostgreSQL)
|
||||
must keep the same invariants.
|
||||
|
||||
### File and migrations
|
||||
|
||||
* Own file, conventionally `podcasts.sqlite3` next to `library.db`; the caller
|
||||
passes the path. WAL, foreign keys, 5 s busy timeout.
|
||||
* `PRAGMA user_version` drives migrations. Each runs in a `BEGIN IMMEDIATE`
|
||||
transaction that re-reads the version, so concurrent openers and
|
||||
interrupted upgrades are safe and reopening is idempotent.
|
||||
|
||||
| Version | Content |
|
||||
| --- | --- |
|
||||
| 0 | empty file |
|
||||
| 1 | prototype schema of the original podcast PR (`podcasts`, `podcast_episodes`), verbatim |
|
||||
| 2 | the model below; the prototype tables are renamed and imported |
|
||||
|
||||
* Before writing anything (including the WAL switch), the store refuses a
|
||||
file with a `user_version` above its own, and a file whose tables are not
|
||||
the expected ones for its version (for example `library.db` passed by
|
||||
mistake). The file is left byte-for-byte unchanged.
|
||||
* Prototype import (migration 2). Tables are renamed `legacy_podcasts` and
|
||||
`legacy_podcast_episodes`, never dropped.
|
||||
* Each feed becomes a podcast with `podcast_id` from the canonical feed URL
|
||||
(`FeedUrl::parse`, then `FeedUrl::from_user_input`) and a subscription
|
||||
register `subscribed = true`, generation 0, `hlc_ms` = `added_at`,
|
||||
`op_id` = `legacy-import:<podcast_id>`, not shared. Every contract write
|
||||
orders after it; other devices learn it through the snapshot sections.
|
||||
Rows with the same canonical URL merge (first row wins).
|
||||
* URLs that cannot be canonicalized, episodes with a non-`http(s)` audio URL,
|
||||
invalid metadata or no identity are listed in `legacy_quarantine`; the
|
||||
rows stay in the `legacy_*` tables.
|
||||
* Episode ids come from `resolve_episode_ids` over the stored items. The
|
||||
prototype stored the enclosure URL as `guid` when an item had none: such
|
||||
rows are imported as GUID-less items and keep the value in
|
||||
`episodes.legacy_guid`. The prototype kept one row per enclosure URL, so
|
||||
rows with the same publication second, normalized title and `<link>`
|
||||
(or none) are one item whose audio URL moved: the newest row
|
||||
(`first_seen_at`, then `id`) is imported, the others are quarantined as
|
||||
duplicates.
|
||||
* Imported GUID-less episodes are **provisional**: stale rows can still
|
||||
change which key is unique among the stored items, so their ids may
|
||||
differ from what a device without the prototype derives. On a refresh,
|
||||
item ids are resolved against the stored episodes except provisional
|
||||
ones without progress or sessions; each such provisional episode is
|
||||
matched to its item (enclosure, then link, then date and title, GUIDs
|
||||
ignored) and re-keyed to that item's id. When another episode that
|
||||
stays already holds that id (for example one a remote operation created
|
||||
before this device's first refresh), it is the item's episode and the
|
||||
provisional one is a duplicate: it is deleted rather than kept as a
|
||||
second copy. The same applies when `legacy_guid` turns out to be the
|
||||
item's real GUID. Listing an episode clears `legacy_guid`, which makes
|
||||
its id final. Provisional episodes carry no personal state, so nothing
|
||||
personal moves or is lost; an imported episode played before its first
|
||||
refresh is not provisional and keeps its id (the feed item then becomes
|
||||
a separate episode if the ids differ).
|
||||
* ETag, Last-Modified, check time, last error and the cover cache move to the
|
||||
fetch state. The prototype stores no progress or history, so none is
|
||||
created.
|
||||
|
||||
### Tables (version 2)
|
||||
|
||||
| Table | Key | Content | Synced |
|
||||
| --- | --- | --- | --- |
|
||||
| `podcasts` | `podcast_id` | local channel metadata, `feed_url` = URL this device fetches, `meta_from_feed` | catalog, no |
|
||||
| `podcast_feed_aliases` | `(podcast_id, feed_url)` | bounded grow-only alias set (`merge_feed_aliases`) | yes, inside subscription records and `podcast_feed_aliases_added` |
|
||||
| `episodes` | `episode_id` | `EpisodeMeta` columns (season, number, type, published, duration, audio URL, MIME, length, artwork, link), `source` (`feed`/`sync`), `in_feed`, `legacy_guid` | catalog, no |
|
||||
| `podcast_seasons` | `(podcast_id, season)` | derived from `episodes`: episode count, first/last publication; `season` NULL groups episodes without a season | no |
|
||||
| `podcast_subscriptions` | `podcast_id` | subscription register: `subscribed`, `PodcastMeta` JSON, `generation`, `hlc_ms`, `op_id` | yes |
|
||||
| `podcast_shares` | `podcast_id` | share register | yes |
|
||||
| `episode_progress` | `episode_id` | progress register (`ProgressRecord`) | yes |
|
||||
| `podcast_listen_sessions` | `session_id` | grow-only set of `ListenSession` | yes |
|
||||
| `podcast_fetch_state` | `podcast_id` | ETag, Last-Modified, checked / last success / next check times, last error, consecutive failures, cover cache | **never** |
|
||||
| `legacy_quarantine` | `(legacy_table, legacy_id)` | prototype rows that could not be imported | no |
|
||||
|
||||
### Invariants
|
||||
|
||||
* Registers and sessions have no foreign keys to the catalog: operations for
|
||||
podcasts or episodes the device has not fetched are stored as they arrive,
|
||||
in any order. An operation's `PodcastEpisodeRef` adds a `source = 'sync'`
|
||||
catalog row only when the episode is unknown; feed metadata always wins.
|
||||
* One apply path. Local actions validate the payload, draw a stamp from the
|
||||
client's sync log, build the `SyncOpWire`, and apply it like a received
|
||||
operation: `SyncOpWire::podcast_*` materializers, then `merge_*`. The
|
||||
returned operation must be appended with exactly that stamp. Snapshot
|
||||
records go through the same functions after validation. The result is
|
||||
therefore commutative, associative and idempotent, and a snapshot never
|
||||
replaces newer local state.
|
||||
* Invalid operations (`validate_podcast`, a generation above `i64::MAX`, a
|
||||
session whose identity differs from the stored one) are skipped and
|
||||
reported as rejected, so the client can still acknowledge them.
|
||||
* Only `subscribe`/`unsubscribe` write the subscription register and only
|
||||
`set_shared`/`unsubscribe` the share register, with
|
||||
`next_podcast_generation`. No-op actions draw no stamp and emit nothing.
|
||||
`unsubscribe` emits `subscribed = false` and `shared = false` and keeps
|
||||
progress, sessions and the catalog. The store only records the decision;
|
||||
publishing and revoking the public card is `PodcastDht::sync_shares` (see
|
||||
[Public discovery](#public-discovery-podcast_dht)).
|
||||
* Explicit progress actions use `explicit_generation(max(local, carried))`;
|
||||
checkpoints keep the session generation and are refused (`Superseded`)
|
||||
once the register holds a greater one, or after the session completed.
|
||||
* RSS refreshes update the catalog (channel metadata, episode metadata and
|
||||
audio URLs, `in_feed`) and the alias set only; episodes missing from the
|
||||
feed are kept. Permanent redirects and `itunes:new-feed-url` become the
|
||||
fetch URL and aliases, and emit `podcast_feed_aliases_added` only when the
|
||||
stored alias set changes.
|
||||
* Snapshot export reads each section newest first up to its count, then
|
||||
applies `bound_podcast_state_to(budget)`. Fetch state and the catalog
|
||||
(descriptions, episodes) are never exported.
|
||||
* Feed URLs resolve through `resolve_podcast_id` over the alias set, the
|
||||
catalog fetch URL and the register's `podcast.feed_url`.
|
||||
* Fetch-state writes (`record_fetch_*`, `set_cover_cache`) read and write
|
||||
the row in one immediate transaction, so concurrent writers on one or
|
||||
several connections never lose a column or a failure increment.
|
||||
* `TrackItem::podcast_episode` builds queue items whose identity is the
|
||||
explicit source `TrackItem.podcast` alone. Their `id`, `release_id` and
|
||||
artist id are the fixed sentinel `TrackItem::NOT_A_LIBRARY_ROW` (-1),
|
||||
shared by every episode, the same "not a library row" convention as
|
||||
pending federated tracks. The sentinel carries no identity and is never
|
||||
decoded; it only keeps music-only code that predates podcasts
|
||||
(`id >= 0`, `release_id < 0` checks) from taking an episode for library
|
||||
track, release or artist 0. There is no reserved range of negative ids.
|
||||
* `TrackItem::key` (a typed `TrackItemKey`; string form `lib:<id>`,
|
||||
`fed:<owner>:<item_id>`, `ep:<episode id>`) is the source identity of a
|
||||
queue item; on the wire it is `PlaybackSource`. An episode is identified
|
||||
by `TrackItem.podcast` / `ep:` only and must never be keyed by its numeric
|
||||
id: keying by id (for example a `local:{id}` string) would merge all
|
||||
episodes into one. `TrackItemKey` has no content-id equivalence (a pending
|
||||
federated item and its imported library copy differ), so clients that
|
||||
match music content-first keep doing that and add the podcast branch
|
||||
before any fallback to the id.
|
||||
* Wire ids of podcast items and of Jam private placeholders are `0`
|
||||
(`PlaybackTrack::podcast_episode`, `PlaybackTrack::private_placeholder`)
|
||||
and must not be copied into a `TrackItem`.
|
||||
`TrackItem::from_playback_track` classifies a received item as
|
||||
`PlaybackItem::Music` (the client's music path), `PodcastEpisode` (built
|
||||
with `TrackItem::podcast_episode`, sentinel ids) or `Private` (not
|
||||
playable, no `TrackItem`). Besides the ids, `TrackItem::podcast_episode`
|
||||
also differs from the wire form in `release_year` (publication year) and
|
||||
`audio_format` (`None`; the MIME type stays in the reference).
|
||||
|
||||
## Versioning
|
||||
|
||||
* `music-dht` 0.6.0 and `furumi-library` 0.3.0: new public types, enum
|
||||
@@ -525,5 +1036,8 @@ Text cleanup is linear in the input size. It performs no I/O.
|
||||
* Feed URL canonicalization, the identity bounds (`MAX_URL_BYTES`,
|
||||
`MAX_GUID_BYTES`) and the no-GUID rule are frozen with the `v1` domains:
|
||||
receivers reject non-canonical ids and URLs while parsing.
|
||||
* Public podcast cards will use a separate signed overlay with its own ALPN
|
||||
and version; the music DHT schema and `federation-net` do not change.
|
||||
* Public podcast cards use the separate signed overlay `podcast_dht` with
|
||||
its own ALPN (`furumi-fd/podcast-dht/1`), wire/record version 1, signing
|
||||
domains `furumi-podcast-card:v1` / `furumi-podcast-card-tombstone:v1`, key
|
||||
domain `furumi-podcast-dht:key:v1`, all frozen, and cache schema 2; the music
|
||||
DHT schema and `federation-net` do not change.
|
||||
@@ -50,13 +50,13 @@ crates/
|
||||
|
||||
## Using the libraries
|
||||
|
||||
Until crates are published to a registry, depend on the repository directly:
|
||||
Use the published crates from crates.io:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
federation-net = { git = "https://gt.hexor.cy/ab/frid.git" }
|
||||
music-dht = { git = "https://gt.hexor.cy/ab/frid.git" }
|
||||
furumi-library = { git = "https://gt.hexor.cy/ab/frid.git" }
|
||||
federation-net = "0.3.1"
|
||||
music-dht = "0.6.0"
|
||||
furumi-library = "0.3.0"
|
||||
```
|
||||
|
||||
Start with the crate documentation:
|
||||
@@ -80,6 +80,48 @@ The integration tests open real local sockets and establish iroh connections.
|
||||
Some rendezvous tests also use public discovery infrastructure, so a restricted
|
||||
sandbox may need explicit network permission.
|
||||
|
||||
## Publishing
|
||||
|
||||
Add these repository Actions secrets:
|
||||
|
||||
- `CARGO_REGISTRY_TOKEN`: a crates.io API token with publish access to
|
||||
`federation-net`, `music-dht` and `furumi-library`.
|
||||
- `CARGO_REGISTRIES_GITEA_TOKEN`: a raw Gitea personal access token with
|
||||
`write:package` and permission to publish under the `Furumi` organization.
|
||||
Create it under user Settings → Applications. The workflow adds the
|
||||
required `Bearer` prefix itself; no username secret is needed.
|
||||
|
||||
Update changed crate versions, workspace dependency requirements, `Cargo.lock`
|
||||
and `CHANGELOG.md`, then merge to `main`. Create `<crate>-v<version>` tags for
|
||||
the changed crates and one workspace release tag, for example:
|
||||
|
||||
```bash
|
||||
git tag -a music-dht-v0.6.0 -m 'music-dht 0.6.0'
|
||||
git tag -a furumi-library-v0.3.0 -m 'furumi-library 0.3.0'
|
||||
git tag -a v0.6.0 -m 'Frid podcast release'
|
||||
git push origin music-dht-v0.6.0 furumi-library-v0.3.0 v0.6.0
|
||||
```
|
||||
|
||||
Only the `v*` workspace tag triggers `.github/workflows/publish.yml`; the
|
||||
crate tags identify each package release without starting duplicate uploads.
|
||||
The workflow runs the workspace checks and verifies the release packages in
|
||||
one build job. Two publishing jobs then upload without rebuilding
|
||||
(`cargo publish --no-verify`), selecting missing versions independently in
|
||||
crates.io and the Gitea mirror at
|
||||
`https://gt.hexor.cy/api/packages/Furumi/cargo/`. The mirror includes all three
|
||||
crates, including an unchanged version missing there. Already published
|
||||
versions are skipped on retries; a registry lookup error fails the release
|
||||
instead of assuming absence. Mirrored manifests retain crates.io dependency
|
||||
references, so the mirror waits up to ten minutes for new workspace
|
||||
dependencies to appear in crates.io before packaging its uploads.
|
||||
|
||||
Preview the selection with `python3 scripts/publish_crates.py --plan`, or
|
||||
verify the packages without uploading with `--dry-run`. Add `--registry gitea`
|
||||
to preview or publish to the mirror. If a secret was
|
||||
missing or a publish failed, fix the cause and rerun the existing tag workflow;
|
||||
do not move release tags. Manual dispatch must select the release tag as its
|
||||
ref. Publication is performed by CI, not by the tagging commands above.
|
||||
|
||||
## License
|
||||
|
||||
Frid is released under the
|
||||
|
||||
@@ -16,8 +16,11 @@ anyhow = { workspace = true }
|
||||
blake3 = { workspace = true }
|
||||
directories = { workspace = true }
|
||||
lofty = { workspace = true }
|
||||
music-dht = { workspace = true }
|
||||
music-dht = { workspace = true, features = ["podcast-feed"] }
|
||||
rusqlite = { workspace = true, features = ["functions"] }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
@@ -21,5 +21,67 @@ library used by every client. Connections use WAL, foreign keys, and a busy
|
||||
timeout so independently running clients can safely share the database.
|
||||
|
||||
Version 0.3 follows `music-dht` 0.6, whose device-sync and podcast contract
|
||||
types it re-exposes. Podcast persistence is not part of this release; it will
|
||||
implement the rules of the workspace `PODCASTS.md`.
|
||||
types it re-exposes.
|
||||
|
||||
## Podcasts
|
||||
|
||||
`furumi_library::podcasts::PodcastStore` is the SQLite reference store of the
|
||||
podcast contract (workspace `PODCASTS.md`, section "SQLite reference store").
|
||||
It lives in its own file, conventionally `podcasts.sqlite3`
|
||||
(`PODCAST_DB_FILE`) next to `library.db`; the caller passes the path.
|
||||
|
||||
- **Migrations** run on `PRAGMA user_version`, each in its own transaction.
|
||||
A database of the original podcast prototype (`user_version` 1) is imported:
|
||||
its tables are renamed `legacy_podcasts` / `legacy_podcast_episodes` (never
|
||||
dropped), feeds become subscriptions with contract ids, rows that cannot be
|
||||
imported are listed in `legacy_quarantine`. A database from a newer Furumi,
|
||||
or a file that is not a podcast database, is refused without being touched.
|
||||
- **Local actions** (`subscribe`, `unsubscribe`, `set_shared`,
|
||||
`start_session`, `checkpoint`, `seek`, `restart`, `complete_session`,
|
||||
`mark_played`, `mark_unplayed`, `record_listen_session`) draw stamps from a
|
||||
`StampSource` (usually the client's sync log), apply the operation and
|
||||
return `PodcastLocalOp`s to append to the log with exactly those stamps.
|
||||
- **Sync**: `apply_remote_op` applies the five `podcast_*` operations,
|
||||
`fill_snapshot` / `export_snapshot` produce the bounded snapshot sections
|
||||
and `apply_snapshot` merges them. Everything goes through the `music-dht`
|
||||
materializers and merge functions, so the result does not depend on
|
||||
delivery order and never loses newer local state.
|
||||
- **RSS**: `apply_feed` ingests a document parsed by
|
||||
`music_dht::podcast::feed` (the crate enables the `podcast-feed` feature):
|
||||
channel metadata, episodes with stable ids and current audio URLs, feed
|
||||
aliases from permanent redirects and `itunes:new-feed-url`. It never
|
||||
touches subscriptions, progress or history. HTTP, conditional requests and
|
||||
scheduling stay in the client; `record_fetch_*`, `fetch_state` and
|
||||
`podcasts_due_for_refresh` keep the local-only fetch state, which is never
|
||||
synced.
|
||||
- **Queries**: `list_podcasts`, `seasons`, `episodes` (season, played-state
|
||||
and text filters, date or episode-number order), `resume_position_ms`,
|
||||
`episode_ref` / `episode_track` for queueing, `listen_history`.
|
||||
- **Public discovery**: the store records share decisions only. Build each
|
||||
`music_dht::podcast_dht::PodcastShareState` for `PodcastDht::sync_shares`
|
||||
from `subscription` and `share` (workspace `PODCASTS.md`, section "Public
|
||||
discovery (podcast_dht)"); the client keeps the user's `privacy_confirmed` decision.
|
||||
|
||||
The store is synchronous: methods take `&self` and lock one connection, so
|
||||
`Arc<PodcastStore>` can be shared; run calls on a blocking pool.
|
||||
|
||||
`TrackItem.podcast` marks a queue item as a podcast episode
|
||||
(`TrackItem::podcast_episode`). Music-only code (likes, playlists, history,
|
||||
content ids, federation downloads) must check `TrackItem::is_podcast`; the
|
||||
music library never stores episodes.
|
||||
|
||||
`TrackItem::key` returns the typed source identity of a queue item
|
||||
(`TrackItemKey`: `lib:<id>`, `fed:<owner>:<item_id>` or `ep:<episode id>` as
|
||||
a string). A negative `TrackItem::id` only means "not a library row". Every
|
||||
podcast item has the same sentinel `id`, `release_id` and artist id
|
||||
(`TrackItem::NOT_A_LIBRARY_ROW` = -1). It carries no identity and must not
|
||||
be decoded; it only keeps music-only code that predates podcasts from taking
|
||||
an episode for library track, release or artist 0. The episode's identity is
|
||||
`TrackItem.podcast` alone, so queue, current-track and position matching
|
||||
must check it before any fallback to the id. `TrackItemKey` does not equate
|
||||
a pending federated item with its library copy; clients that match music by
|
||||
content id keep doing so.
|
||||
|
||||
Convert received `PlaybackTrack`s with `TrackItem::from_playback_track`
|
||||
(`PlaybackItem::Music`, `PodcastEpisode` or `Private`): podcast items and
|
||||
Jam private placeholders carry id 0 on the wire, which must never be copied.
|
||||
@@ -6,9 +6,13 @@
|
||||
//!
|
||||
//! All methods take `&self` and lock the connection internally, so an
|
||||
//! `Arc<Library>` can be shared across blocking tasks.
|
||||
//!
|
||||
//! Podcasts live in a separate database, [`podcasts::PodcastStore`]
|
||||
//! (`podcasts.sqlite3`); the music library never stores episodes.
|
||||
|
||||
pub mod import;
|
||||
pub mod models;
|
||||
pub mod podcasts;
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Write as _;
|
||||
@@ -22,8 +26,8 @@ use rusqlite::{Connection, OptionalExtension as _, params};
|
||||
|
||||
pub use models::{
|
||||
ArtistCard, ArtistDetail, ArtistRef, ArtistsPage, Availability, FederatedTrack, LibraryFilters,
|
||||
LibrarySourceMode, PlaylistCard, PlaylistDetail, ReleaseCard, ReleaseDetail, ReleaseEdit,
|
||||
SearchResults, TrackEdit, TrackItem,
|
||||
LibrarySourceMode, PlaybackItem, PlaylistCard, PlaylistDetail, ReleaseCard, ReleaseDetail,
|
||||
ReleaseEdit, SearchResults, TrackEdit, TrackItem, TrackItemKey,
|
||||
};
|
||||
|
||||
/// The virtual "Liked tracks" playlist id, kept from the server API.
|
||||
@@ -3283,6 +3287,7 @@ fn query_tracks(
|
||||
artists: Vec::new(),
|
||||
featured_artists: Vec::new(),
|
||||
fed: None,
|
||||
podcast: None,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
@@ -3348,6 +3353,7 @@ fn pending_track(fed: &FederatedTrack) -> TrackItem {
|
||||
file_size_bytes: None,
|
||||
play_count: 0,
|
||||
fed: Some(fed.clone()),
|
||||
podcast: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
//! Data shapes the views render. They mirror what the furumusic API used to
|
||||
//! return, but every field is now filled from the local SQLite library.
|
||||
|
||||
use music_dht::device_sync::{PlaybackSource, PlaybackTrack};
|
||||
use music_dht::podcast::{EpisodeId, PodcastEpisodeRef};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Which catalog sources a client wants library queries to include.
|
||||
@@ -120,6 +122,56 @@ pub struct ArtistRef {
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// A playable queue item: a library track, a pending federated track or a
|
||||
/// podcast episode.
|
||||
///
|
||||
/// # Identity
|
||||
///
|
||||
/// What an item *is* comes from its explicit source fields, never from the
|
||||
/// numeric `id` alone; [`TrackItem::key`] returns it as a typed
|
||||
/// [`TrackItemKey`]:
|
||||
///
|
||||
/// * a library track is a library row: `id >= 0`, `fed` and `podcast` are
|
||||
/// `None`;
|
||||
/// * a federated track is identified by [`TrackItem::fed`] (owner and item
|
||||
/// id). While it is not in the library its `id` is a negative,
|
||||
/// process-local counter value (pending federated tracks count down from
|
||||
/// -1) and `release_id` / artist ids are -1;
|
||||
/// * a podcast episode is identified **only** by [`TrackItem::podcast`] (its
|
||||
/// [`EpisodeId`]). Its `id`, `release_id` and artist id are the fixed
|
||||
/// sentinel [`TrackItem::NOT_A_LIBRARY_ROW`] (-1), shared by every episode.
|
||||
///
|
||||
/// A negative `id` therefore means "not a library row" and nothing else: it
|
||||
/// carries no identity and must never be decoded, compared across items or
|
||||
/// used as a key. The -1 sentinel only keeps music-only code that predates
|
||||
/// podcasts (`id >= 0` / `release_id < 0` checks) from treating an episode
|
||||
/// as library track, release or artist 0.
|
||||
///
|
||||
/// [`TrackItem::key`] is the *source* identity. It does not merge a pending
|
||||
/// federated item with its imported library copy, which share a content id
|
||||
/// but have different keys (`fed:` and `lib:`). Clients that key playback
|
||||
/// items content-first (content id, then federated source, then id) keep
|
||||
/// doing so for music; what they must add is a podcast branch
|
||||
/// ([`TrackItem::podcast`], or [`TrackItemKey::PodcastEpisode`]) checked
|
||||
/// before any fallback to `id`. An episode must never be keyed by `id`: a
|
||||
/// `local:{id}` key would merge all episodes into one. On the wire the item
|
||||
/// kind is `music_dht::device_sync::PlaybackSource`; convert received items
|
||||
/// with [`TrackItem::from_playback_track`], never by copying wire ids.
|
||||
///
|
||||
/// # Podcast episodes
|
||||
///
|
||||
/// Podcast episodes carry their [`PodcastEpisodeRef`] in
|
||||
/// [`TrackItem::podcast`]: `file_path` is empty, `content_id`, `fed` and
|
||||
/// `cover_path` are `None`, and the display fields mirror the reference
|
||||
/// (episode title, podcast title as artist and release). The audio URL is
|
||||
/// only in the reference: it is never a local path, never content-addressed
|
||||
/// and never offered for P2P transfer.
|
||||
///
|
||||
/// **Music-only code paths must check [`TrackItem::is_podcast`]** (or
|
||||
/// `podcast.is_none()`) before treating an item as music: likes, playlists,
|
||||
/// listening history, content-id lookups, federation downloads, similarity
|
||||
/// and file operations do not apply to episodes. Library queries never
|
||||
/// return podcast items; the library database stores none.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TrackItem {
|
||||
pub id: i64,
|
||||
@@ -149,9 +201,187 @@ pub struct TrackItem {
|
||||
/// carries everything needed to download them from the owning peer.
|
||||
/// With an empty `file_path` the player resolves the track on demand.
|
||||
pub fed: Option<FederatedTrack>,
|
||||
/// Set for podcast episodes (see the type documentation); `None` for music.
|
||||
pub podcast: Option<PodcastEpisodeRef>,
|
||||
}
|
||||
|
||||
/// Typed source identity of a [`TrackItem`], from [`TrackItem::key`].
|
||||
///
|
||||
/// Two items with equal keys are the same track or episode; the same track
|
||||
/// may still appear more than once in a queue. The key has no content-id
|
||||
/// equivalence: a pending federated item and its library copy differ. Use it
|
||||
/// where the source matters (and always for podcast episodes, which have no
|
||||
/// other identity); code that matches music by content id keeps doing that
|
||||
/// and checks the podcast variant first (see [`TrackItem`]).
|
||||
///
|
||||
/// The [`Display`](std::fmt::Display) form is stable and suitable as a map
|
||||
/// key or in logs:
|
||||
///
|
||||
/// | Variant | String form |
|
||||
/// | --- | --- |
|
||||
/// | `Library(123)` | `lib:123` |
|
||||
/// | `Federated { owner, item_id }` | `fed:<owner>:<item_id>` |
|
||||
/// | `PodcastEpisode(id)` | `ep:<episode id>`, e.g. `ep:ep1:...` |
|
||||
/// | `Transient(-5)` | `tmp:-5` |
|
||||
///
|
||||
/// `owner` is a peer endpoint id and contains no `:`, so the federated form
|
||||
/// is unambiguous. Keys are not persisted by this crate; a `Transient` key is
|
||||
/// only meaningful inside the process that built the item.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum TrackItemKey {
|
||||
/// A row of the local music library (`TrackItem::id >= 0`).
|
||||
Library(i64),
|
||||
/// A federated track that is not a library row, identified by its owner
|
||||
/// peer and DHT item id ([`FederatedTrack::owner`] /
|
||||
/// [`FederatedTrack::item_id`]), not by its process-local negative `id`.
|
||||
Federated { owner: String, item_id: String },
|
||||
/// A podcast episode, identified by its [`EpisodeId`] only.
|
||||
PodcastEpisode(EpisodeId),
|
||||
/// A non-library item with no explicit source (negative `id`, no `fed`,
|
||||
/// no `podcast`). furumi-library never builds one; clients that do must
|
||||
/// keep its `id` unique within the process.
|
||||
Transient(i64),
|
||||
}
|
||||
|
||||
/// What a received `music_dht::device_sync::PlaybackTrack` is locally, from
|
||||
/// [`TrackItem::from_playback_track`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum PlaybackItem {
|
||||
/// A music track. Convert it with the client's music path (content id,
|
||||
/// then federated source); its wire ids are the sender's.
|
||||
Music,
|
||||
/// A podcast episode, built with [`TrackItem::podcast_episode`].
|
||||
PodcastEpisode(Box<TrackItem>),
|
||||
/// A Jam private placeholder: not playable and without identity. Show it
|
||||
/// as `music_dht::device_sync::PRIVATE_ITEM_TITLE` where the queue is
|
||||
/// displayed; never insert it as a library or queue track, never key it.
|
||||
Private,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for TrackItemKey {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Library(id) => write!(f, "lib:{id}"),
|
||||
Self::Federated { owner, item_id } => write!(f, "fed:{owner}:{item_id}"),
|
||||
Self::PodcastEpisode(episode_id) => write!(f, "ep:{}", episode_id.as_str()),
|
||||
Self::Transient(id) => write!(f, "tmp:{id}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrackItem {
|
||||
/// The `id`, `release_id` and artist id of items that are not library
|
||||
/// rows and have no numeric identity: every podcast episode uses it.
|
||||
///
|
||||
/// It is a sentinel, not an identifier: it is shared by all episodes
|
||||
/// (and can coincide with the first pending federated track's id), so it
|
||||
/// must never be decoded or used as a key. See [`TrackItem::key`].
|
||||
pub const NOT_A_LIBRARY_ROW: i64 = -1;
|
||||
|
||||
/// The typed identity of this item; see [`TrackItemKey`].
|
||||
///
|
||||
/// Precedence: [`TrackItem::podcast`] makes it a podcast episode; else a
|
||||
/// non-negative `id` is a library row; else [`TrackItem::fed`] makes it
|
||||
/// a federated track; anything else is [`TrackItemKey::Transient`].
|
||||
pub fn key(&self) -> TrackItemKey {
|
||||
if let Some(episode) = &self.podcast {
|
||||
return TrackItemKey::PodcastEpisode(episode.episode_id.clone());
|
||||
}
|
||||
if self.id >= 0 {
|
||||
return TrackItemKey::Library(self.id);
|
||||
}
|
||||
match &self.fed {
|
||||
Some(fed) => TrackItemKey::Federated {
|
||||
owner: fed.owner.clone(),
|
||||
item_id: fed.item_id.clone(),
|
||||
},
|
||||
None => TrackItemKey::Transient(self.id),
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue item for a podcast episode streamed from the publisher.
|
||||
///
|
||||
/// Title, artist and release follow `PlaybackTrack::podcast_episode` of
|
||||
/// `music_dht::device_sync` (the episode title, the podcast title as
|
||||
/// artist and release, the announced duration; no track or disc number).
|
||||
/// The other fields differ from that wire form on purpose:
|
||||
///
|
||||
/// * `id`, `release_id` and the artist id are the sentinel
|
||||
/// [`TrackItem::NOT_A_LIBRARY_ROW`] (-1) for every episode, where the
|
||||
/// wire form carries `0`; the item's identity is [`TrackItem::podcast`]
|
||||
/// alone ([`TrackItem::key`]);
|
||||
/// * `release_year` is the publication year (`None` on the wire);
|
||||
/// * `audio_format` is `None` (the wire form carries the MIME type, which
|
||||
/// stays in the reference).
|
||||
///
|
||||
/// Season and episode number stay in the reference.
|
||||
pub fn podcast_episode(episode: PodcastEpisodeRef) -> Self {
|
||||
Self {
|
||||
id: Self::NOT_A_LIBRARY_ROW,
|
||||
title: episode.episode_title.clone(),
|
||||
track_number: None,
|
||||
disc_number: None,
|
||||
duration_seconds: episode
|
||||
.duration_ms
|
||||
.map_or(0.0, |ms| ms.max(0) as f64 / 1_000.0),
|
||||
artists: vec![ArtistRef {
|
||||
id: Self::NOT_A_LIBRARY_ROW,
|
||||
name: episode.podcast_title.clone(),
|
||||
}],
|
||||
featured_artists: Vec::new(),
|
||||
release_id: Self::NOT_A_LIBRARY_ROW,
|
||||
release_title: episode.podcast_title.clone(),
|
||||
release_year: episode.published_at_ms.map(year_of_unix_ms),
|
||||
file_path: String::new(),
|
||||
content_id: None,
|
||||
cover_path: None,
|
||||
audio_format: None,
|
||||
audio_bitrate: None,
|
||||
audio_sample_rate: None,
|
||||
audio_bit_depth: None,
|
||||
file_size_bytes: None,
|
||||
play_count: 0,
|
||||
fed: None,
|
||||
podcast: Some(episode),
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies a received `music_dht::device_sync::PlaybackTrack` and
|
||||
/// builds the local queue item for non-music sources.
|
||||
///
|
||||
/// The wire `id` and `release_id` of podcast episodes and of Jam private
|
||||
/// placeholders are `0`, and those of music items are the sender's local
|
||||
/// ids. None of them identifies anything on this device, so they are
|
||||
/// never copied:
|
||||
///
|
||||
/// * a podcast episode becomes [`TrackItem::podcast_episode`] of its
|
||||
/// [`PlaybackSource::PodcastEpisode`] reference (sentinel ids, key
|
||||
/// `ep:`), whatever the legacy display fields say;
|
||||
/// * a private placeholder ([`PlaybackSource::Private`]) is not playable
|
||||
/// and has no identity, so no `TrackItem` exists for it;
|
||||
/// * music (no source, or [`PlaybackSource::Music`]) is left to the
|
||||
/// client's music path, which resolves it by content id or federated
|
||||
/// source.
|
||||
pub fn from_playback_track(track: &PlaybackTrack) -> PlaybackItem {
|
||||
match &track.source {
|
||||
None | Some(PlaybackSource::Music) => PlaybackItem::Music,
|
||||
Some(PlaybackSource::PodcastEpisode(episode)) => PlaybackItem::PodcastEpisode(
|
||||
Box::new(Self::podcast_episode(episode.as_ref().clone())),
|
||||
),
|
||||
Some(PlaybackSource::Private) => PlaybackItem::Private,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this item is a podcast episode (not music).
|
||||
pub fn is_podcast(&self) -> bool {
|
||||
self.podcast.is_some()
|
||||
}
|
||||
|
||||
/// The episode reference of a podcast item.
|
||||
pub fn podcast_episode_ref(&self) -> Option<&PodcastEpisodeRef> {
|
||||
self.podcast.as_ref()
|
||||
}
|
||||
|
||||
/// A federated track that still needs downloading before playback.
|
||||
pub fn is_fed_pending(&self) -> bool {
|
||||
self.fed.is_some() && self.file_path.is_empty()
|
||||
@@ -202,6 +432,21 @@ impl TrackItem {
|
||||
}
|
||||
}
|
||||
|
||||
/// Proleptic Gregorian year of a unix timestamp in milliseconds (UTC).
|
||||
fn year_of_unix_ms(ms: i64) -> i32 {
|
||||
// Howard Hinnant's civil_from_days.
|
||||
let days = ms.div_euclid(86_400_000);
|
||||
let z = days + 719_468;
|
||||
let era = z.div_euclid(146_097);
|
||||
let doe = z - era * 146_097;
|
||||
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let month = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let year = yoe + era * 400 + i64::from(month <= 2);
|
||||
i32::try_from(year).unwrap_or(if year < 0 { i32::MIN } else { i32::MAX })
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReleaseCard {
|
||||
pub id: i64,
|
||||
@@ -340,8 +585,228 @@ mod tests {
|
||||
file_size_bytes: None,
|
||||
play_count: 0,
|
||||
fed: None,
|
||||
podcast: None,
|
||||
};
|
||||
|
||||
assert_eq!(track.artist_line(), "Main feat. Guest, Other");
|
||||
assert!(!track.is_podcast());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn year_of_unix_ms_handles_epoch_and_leap_years() {
|
||||
assert_eq!(year_of_unix_ms(0), 1970);
|
||||
assert_eq!(year_of_unix_ms(-1), 1969);
|
||||
// 2024-02-29T12:00:00Z and 2024-12-31T23:59:59Z.
|
||||
assert_eq!(year_of_unix_ms(1_709_208_000_000), 2024);
|
||||
assert_eq!(year_of_unix_ms(1_735_689_599_000), 2024);
|
||||
assert_eq!(year_of_unix_ms(1_735_689_600_000), 2025);
|
||||
}
|
||||
|
||||
fn episode_ref(guid: &str) -> PodcastEpisodeRef {
|
||||
use music_dht::podcast::{FeedUrl, PodcastId};
|
||||
let feed_url = FeedUrl::parse("https://example.test/feed.xml").unwrap();
|
||||
let podcast_id = PodcastId::from_feed_url(&feed_url);
|
||||
PodcastEpisodeRef {
|
||||
episode_id: EpisodeId::from_guid(&podcast_id, guid).unwrap(),
|
||||
podcast_id,
|
||||
feed_url,
|
||||
podcast_title: "Show".into(),
|
||||
episode_title: "Pilot".into(),
|
||||
audio_url: "https://cdn.example.test/1.mp3".into(),
|
||||
mime: Some("audio/mpeg".into()),
|
||||
duration_ms: Some(90_500),
|
||||
published_at_ms: Some(1_709_208_000_000),
|
||||
season: Some(2),
|
||||
episode_number: Some(7),
|
||||
image_url: None,
|
||||
progress_generation: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn library_track(id: i64) -> TrackItem {
|
||||
TrackItem {
|
||||
id,
|
||||
title: "Track".into(),
|
||||
track_number: None,
|
||||
disc_number: None,
|
||||
duration_seconds: 1.0,
|
||||
artists: vec![artist("Main")],
|
||||
featured_artists: Vec::new(),
|
||||
release_id: 1,
|
||||
release_title: "Release".into(),
|
||||
release_year: None,
|
||||
file_path: "/tmp/track.mp3".into(),
|
||||
content_id: None,
|
||||
cover_path: None,
|
||||
audio_format: None,
|
||||
audio_bitrate: None,
|
||||
audio_sample_rate: None,
|
||||
audio_bit_depth: None,
|
||||
file_size_bytes: None,
|
||||
play_count: 0,
|
||||
fed: None,
|
||||
podcast: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn federated(owner: &str, item_id: &str) -> FederatedTrack {
|
||||
FederatedTrack {
|
||||
item_id: item_id.into(),
|
||||
owner: owner.into(),
|
||||
own: false,
|
||||
title: "Remote".into(),
|
||||
artist_names: vec!["Someone".into()],
|
||||
featured_artist_names: Vec::new(),
|
||||
year: None,
|
||||
duration_seconds: None,
|
||||
content_id: None,
|
||||
release_title: None,
|
||||
track_number: None,
|
||||
disc_number: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_track_mirrors_the_episode_reference() {
|
||||
let episode = episode_ref("e1");
|
||||
let track = TrackItem::podcast_episode(episode.clone());
|
||||
assert!(track.is_podcast());
|
||||
assert_eq!(track.podcast_episode_ref(), Some(&episode));
|
||||
// The sentinel marks "not a library row, release or artist" and
|
||||
// carries no identity.
|
||||
assert_eq!(TrackItem::NOT_A_LIBRARY_ROW, -1);
|
||||
assert_eq!(track.id, -1);
|
||||
assert_eq!(track.release_id, -1);
|
||||
assert!(track.artists.iter().all(|artist| artist.id == -1));
|
||||
assert!(track.featured_artists.is_empty());
|
||||
assert!(track.file_path.is_empty());
|
||||
assert!(track.content_id.is_none() && track.fed.is_none());
|
||||
assert!(!track.is_fed_pending());
|
||||
assert_eq!(track.title, "Pilot");
|
||||
assert_eq!(track.artist_line(), "Show");
|
||||
assert_eq!(track.release_title, "Show");
|
||||
assert_eq!(track.release_year, Some(2024));
|
||||
assert_eq!(track.duration_label(), "1:31");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_is_typed_by_source() {
|
||||
let library = library_track(123);
|
||||
assert_eq!(library.key(), TrackItemKey::Library(123));
|
||||
assert_eq!(library.key().to_string(), "lib:123");
|
||||
assert_eq!(library_track(0).key(), TrackItemKey::Library(0));
|
||||
|
||||
let mut pending = library_track(-5);
|
||||
pending.release_id = -1;
|
||||
pending.file_path.clear();
|
||||
pending.fed = Some(federated("peer1", "item-9"));
|
||||
let fed_key = TrackItemKey::Federated {
|
||||
owner: "peer1".into(),
|
||||
item_id: "item-9".into(),
|
||||
};
|
||||
assert_eq!(pending.key(), fed_key);
|
||||
assert_eq!(pending.key().to_string(), "fed:peer1:item-9");
|
||||
// The process-local negative id is not part of a federated identity.
|
||||
let mut renumbered = pending.clone();
|
||||
renumbered.id = -6;
|
||||
assert_eq!(renumbered.key(), pending.key());
|
||||
|
||||
let episode = episode_ref("e1");
|
||||
let track = TrackItem::podcast_episode(episode.clone());
|
||||
let ep_key = TrackItemKey::PodcastEpisode(episode.episode_id.clone());
|
||||
assert_eq!(track.key(), ep_key);
|
||||
assert_eq!(
|
||||
track.key().to_string(),
|
||||
format!("ep:{}", episode.episode_id.as_str())
|
||||
);
|
||||
assert!(track.key().to_string().starts_with("ep:ep1:"));
|
||||
|
||||
let transient = library_track(-7);
|
||||
assert_eq!(transient.key(), TrackItemKey::Transient(-7));
|
||||
assert_eq!(transient.key().to_string(), "tmp:-7");
|
||||
|
||||
// The source wins over the numeric id: a podcast item is never keyed
|
||||
// as a library row, a federated item never by its counter value.
|
||||
let mut odd = TrackItem::podcast_episode(episode);
|
||||
odd.id = 42;
|
||||
assert_eq!(odd.key(), ep_key);
|
||||
let mut pending_minus_one = pending.clone();
|
||||
pending_minus_one.id = TrackItem::NOT_A_LIBRARY_ROW;
|
||||
assert_eq!(pending_minus_one.key(), fed_key);
|
||||
assert_ne!(pending_minus_one.key(), track.key());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn received_playback_tracks_never_copy_wire_ids() {
|
||||
// F1 wire form of an episode: id 0, release_id 0.
|
||||
let episode = episode_ref("e1");
|
||||
let wire = PlaybackTrack::podcast_episode(episode.clone());
|
||||
assert_eq!((wire.id, wire.release_id), (0, 0));
|
||||
let PlaybackItem::PodcastEpisode(item) = TrackItem::from_playback_track(&wire) else {
|
||||
panic!("an episode is not music");
|
||||
};
|
||||
assert_eq!(item.id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
assert_eq!(item.release_id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
assert!(item.artists.iter().all(|artist| artist.id < 0));
|
||||
assert_eq!(
|
||||
item.key(),
|
||||
TrackItemKey::PodcastEpisode(episode.episode_id.clone())
|
||||
);
|
||||
assert_eq!(item.podcast_episode_ref(), Some(&episode));
|
||||
// Legacy display fields an older sender filled in do not matter.
|
||||
let mut odd = wire.clone();
|
||||
odd.id = 42;
|
||||
odd.release_id = 7;
|
||||
odd.title = "Something else".into();
|
||||
let PlaybackItem::PodcastEpisode(odd) = TrackItem::from_playback_track(&odd) else {
|
||||
panic!("the source decides");
|
||||
};
|
||||
assert_eq!(odd.key(), item.key());
|
||||
assert_eq!(odd.id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
|
||||
// A private placeholder (also id 0) never becomes library row 0.
|
||||
let private = PlaybackTrack::private_placeholder();
|
||||
assert_eq!(private.id, 0);
|
||||
assert!(matches!(
|
||||
TrackItem::from_playback_track(&private),
|
||||
PlaybackItem::Private
|
||||
));
|
||||
|
||||
// Music is left to the client's music path.
|
||||
let mut music = wire;
|
||||
music.source = None;
|
||||
assert!(matches!(
|
||||
TrackItem::from_playback_track(&music),
|
||||
PlaybackItem::Music
|
||||
));
|
||||
music.source = Some(PlaybackSource::Music);
|
||||
assert!(matches!(
|
||||
TrackItem::from_playback_track(&music),
|
||||
PlaybackItem::Music
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn episodes_share_the_sentinel_id_but_not_the_key() {
|
||||
let first = TrackItem::podcast_episode(episode_ref("e1"));
|
||||
let second = TrackItem::podcast_episode(episode_ref("e2"));
|
||||
assert_eq!(first.id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
assert_eq!(second.id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
assert_eq!(first.id, second.id);
|
||||
assert_eq!(first.release_id, second.release_id);
|
||||
assert_ne!(first.key(), second.key());
|
||||
assert_ne!(first.key().to_string(), second.key().to_string());
|
||||
assert_eq!(
|
||||
TrackItem::podcast_episode(episode_ref("e1")).key(),
|
||||
first.key()
|
||||
);
|
||||
let keys: std::collections::HashSet<TrackItemKey> =
|
||||
[first.key(), second.key(), first.key()]
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert_eq!(keys.len(), 2);
|
||||
let mut ordered = [second.key(), first.key(), TrackItemKey::Library(1)];
|
||||
ordered.sort();
|
||||
assert_eq!(ordered[0], TrackItemKey::Library(1));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,544 @@
|
||||
//! Local user actions: they build the device-sync operation, apply it through
|
||||
//! the shared code path and return it for the client's sync log.
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::device_sync::SyncOpPayload;
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord,
|
||||
ProgressStatus, effective_share, merge_feed_aliases, merge_listen_session,
|
||||
next_podcast_generation,
|
||||
};
|
||||
use rusqlite::{Connection, TransactionBehavior};
|
||||
|
||||
use super::apply::{self, ApplyOutcome};
|
||||
use super::{OpStamp, PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource};
|
||||
|
||||
/// Result of [`PodcastStore::subscribe`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct SubscribeOutcome {
|
||||
/// Podcast the URL resolved to (an existing one when the URL is a known
|
||||
/// feed URL or alias).
|
||||
pub podcast_id: PodcastId,
|
||||
/// The podcast was already subscribed; no operation was emitted.
|
||||
pub already_subscribed: bool,
|
||||
/// Operations to append to the sync log.
|
||||
pub ops: Vec<PodcastLocalOp>,
|
||||
}
|
||||
|
||||
/// A listening session of this device as seen by the progress register.
|
||||
///
|
||||
/// Returned by explicit actions; periodic checkpoints of the session use its
|
||||
/// `generation` ([`PodcastStore::checkpoint`]).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProgressSession {
|
||||
/// Episode being played.
|
||||
pub episode: PodcastEpisodeRef,
|
||||
/// Session id (also the `ListenSession::session_id`).
|
||||
pub session_id: String,
|
||||
/// Generation of the session's latest explicit action.
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
/// A progress register written by a local action.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ProgressWrite {
|
||||
/// The register as written.
|
||||
pub record: ProgressRecord,
|
||||
/// The session after the action (its generation for later checkpoints).
|
||||
pub session: ProgressSession,
|
||||
/// Operation to append to the sync log.
|
||||
pub op: PodcastLocalOp,
|
||||
}
|
||||
|
||||
/// Result of [`PodcastStore::checkpoint`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum CheckpointOutcome {
|
||||
/// The checkpoint was written.
|
||||
Written(Box<ProgressWrite>),
|
||||
/// The register holds a greater generation (another device or an
|
||||
/// explicit action took over). Nothing was written: stop writing at the
|
||||
/// old generation, or continue locally with an explicit action
|
||||
/// ([`PodcastStore::start_session`] / [`PodcastStore::seek`]).
|
||||
Superseded {
|
||||
/// Current register.
|
||||
current: ProgressRecord,
|
||||
},
|
||||
/// This session already completed the episode at its generation; a
|
||||
/// checkpoint would turn it back into "in progress". Nothing was written.
|
||||
AlreadyCompleted {
|
||||
/// Current register.
|
||||
current: ProgressRecord,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum ExplicitAction {
|
||||
Start(i64),
|
||||
Seek(i64),
|
||||
Restart,
|
||||
Complete(i64),
|
||||
MarkPlayed,
|
||||
MarkUnplayed,
|
||||
}
|
||||
|
||||
fn invalid(reason: impl ToString) -> anyhow::Error {
|
||||
PodcastStoreError::Invalid(reason.to_string()).into()
|
||||
}
|
||||
|
||||
/// Validates the payload (with a provisional stamp), draws the real stamp,
|
||||
/// and applies the operation through [`apply::apply_op`].
|
||||
pub(crate) fn emit(
|
||||
conn: &Connection,
|
||||
stamps: &mut dyn StampSource,
|
||||
make: impl Fn(&OpStamp) -> SyncOpPayload,
|
||||
) -> Result<PodcastLocalOp> {
|
||||
let device_id = stamps.device_id().to_string();
|
||||
let provisional = OpStamp::new("provisional", device_id.clone(), 0);
|
||||
provisional
|
||||
.wire(make(&provisional))
|
||||
.validate_podcast()
|
||||
.map_err(invalid)?;
|
||||
let stamp = stamps.next_stamp()?;
|
||||
if stamp.origin_device_id != device_id {
|
||||
return Err(invalid("stamp origin differs from the local device id"));
|
||||
}
|
||||
let payload = make(&stamp);
|
||||
let op = stamp.wire(payload.clone());
|
||||
match apply::apply_op(conn, &op)? {
|
||||
ApplyOutcome::Rejected(reason) => Err(invalid(reason)),
|
||||
_ => Ok(PodcastLocalOp { stamp, payload }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves a feed URL against every known URL of every podcast.
|
||||
pub(crate) fn resolve_podcast(conn: &Connection, url: &FeedUrl) -> Result<PodcastId> {
|
||||
let mut statement = conn.prepare_cached(
|
||||
"SELECT podcast_id FROM podcast_feed_aliases WHERE feed_url = ?1
|
||||
UNION SELECT podcast_id FROM podcasts WHERE feed_url = ?1
|
||||
UNION SELECT podcast_id FROM podcast_subscriptions
|
||||
WHERE json_extract(podcast_json, '$.feed_url') = ?1",
|
||||
)?;
|
||||
let ids = statement
|
||||
.query_map([url.as_str()], |row| super::podcast_id_col(row.get(0)?))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let pairs: Vec<(PodcastId, FeedUrl)> = ids.into_iter().map(|id| (id, url.clone())).collect();
|
||||
Ok(music_dht::podcast::resolve_podcast_id(
|
||||
url,
|
||||
pairs.iter().map(|(id, url)| (id, url)),
|
||||
))
|
||||
}
|
||||
|
||||
fn catalog_meta(conn: &Connection, podcast_id: &PodcastId) -> Result<Option<PodcastMeta>> {
|
||||
use rusqlite::OptionalExtension as _;
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT feed_url, title, author, description, link, image_url
|
||||
FROM podcasts WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
Ok(PodcastMeta {
|
||||
feed_url: super::feed_url_col(row.get(0)?)?,
|
||||
title: row.get(1)?,
|
||||
author: row.get(2)?,
|
||||
description: row.get(3)?,
|
||||
link: row.get(4)?,
|
||||
image_url: row.get(5)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
fn share_payload(conn: &Connection, podcast_id: &PodcastId, shared: bool) -> Result<SyncOpPayload> {
|
||||
let subscription = apply::load_subscription_register(conn, podcast_id)?;
|
||||
let share = apply::load_share(conn, podcast_id)?;
|
||||
Ok(SyncOpPayload::PodcastShareSet {
|
||||
podcast_id: podcast_id.clone(),
|
||||
shared,
|
||||
generation: next_podcast_generation(subscription.as_ref(), share.as_ref()),
|
||||
})
|
||||
}
|
||||
|
||||
/// The reference carried by progress operations: identity and display only.
|
||||
fn op_ref(episode: &PodcastEpisodeRef) -> PodcastEpisodeRef {
|
||||
let mut episode = episode.clone();
|
||||
episode.progress_generation = None;
|
||||
episode
|
||||
}
|
||||
|
||||
impl PodcastStore {
|
||||
/// Subscribes to the podcast of `podcast.feed_url` (explicit user action).
|
||||
///
|
||||
/// The URL is resolved against known feed URLs and aliases first, so
|
||||
/// subscribing again, through a redirect target or from someone else's
|
||||
/// card reuses the existing podcast with its progress and history. When
|
||||
/// the podcast is already subscribed nothing is emitted. Otherwise one
|
||||
/// `podcast_subscription_set` with generation
|
||||
/// `next_podcast_generation(..)` and the known aliases is applied and
|
||||
/// returned. The new subscription is private until
|
||||
/// [`PodcastStore::set_shared`].
|
||||
///
|
||||
/// Pass the channel metadata of the fetched feed
|
||||
/// (`ParsedFeed::to_podcast_meta(url)`), then ingest the episodes with
|
||||
/// [`PodcastStore::apply_feed`].
|
||||
pub fn subscribe(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
podcast: &PodcastMeta,
|
||||
) -> Result<SubscribeOutcome> {
|
||||
let meta = podcast.bounded();
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let podcast_id = resolve_podcast(&tx, &meta.feed_url)?;
|
||||
let current = apply::load_subscription_register(&tx, &podcast_id)?;
|
||||
if current.as_ref().is_some_and(|record| record.subscribed) {
|
||||
return Ok(SubscribeOutcome {
|
||||
podcast_id,
|
||||
already_subscribed: true,
|
||||
ops: Vec::new(),
|
||||
});
|
||||
}
|
||||
let share = apply::load_share(&tx, &podcast_id)?;
|
||||
let generation = next_podcast_generation(current.as_ref(), share.as_ref());
|
||||
let aliases = merge_feed_aliases(
|
||||
&podcast_id,
|
||||
&apply::load_aliases(&tx, &podcast_id)?,
|
||||
std::slice::from_ref(&meta.feed_url),
|
||||
);
|
||||
let op = emit(&tx, stamps, |_| SyncOpPayload::PodcastSubscriptionSet {
|
||||
podcast_id: podcast_id.clone(),
|
||||
subscribed: true,
|
||||
podcast: meta.clone(),
|
||||
feed_aliases: aliases.clone(),
|
||||
generation,
|
||||
})?;
|
||||
tx.commit()?;
|
||||
Ok(SubscribeOutcome {
|
||||
podcast_id,
|
||||
already_subscribed: false,
|
||||
ops: vec![op],
|
||||
})
|
||||
}
|
||||
|
||||
/// Unsubscribes (explicit user action).
|
||||
///
|
||||
/// Emits `podcast_subscription_set { subscribed: false }` followed by
|
||||
/// `podcast_share_set { shared: false }`, each with the next podcast
|
||||
/// generation. Progress, sessions and the catalog are kept, so
|
||||
/// subscribing again continues where the user stopped. Does nothing when
|
||||
/// the podcast is not subscribed.
|
||||
pub fn unsubscribe(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Vec<PodcastLocalOp>> {
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let Some(current) = apply::load_subscription_register(&tx, podcast_id)? else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
if !current.subscribed {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let share = apply::load_share(&tx, podcast_id)?;
|
||||
let generation = next_podcast_generation(Some(¤t), share.as_ref());
|
||||
let mut meta = catalog_meta(&tx, podcast_id)?
|
||||
.map(|meta| meta.bounded())
|
||||
.unwrap_or_else(|| current.podcast.clone());
|
||||
meta.feed_url = current.podcast.feed_url.clone();
|
||||
let aliases = current.feed_aliases.clone();
|
||||
let unsubscribe = emit(&tx, stamps, |_| SyncOpPayload::PodcastSubscriptionSet {
|
||||
podcast_id: podcast_id.clone(),
|
||||
subscribed: false,
|
||||
podcast: meta.clone(),
|
||||
feed_aliases: aliases.clone(),
|
||||
generation,
|
||||
})?;
|
||||
let revoke_payload = share_payload(&tx, podcast_id, false)?;
|
||||
let revoke = emit(&tx, stamps, |_| revoke_payload.clone())?;
|
||||
tx.commit()?;
|
||||
Ok(vec![unsubscribe, revoke])
|
||||
}
|
||||
|
||||
/// Sets the decision to publish the podcast to the federation.
|
||||
///
|
||||
/// Sharing needs an active subscription and is a no-op when the podcast
|
||||
/// is already effectively shared (`effective_share`). Revoking is a no-op
|
||||
/// when no share decision says `true`. The write uses the next podcast
|
||||
/// generation, so it orders after the subscription it observed.
|
||||
pub fn set_shared(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
podcast_id: &PodcastId,
|
||||
shared: bool,
|
||||
) -> Result<Vec<PodcastLocalOp>> {
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let subscription = apply::load_subscription_register(&tx, podcast_id)?;
|
||||
let share = apply::load_share(&tx, podcast_id)?;
|
||||
if shared {
|
||||
if !subscription
|
||||
.as_ref()
|
||||
.is_some_and(|record| record.subscribed)
|
||||
{
|
||||
return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into());
|
||||
}
|
||||
if effective_share(subscription.as_ref(), share.as_ref()) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
} else if !share.as_ref().is_some_and(|record| record.shared) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let payload = share_payload(&tx, podcast_id, shared)?;
|
||||
let op = emit(&tx, stamps, |_| payload.clone())?;
|
||||
tx.commit()?;
|
||||
Ok(vec![op])
|
||||
}
|
||||
|
||||
fn explicit_progress(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
session_id: Option<&str>,
|
||||
carried_generation: Option<u64>,
|
||||
action: ExplicitAction,
|
||||
) -> Result<ProgressWrite> {
|
||||
episode.validate().map_err(invalid)?;
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let current = apply::load_progress(&tx, &episode.episode_id)?;
|
||||
let max_seen = current
|
||||
.as_ref()
|
||||
.map(|record| record.generation)
|
||||
.into_iter()
|
||||
.chain(carried_generation)
|
||||
.max();
|
||||
let generation = ProgressRecord::explicit_generation(max_seen);
|
||||
let current_position = current.as_ref().map_or(0, |record| record.position_ms);
|
||||
let (position_ms, status) = match action {
|
||||
ExplicitAction::Start(position) | ExplicitAction::Seek(position) => {
|
||||
(position, ProgressStatus::InProgress)
|
||||
}
|
||||
ExplicitAction::Restart => (0, ProgressStatus::InProgress),
|
||||
ExplicitAction::Complete(position) => (position, ProgressStatus::Completed),
|
||||
ExplicitAction::MarkPlayed => (current_position, ProgressStatus::Completed),
|
||||
ExplicitAction::MarkUnplayed => (0, ProgressStatus::NotStarted),
|
||||
};
|
||||
let reference = op_ref(episode);
|
||||
let op = emit(&tx, stamps, |stamp| SyncOpPayload::PodcastProgressSet {
|
||||
episode_id: episode.episode_id.clone(),
|
||||
podcast_id: episode.podcast_id.clone(),
|
||||
position_ms,
|
||||
status,
|
||||
generation,
|
||||
session_id: session_id.unwrap_or(&stamp.op_id).to_string(),
|
||||
episode: Some(reference.clone()),
|
||||
})?;
|
||||
tx.commit()?;
|
||||
let wire = op.stamp.wire(op.payload.clone());
|
||||
let record = wire.podcast_progress().expect("progress payload");
|
||||
Ok(ProgressWrite {
|
||||
session: ProgressSession {
|
||||
episode: episode.clone(),
|
||||
session_id: record.session_id.clone(),
|
||||
generation,
|
||||
},
|
||||
record,
|
||||
op,
|
||||
})
|
||||
}
|
||||
|
||||
/// Starts a listening session at `position_ms` (explicit action).
|
||||
///
|
||||
/// The generation is `explicit_generation(max(local, carried_generation))`;
|
||||
/// pass `PodcastEpisodeRef::progress_generation` of a queue item received
|
||||
/// through a handoff as `carried_generation`. Ordinary Play resumes from
|
||||
/// [`PodcastStore::resume_position_ms`].
|
||||
pub fn start_session(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
session_id: &str,
|
||||
position_ms: i64,
|
||||
carried_generation: Option<u64>,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(
|
||||
stamps,
|
||||
episode,
|
||||
Some(session_id),
|
||||
carried_generation,
|
||||
ExplicitAction::Start(position_ms),
|
||||
)
|
||||
}
|
||||
|
||||
/// Periodic checkpoint (or pause/stop save) of a running session.
|
||||
///
|
||||
/// Keeps the session generation. When the register already holds a
|
||||
/// greater generation nothing is written ([`CheckpointOutcome::Superseded`]).
|
||||
pub fn checkpoint(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
session: &ProgressSession,
|
||||
position_ms: i64,
|
||||
) -> Result<CheckpointOutcome> {
|
||||
let episode = &session.episode;
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
if let Some(current) = apply::load_progress(&tx, &episode.episode_id)? {
|
||||
if current.generation > session.generation {
|
||||
return Ok(CheckpointOutcome::Superseded { current });
|
||||
}
|
||||
if current.generation == session.generation
|
||||
&& current.session_id == session.session_id
|
||||
&& current.status == ProgressStatus::Completed
|
||||
{
|
||||
return Ok(CheckpointOutcome::AlreadyCompleted { current });
|
||||
}
|
||||
}
|
||||
let reference = op_ref(episode);
|
||||
let op = emit(&tx, stamps, |_| SyncOpPayload::PodcastProgressSet {
|
||||
episode_id: episode.episode_id.clone(),
|
||||
podcast_id: episode.podcast_id.clone(),
|
||||
position_ms,
|
||||
status: ProgressStatus::InProgress,
|
||||
generation: session.generation,
|
||||
session_id: session.session_id.clone(),
|
||||
episode: Some(reference.clone()),
|
||||
})?;
|
||||
tx.commit()?;
|
||||
let record = op
|
||||
.stamp
|
||||
.wire(op.payload.clone())
|
||||
.podcast_progress()
|
||||
.expect("progress payload");
|
||||
Ok(CheckpointOutcome::Written(Box::new(ProgressWrite {
|
||||
record,
|
||||
session: session.clone(),
|
||||
op,
|
||||
})))
|
||||
}
|
||||
|
||||
/// User seek during a session (explicit action; the returned session
|
||||
/// carries the new generation for later checkpoints).
|
||||
pub fn seek(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
session: &ProgressSession,
|
||||
position_ms: i64,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(
|
||||
stamps,
|
||||
&session.episode,
|
||||
Some(&session.session_id),
|
||||
Some(session.generation),
|
||||
ExplicitAction::Seek(position_ms),
|
||||
)
|
||||
}
|
||||
|
||||
/// The session reached the end (`reaches_completion` or end of stream):
|
||||
/// writes `completed` with an explicit generation. Record the session
|
||||
/// itself with [`PodcastStore::record_listen_session`].
|
||||
pub fn complete_session(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
session: &ProgressSession,
|
||||
position_ms: i64,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(
|
||||
stamps,
|
||||
&session.episode,
|
||||
Some(&session.session_id),
|
||||
Some(session.generation),
|
||||
ExplicitAction::Complete(position_ms),
|
||||
)
|
||||
}
|
||||
|
||||
/// "Start over": position 0, in progress, explicit generation.
|
||||
pub fn restart(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
session_id: &str,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(
|
||||
stamps,
|
||||
episode,
|
||||
Some(session_id),
|
||||
None,
|
||||
ExplicitAction::Restart,
|
||||
)
|
||||
}
|
||||
|
||||
/// "Mark played": status completed, explicit generation. Creates no
|
||||
/// listening session and invents no listening time.
|
||||
pub fn mark_played(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(stamps, episode, None, None, ExplicitAction::MarkPlayed)
|
||||
}
|
||||
|
||||
/// "Mark unplayed": status not started at position 0, explicit generation.
|
||||
pub fn mark_unplayed(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(stamps, episode, None, None, ExplicitAction::MarkUnplayed)
|
||||
}
|
||||
|
||||
/// Records (or grows) a listening session of this device.
|
||||
///
|
||||
/// Sessions below the history threshold (`ListenSession::should_record`)
|
||||
/// and versions that do not grow the stored session emit nothing.
|
||||
/// `session.device_id` must be the local device id. The episode reference
|
||||
/// defaults to the catalog's.
|
||||
pub fn record_listen_session(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
session: &ListenSession,
|
||||
episode: Option<&PodcastEpisodeRef>,
|
||||
) -> Result<Option<PodcastLocalOp>> {
|
||||
if session.device_id != stamps.device_id() {
|
||||
return Err(invalid(
|
||||
"session device_id differs from the local device id",
|
||||
));
|
||||
}
|
||||
session.validate().map_err(invalid)?;
|
||||
if !session.should_record() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
if let Some(current) = apply::load_session(&tx, &session.session_id)? {
|
||||
let merged = merge_listen_session(¤t, session).map_err(invalid)?;
|
||||
if merged == current {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
let reference = match episode {
|
||||
Some(episode) => Some(op_ref(episode)),
|
||||
None => super::queries::episode_ref_in(&tx, &session.episode_id)?.map(|r| op_ref(&r)),
|
||||
};
|
||||
let op = emit(&tx, stamps, |_| SyncOpPayload::PodcastListenRecorded {
|
||||
session: session.clone(),
|
||||
episode: reference.clone(),
|
||||
})?;
|
||||
tx.commit()?;
|
||||
Ok(Some(op))
|
||||
}
|
||||
|
||||
/// Resolves a feed URL to a podcast id: the existing podcast when the
|
||||
/// URL is a known feed URL or alias (`resolve_podcast_id`), otherwise
|
||||
/// the id a first subscription through this URL would derive.
|
||||
pub fn resolve_podcast_id(&self, url: &FeedUrl) -> Result<PodcastId> {
|
||||
resolve_podcast(&self.lock(), url)
|
||||
}
|
||||
|
||||
/// The episode reference for queueing or progress actions, or `None`
|
||||
/// when the episode is unknown.
|
||||
pub fn episode_ref(&self, episode_id: &EpisodeId) -> Result<Option<PodcastEpisodeRef>> {
|
||||
super::queries::episode_ref_in(&self.lock(), episode_id)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,742 @@
|
||||
//! The single code path that applies podcast records to the store.
|
||||
//!
|
||||
//! Remote operations, local actions and snapshot sections all end here.
|
||||
//! Records come from the `SyncOpWire` materializers of
|
||||
//! `music_dht::device_sync` and are merged with the functions of
|
||||
//! `music_dht::podcast`, so applying is idempotent and order-independent.
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot};
|
||||
use music_dht::podcast::{
|
||||
EpisodeIdentityHints, EpisodeMeta, FeedUrl, KnownEpisode, ListenSession, PodcastEpisodeRef,
|
||||
PodcastError, PodcastId, ProgressRecord, ShareRecord, SubscriptionRecord, merge_feed_aliases,
|
||||
merge_listen_session,
|
||||
};
|
||||
use rusqlite::{Connection, OptionalExtension as _, TransactionBehavior, params};
|
||||
|
||||
use super::{
|
||||
PROGRESS_COLUMNS, PodcastStore, PodcastStoreError, SESSION_COLUMNS, episode_id_col,
|
||||
feed_url_col, generation_to_sql, now_ms, podcast_meta_col, progress_from_row, session_from_row,
|
||||
status_to_sql,
|
||||
};
|
||||
|
||||
/// Result of applying one operation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum ApplyOutcome {
|
||||
/// Stored state changed.
|
||||
Changed,
|
||||
/// The operation was already reflected (duplicate, or older than the
|
||||
/// stored register).
|
||||
Unchanged,
|
||||
/// Not a podcast operation; nothing to do here.
|
||||
Ignored,
|
||||
/// The operation violates the contract and was skipped. Acknowledge it
|
||||
/// anyway so the log keeps moving (`PODCASTS.md`).
|
||||
Rejected(String),
|
||||
}
|
||||
|
||||
/// Counters returned by [`PodcastStore::apply_snapshot`].
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct SnapshotApplyStats {
|
||||
/// Records that changed stored state.
|
||||
pub changed: usize,
|
||||
/// Valid records already reflected locally.
|
||||
pub unchanged: usize,
|
||||
/// Records that failed validation and were skipped.
|
||||
pub rejected: usize,
|
||||
/// Whether the podcast sections were ignored for exceeding the counts.
|
||||
pub ignored_over_limits: bool,
|
||||
}
|
||||
|
||||
/// Where an episode row came from.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum EpisodeSource {
|
||||
/// Parsed from the podcast's feed on this device.
|
||||
Feed,
|
||||
/// Learned from a `PodcastEpisodeRef` (sync operation or queue item).
|
||||
Sync,
|
||||
}
|
||||
|
||||
impl EpisodeSource {
|
||||
fn as_sql(self) -> &'static str {
|
||||
match self {
|
||||
Self::Feed => "feed",
|
||||
Self::Sync => "sync",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid(error: impl ToString) -> anyhow::Error {
|
||||
PodcastStoreError::Invalid(error.to_string()).into()
|
||||
}
|
||||
|
||||
pub(crate) fn load_subscription_register(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Option<SubscriptionRecord>> {
|
||||
let row = conn
|
||||
.query_row(
|
||||
"SELECT subscribed, podcast_json, generation, hlc_ms, op_id
|
||||
FROM podcast_subscriptions WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
let generation: i64 = row.get(2)?;
|
||||
Ok(SubscriptionRecord {
|
||||
podcast_id: podcast_id.clone(),
|
||||
subscribed: row.get(0)?,
|
||||
podcast: podcast_meta_col(&row.get::<_, String>(1)?)?,
|
||||
feed_aliases: Vec::new(),
|
||||
generation: u64::try_from(generation).unwrap_or(0),
|
||||
hlc_ms: row.get(3)?,
|
||||
op_id: row.get(4)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?;
|
||||
let Some(mut record) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
record.feed_aliases = load_aliases(conn, podcast_id)?;
|
||||
Ok(Some(record))
|
||||
}
|
||||
|
||||
pub(crate) fn load_aliases(conn: &Connection, podcast_id: &PodcastId) -> Result<Vec<FeedUrl>> {
|
||||
let mut statement = conn.prepare_cached(
|
||||
"SELECT feed_url FROM podcast_feed_aliases WHERE podcast_id = ?1 ORDER BY feed_url",
|
||||
)?;
|
||||
let urls = statement
|
||||
.query_map([podcast_id.as_str()], |row| feed_url_col(row.get(0)?))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(urls)
|
||||
}
|
||||
|
||||
pub(crate) fn load_share(conn: &Connection, podcast_id: &PodcastId) -> Result<Option<ShareRecord>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT shared, generation, hlc_ms, op_id FROM podcast_shares WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
let generation: i64 = row.get(1)?;
|
||||
Ok(ShareRecord {
|
||||
podcast_id: podcast_id.clone(),
|
||||
shared: row.get(0)?,
|
||||
generation: u64::try_from(generation).unwrap_or(0),
|
||||
hlc_ms: row.get(2)?,
|
||||
op_id: row.get(3)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
pub(crate) fn load_progress(
|
||||
conn: &Connection,
|
||||
episode_id: &music_dht::podcast::EpisodeId,
|
||||
) -> Result<Option<ProgressRecord>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
&format!("SELECT {PROGRESS_COLUMNS} FROM episode_progress p WHERE p.episode_id = ?1"),
|
||||
[episode_id.as_str()],
|
||||
|row| progress_from_row(row, 0),
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
pub(crate) fn load_session(conn: &Connection, session_id: &str) -> Result<Option<ListenSession>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
&format!(
|
||||
"SELECT {SESSION_COLUMNS} FROM podcast_listen_sessions s WHERE s.session_id = ?1"
|
||||
),
|
||||
[session_id],
|
||||
|row| session_from_row(row, 0),
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
/// Replaces the alias set of a podcast with `merged` when it differs.
|
||||
fn store_aliases(conn: &Connection, podcast_id: &PodcastId, merged: &[FeedUrl]) -> Result<bool> {
|
||||
let current = load_aliases(conn, podcast_id)?;
|
||||
if current == merged {
|
||||
return Ok(false);
|
||||
}
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_feed_aliases WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
)?;
|
||||
let mut insert = conn.prepare_cached(
|
||||
"INSERT INTO podcast_feed_aliases (podcast_id, feed_url) VALUES (?1, ?2)",
|
||||
)?;
|
||||
for url in merged {
|
||||
insert.execute(params![podcast_id.as_str(), url.as_str()])?;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Merges feed URLs into the bounded grow-only alias set of a podcast.
|
||||
///
|
||||
/// Returns whether the stored set changed.
|
||||
pub(crate) fn merge_aliases(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
urls: &[FeedUrl],
|
||||
) -> Result<bool> {
|
||||
let current = load_aliases(conn, podcast_id)?;
|
||||
let merged = merge_feed_aliases(podcast_id, ¤t, urls);
|
||||
store_aliases(conn, podcast_id, &merged)
|
||||
}
|
||||
|
||||
/// Inserts the catalog row of a podcast if it does not exist yet.
|
||||
fn ensure_podcast_row(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
feed_url: &FeedUrl,
|
||||
title: &str,
|
||||
now: i64,
|
||||
) -> Result<()> {
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO podcasts
|
||||
(podcast_id, feed_url, title, meta_from_feed, created_at_ms, updated_at_ms)
|
||||
VALUES (?1, ?2, ?3, 0, ?4, ?4)",
|
||||
params![podcast_id.as_str(), feed_url.as_str(), title, now],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Applies a subscription register record (operation or snapshot).
|
||||
///
|
||||
/// The register is last-writer-wins ([`SubscriptionRecord::supersedes`]);
|
||||
/// the alias set is a bounded grow-only set merged independently, so the
|
||||
/// stored aliases are the bounded union of every URL seen (equal to what
|
||||
/// `merge_subscription` computes). The catalog row is created from the
|
||||
/// register metadata and follows the winning register until this device has
|
||||
/// refreshed the feed itself.
|
||||
pub(crate) fn apply_subscription_record(
|
||||
conn: &Connection,
|
||||
record: &SubscriptionRecord,
|
||||
now: i64,
|
||||
) -> Result<bool> {
|
||||
generation_to_sql(record.generation)?;
|
||||
let podcast_id = &record.podcast_id;
|
||||
let current = load_subscription_register(conn, podcast_id)?;
|
||||
let wins = current
|
||||
.as_ref()
|
||||
.is_none_or(|current| record.supersedes(current));
|
||||
let mut changed = false;
|
||||
if wins {
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_subscriptions
|
||||
(podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(podcast_id) DO UPDATE SET
|
||||
subscribed = excluded.subscribed, podcast_json = excluded.podcast_json,
|
||||
generation = excluded.generation, hlc_ms = excluded.hlc_ms,
|
||||
op_id = excluded.op_id",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
record.subscribed,
|
||||
serde_json::to_string(&record.podcast)?,
|
||||
generation_to_sql(record.generation)?,
|
||||
record.hlc_ms,
|
||||
record.op_id,
|
||||
],
|
||||
)?;
|
||||
changed = true;
|
||||
}
|
||||
let mut urls = record.feed_aliases.clone();
|
||||
urls.push(record.podcast.feed_url.clone());
|
||||
changed |= merge_aliases(conn, podcast_id, &urls)?;
|
||||
|
||||
ensure_podcast_row(
|
||||
conn,
|
||||
podcast_id,
|
||||
&record.podcast.feed_url,
|
||||
&record.podcast.title,
|
||||
now,
|
||||
)?;
|
||||
if wins {
|
||||
let meta = &record.podcast;
|
||||
conn.execute(
|
||||
"UPDATE podcasts
|
||||
SET feed_url = ?2, title = ?3, author = ?4,
|
||||
description = COALESCE(?5, description), link = ?6, image_url = ?7,
|
||||
updated_at_ms = ?8
|
||||
WHERE podcast_id = ?1 AND meta_from_feed = 0",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
meta.feed_url.as_str(),
|
||||
meta.title,
|
||||
meta.author,
|
||||
meta.description,
|
||||
meta.link,
|
||||
meta.image_url,
|
||||
now
|
||||
],
|
||||
)?;
|
||||
}
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
/// Applies a share register record.
|
||||
pub(crate) fn apply_share_record(conn: &Connection, record: &ShareRecord) -> Result<bool> {
|
||||
let generation = generation_to_sql(record.generation)?;
|
||||
let current = load_share(conn, &record.podcast_id)?;
|
||||
if current
|
||||
.as_ref()
|
||||
.is_some_and(|current| !record.supersedes(current))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_shares (podcast_id, shared, generation, hlc_ms, op_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)
|
||||
ON CONFLICT(podcast_id) DO UPDATE SET
|
||||
shared = excluded.shared, generation = excluded.generation,
|
||||
hlc_ms = excluded.hlc_ms, op_id = excluded.op_id",
|
||||
params![
|
||||
record.podcast_id.as_str(),
|
||||
record.shared,
|
||||
generation,
|
||||
record.hlc_ms,
|
||||
record.op_id
|
||||
],
|
||||
)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Applies a progress register record. Never touches the catalog.
|
||||
pub(crate) fn apply_progress_record(conn: &Connection, record: &ProgressRecord) -> Result<bool> {
|
||||
let generation = generation_to_sql(record.generation)?;
|
||||
let current = load_progress(conn, &record.episode_id)?;
|
||||
if current
|
||||
.as_ref()
|
||||
.is_some_and(|current| !record.supersedes(current))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO episode_progress
|
||||
(episode_id, podcast_id, position_ms, status, generation, session_id, hlc_ms,
|
||||
device_id, op_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
|
||||
ON CONFLICT(episode_id) DO UPDATE SET
|
||||
podcast_id = excluded.podcast_id, position_ms = excluded.position_ms,
|
||||
status = excluded.status, generation = excluded.generation,
|
||||
session_id = excluded.session_id, hlc_ms = excluded.hlc_ms,
|
||||
device_id = excluded.device_id, op_id = excluded.op_id",
|
||||
params![
|
||||
record.episode_id.as_str(),
|
||||
record.podcast_id.as_str(),
|
||||
record.position_ms,
|
||||
status_to_sql(record.status),
|
||||
generation,
|
||||
record.session_id,
|
||||
record.hlc_ms,
|
||||
record.device_id,
|
||||
record.op_id,
|
||||
],
|
||||
)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Adds or grows a listening session (grow-only set keyed by `session_id`).
|
||||
pub(crate) fn apply_listen_session(conn: &Connection, session: &ListenSession) -> Result<bool> {
|
||||
let current = load_session(conn, &session.session_id)?;
|
||||
let merged = match ¤t {
|
||||
Some(current) => merge_listen_session(current, session).map_err(invalid)?,
|
||||
None => session.clone(),
|
||||
};
|
||||
if current.as_ref() == Some(&merged) {
|
||||
return Ok(false);
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_listen_sessions
|
||||
(session_id, episode_id, podcast_id, device_id, started_at_ms, ended_at_ms,
|
||||
start_position_ms, end_position_ms, listened_ms, completed)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||
ON CONFLICT(session_id) DO UPDATE SET
|
||||
started_at_ms = excluded.started_at_ms, ended_at_ms = excluded.ended_at_ms,
|
||||
start_position_ms = excluded.start_position_ms,
|
||||
end_position_ms = excluded.end_position_ms,
|
||||
listened_ms = excluded.listened_ms, completed = excluded.completed",
|
||||
params![
|
||||
merged.session_id,
|
||||
merged.episode_id.as_str(),
|
||||
merged.podcast_id.as_str(),
|
||||
merged.device_id,
|
||||
merged.started_at_ms,
|
||||
merged.ended_at_ms,
|
||||
merged.start_position_ms,
|
||||
merged.end_position_ms,
|
||||
merged.listened_ms,
|
||||
merged.completed,
|
||||
],
|
||||
)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Inserts an episode row.
|
||||
pub(crate) fn insert_episode(
|
||||
conn: &Connection,
|
||||
meta: &EpisodeMeta,
|
||||
source: EpisodeSource,
|
||||
in_feed: bool,
|
||||
legacy_guid: Option<&str>,
|
||||
first_seen_at_ms: i64,
|
||||
now: i64,
|
||||
) -> Result<()> {
|
||||
conn.execute(
|
||||
"INSERT INTO episodes
|
||||
(episode_id, podcast_id, guid, title, description, season, episode_number,
|
||||
episode_type, published_at_ms, duration_ms, audio_url, mime, byte_len,
|
||||
image_url, link, source, in_feed, legacy_guid, first_seen_at_ms, updated_at_ms)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17,
|
||||
?18, ?19, ?20)",
|
||||
params![
|
||||
meta.episode_id.as_str(),
|
||||
meta.podcast_id.as_str(),
|
||||
meta.guid,
|
||||
meta.title,
|
||||
meta.description,
|
||||
meta.season,
|
||||
meta.episode_number,
|
||||
episode_type_to_sql(meta.episode_type),
|
||||
meta.published_at_ms,
|
||||
meta.duration_ms,
|
||||
meta.audio_url,
|
||||
meta.mime,
|
||||
meta.byte_len.and_then(|len| i64::try_from(len).ok()),
|
||||
meta.image_url,
|
||||
meta.link,
|
||||
source.as_sql(),
|
||||
in_feed,
|
||||
legacy_guid,
|
||||
first_seen_at_ms,
|
||||
now,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn episode_type_to_sql(kind: music_dht::podcast::EpisodeType) -> &'static str {
|
||||
match kind {
|
||||
music_dht::podcast::EpisodeType::Full => "full",
|
||||
music_dht::podcast::EpisodeType::Trailer => "trailer",
|
||||
music_dht::podcast::EpisodeType::Bonus => "bonus",
|
||||
}
|
||||
}
|
||||
|
||||
/// Records an episode learned through sync when the catalog does not know it.
|
||||
///
|
||||
/// Feed metadata always wins: an existing row is left untouched. The
|
||||
/// podcast catalog row is created from the reference when missing (no
|
||||
/// register is written).
|
||||
pub(crate) fn ensure_episode_from_ref(
|
||||
conn: &Connection,
|
||||
episode: &PodcastEpisodeRef,
|
||||
) -> Result<()> {
|
||||
let exists = conn
|
||||
.query_row(
|
||||
"SELECT 1 FROM episodes WHERE episode_id = ?1",
|
||||
[episode.episode_id.as_str()],
|
||||
|_| Ok(()),
|
||||
)
|
||||
.optional()?
|
||||
.is_some();
|
||||
if exists {
|
||||
return Ok(());
|
||||
}
|
||||
let now = now_ms();
|
||||
ensure_podcast_row(
|
||||
conn,
|
||||
&episode.podcast_id,
|
||||
&episode.feed_url,
|
||||
&episode.podcast_title,
|
||||
now,
|
||||
)?;
|
||||
let meta = EpisodeMeta {
|
||||
episode_id: episode.episode_id.clone(),
|
||||
podcast_id: episode.podcast_id.clone(),
|
||||
guid: None,
|
||||
title: episode.episode_title.clone(),
|
||||
description: None,
|
||||
season: episode.season,
|
||||
episode_number: episode.episode_number,
|
||||
episode_type: Default::default(),
|
||||
published_at_ms: episode.published_at_ms,
|
||||
duration_ms: episode.duration_ms,
|
||||
audio_url: episode.audio_url.clone(),
|
||||
mime: episode.mime.clone(),
|
||||
byte_len: None,
|
||||
image_url: episode.image_url.clone(),
|
||||
link: None,
|
||||
};
|
||||
insert_episode(conn, &meta, EpisodeSource::Sync, false, None, now, now)?;
|
||||
refresh_seasons(conn, &episode.podcast_id)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Recomputes the derived season rows of a podcast from its episodes.
|
||||
pub(crate) fn refresh_seasons(conn: &Connection, podcast_id: &PodcastId) -> Result<()> {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_seasons WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
)?;
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_seasons
|
||||
(podcast_id, season, episode_count, first_published_at_ms, last_published_at_ms)
|
||||
SELECT podcast_id, season, COUNT(*), MIN(published_at_ms), MAX(published_at_ms)
|
||||
FROM episodes WHERE podcast_id = ?1 GROUP BY season",
|
||||
[podcast_id.as_str()],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Known episodes of a podcast with the identity hints they were last seen with.
|
||||
pub(crate) fn known_episodes(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Vec<KnownEpisode>> {
|
||||
let mut statement = conn.prepare_cached(
|
||||
"SELECT episode_id, guid, audio_url, link, published_at_ms, title
|
||||
FROM episodes WHERE podcast_id = ?1 ORDER BY episode_id",
|
||||
)?;
|
||||
let known = statement
|
||||
.query_map([podcast_id.as_str()], |row| {
|
||||
Ok(KnownEpisode {
|
||||
episode_id: episode_id_col(row.get(0)?)?,
|
||||
hints: EpisodeIdentityHints {
|
||||
guid: row.get(1)?,
|
||||
enclosure_url: Some(row.get(2)?),
|
||||
link: row.get(3)?,
|
||||
published_at_ms: row.get(4)?,
|
||||
title: Some(row.get(5)?),
|
||||
},
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(known)
|
||||
}
|
||||
|
||||
/// Applies one podcast operation inside the caller's transaction.
|
||||
pub(crate) fn apply_op(conn: &Connection, op: &SyncOpWire) -> Result<ApplyOutcome> {
|
||||
if !op.payload.is_podcast() {
|
||||
return Ok(ApplyOutcome::Ignored);
|
||||
}
|
||||
if let Err(error) = op.validate_podcast() {
|
||||
return Ok(ApplyOutcome::Rejected(error.to_string()));
|
||||
}
|
||||
let result = match &op.payload {
|
||||
SyncOpPayload::PodcastSubscriptionSet { .. } => {
|
||||
let record = op.podcast_subscription().expect("subscription payload");
|
||||
apply_subscription_record(conn, &record, now_ms())
|
||||
}
|
||||
SyncOpPayload::PodcastShareSet { .. } => {
|
||||
apply_share_record(conn, &op.podcast_share().expect("share payload"))
|
||||
}
|
||||
SyncOpPayload::PodcastFeedAliasesAdded { .. } => {
|
||||
let (podcast_id, urls) = op.podcast_feed_aliases().expect("alias payload");
|
||||
merge_aliases(conn, podcast_id, urls)
|
||||
}
|
||||
SyncOpPayload::PodcastProgressSet { episode, .. } => {
|
||||
let record = op.podcast_progress().expect("progress payload");
|
||||
if let Err(error) = generation_to_sql(record.generation) {
|
||||
return Ok(ApplyOutcome::Rejected(error.to_string()));
|
||||
}
|
||||
if let Some(episode) = episode {
|
||||
ensure_episode_from_ref(conn, episode)?;
|
||||
}
|
||||
apply_progress_record(conn, &record)
|
||||
}
|
||||
SyncOpPayload::PodcastListenRecorded { session, episode } => {
|
||||
if let Some(current) = load_session(conn, &session.session_id)?
|
||||
&& let Err(error) = merge_listen_session(¤t, session)
|
||||
{
|
||||
return Ok(ApplyOutcome::Rejected(error.to_string()));
|
||||
}
|
||||
if let Some(episode) = episode {
|
||||
ensure_episode_from_ref(conn, episode)?;
|
||||
}
|
||||
apply_listen_session(conn, session)
|
||||
}
|
||||
_ => return Ok(ApplyOutcome::Ignored),
|
||||
};
|
||||
match result {
|
||||
Ok(true) => Ok(ApplyOutcome::Changed),
|
||||
Ok(false) => Ok(ApplyOutcome::Unchanged),
|
||||
Err(error) => match error.downcast_ref::<PodcastStoreError>() {
|
||||
Some(PodcastStoreError::Invalid(reason)) => Ok(ApplyOutcome::Rejected(reason.clone())),
|
||||
_ => Err(error),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn record_outcome(stats: &mut SnapshotApplyStats, result: Result<bool>) -> Result<()> {
|
||||
match result {
|
||||
Ok(true) => stats.changed += 1,
|
||||
Ok(false) => stats.unchanged += 1,
|
||||
Err(error) => match error.downcast_ref::<PodcastStoreError>() {
|
||||
Some(PodcastStoreError::Invalid(_)) => stats.rejected += 1,
|
||||
_ => return Err(error),
|
||||
},
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check(result: Result<(), PodcastError>) -> Result<()> {
|
||||
result.map_err(invalid)
|
||||
}
|
||||
|
||||
impl PodcastStore {
|
||||
/// Applies a device-sync operation received from a trusted device.
|
||||
///
|
||||
/// Non-podcast operations return [`ApplyOutcome::Ignored`]. Podcast
|
||||
/// operations are validated (`SyncOpWire::validate_podcast`) and merged
|
||||
/// with the contract functions in one transaction; applying the same
|
||||
/// operations in any order, with duplicates, gives the same state.
|
||||
pub fn apply_remote_op(&self, op: &SyncOpWire) -> Result<ApplyOutcome> {
|
||||
if !op.payload.is_podcast() {
|
||||
return Ok(ApplyOutcome::Ignored);
|
||||
}
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let outcome = apply_op(&tx, op)?;
|
||||
tx.commit()?;
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Merges the podcast sections of a sync snapshot.
|
||||
///
|
||||
/// Sections exceeding the contract counts are ignored
|
||||
/// (`SyncSnapshot::podcast_state_within_limits`); every record is
|
||||
/// validated and merged with the same functions as operations, so a
|
||||
/// snapshot never replaces newer local state. One transaction.
|
||||
pub fn apply_snapshot(&self, snapshot: &SyncSnapshot) -> Result<SnapshotApplyStats> {
|
||||
let mut stats = SnapshotApplyStats::default();
|
||||
if !snapshot.podcast_state_within_limits() {
|
||||
stats.ignored_over_limits = true;
|
||||
return Ok(stats);
|
||||
}
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let now = now_ms();
|
||||
for record in &snapshot.podcast_subscriptions {
|
||||
let mut record = record.clone();
|
||||
record.normalize();
|
||||
let result = check(record.validate())
|
||||
.and_then(|()| apply_subscription_record(&tx, &record, now));
|
||||
record_outcome(&mut stats, result)?;
|
||||
}
|
||||
for record in &snapshot.podcast_shares {
|
||||
let result = check(record.validate()).and_then(|()| apply_share_record(&tx, record));
|
||||
record_outcome(&mut stats, result)?;
|
||||
}
|
||||
for record in &snapshot.podcast_progress {
|
||||
let result = check(record.validate()).and_then(|()| apply_progress_record(&tx, record));
|
||||
record_outcome(&mut stats, result)?;
|
||||
}
|
||||
for session in &snapshot.podcast_listen_sessions {
|
||||
let result =
|
||||
check(session.validate()).and_then(|()| apply_listen_session(&tx, session));
|
||||
record_outcome(&mut stats, result)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// Fills the podcast sections of `snapshot` with this store's registers
|
||||
/// and sessions, bounded by `SyncSnapshot::bound_podcast_state_to`.
|
||||
///
|
||||
/// `budget_bytes` is usually `MAX_PODCAST_SNAPSHOT_BYTES` or
|
||||
/// `device_sync::podcast_snapshot_budget(other_bytes)`. Records are read
|
||||
/// newest first and capped by the per-section counts before the byte
|
||||
/// budget is applied. Local fetch state and the catalog are never part
|
||||
/// of the snapshot. Send it only to v4 peers (`SyncSnapshot::for_peer`).
|
||||
pub fn fill_snapshot(&self, snapshot: &mut SyncSnapshot, budget_bytes: usize) -> Result<()> {
|
||||
use music_dht::podcast::{
|
||||
MAX_SNAPSHOT_LISTEN_SESSIONS, MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES,
|
||||
MAX_SNAPSHOT_SUBSCRIPTIONS,
|
||||
};
|
||||
let conn = self.lock();
|
||||
let ids: Vec<PodcastId> = {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id FROM podcast_subscriptions
|
||||
ORDER BY generation DESC, hlc_ms DESC, op_id DESC, podcast_id DESC
|
||||
LIMIT ?1",
|
||||
)?;
|
||||
statement
|
||||
.query_map([MAX_SNAPSHOT_SUBSCRIPTIONS as i64], |row| {
|
||||
super::podcast_id_col(row.get(0)?)
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let mut subscriptions = Vec::with_capacity(ids.len());
|
||||
for id in &ids {
|
||||
if let Some(record) = load_subscription_register(&conn, id)? {
|
||||
subscriptions.push(record);
|
||||
}
|
||||
}
|
||||
let shares = {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id, shared, generation, hlc_ms, op_id FROM podcast_shares
|
||||
ORDER BY generation DESC, hlc_ms DESC, op_id DESC, podcast_id DESC
|
||||
LIMIT ?1",
|
||||
)?;
|
||||
statement
|
||||
.query_map([MAX_SNAPSHOT_SHARES as i64], |row| {
|
||||
let generation: i64 = row.get(2)?;
|
||||
Ok(ShareRecord {
|
||||
podcast_id: super::podcast_id_col(row.get(0)?)?,
|
||||
shared: row.get(1)?,
|
||||
generation: u64::try_from(generation).unwrap_or(0),
|
||||
hlc_ms: row.get(3)?,
|
||||
op_id: row.get(4)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let progress = {
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT {PROGRESS_COLUMNS} FROM episode_progress p
|
||||
ORDER BY p.generation DESC, p.hlc_ms DESC, p.op_id DESC, p.episode_id DESC
|
||||
LIMIT ?1"
|
||||
))?;
|
||||
statement
|
||||
.query_map([MAX_SNAPSHOT_PROGRESS as i64], |row| {
|
||||
progress_from_row(row, 0)
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let sessions = {
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT {SESSION_COLUMNS} FROM podcast_listen_sessions s
|
||||
ORDER BY s.ended_at_ms DESC, s.session_id DESC
|
||||
LIMIT ?1"
|
||||
))?;
|
||||
statement
|
||||
.query_map([MAX_SNAPSHOT_LISTEN_SESSIONS as i64], |row| {
|
||||
session_from_row(row, 0)
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
drop(conn);
|
||||
snapshot.podcast_subscriptions = subscriptions;
|
||||
snapshot.podcast_shares = shares;
|
||||
snapshot.podcast_progress = progress;
|
||||
snapshot.podcast_listen_sessions = sessions;
|
||||
snapshot.bound_podcast_state_to(budget_bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A snapshot holding only this store's podcast sections, bounded with
|
||||
/// the default budget (`MAX_PODCAST_SNAPSHOT_BYTES`).
|
||||
pub fn export_snapshot(&self) -> Result<SyncSnapshot> {
|
||||
let mut snapshot = SyncSnapshot::default();
|
||||
self.fill_snapshot(
|
||||
&mut snapshot,
|
||||
music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES,
|
||||
)?;
|
||||
Ok(snapshot)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
//! SQLite reference store for podcasts (`podcasts.sqlite3`).
|
||||
//!
|
||||
//! [`PodcastStore`] persists the podcast contract of `music_dht::podcast`
|
||||
//! for SQLite clients: the local episode catalog refreshed from RSS, the
|
||||
//! synchronized personal registers (subscription, share, progress), the
|
||||
//! grow-only listening sessions and the local-only fetch state. It owns no
|
||||
//! networking, HTTP, scheduling or audio: clients fetch feeds, parse them
|
||||
//! with `music_dht::podcast::feed`, append sync operations to their own log
|
||||
//! and exchange snapshots. The normative rules live in the workspace
|
||||
//! `PODCASTS.md` (section "SQLite reference store" describes this schema).
|
||||
//!
|
||||
//! # Files and migrations
|
||||
//!
|
||||
//! The store lives in its own file, conventionally [`PODCAST_DB_FILE`] next
|
||||
//! to the library database; the caller passes the path. Connections use
|
||||
//! WAL, foreign keys and a busy timeout. `PRAGMA user_version` drives the
|
||||
//! migrations, each applied in its own `BEGIN IMMEDIATE` transaction that
|
||||
//! re-checks the version, so concurrent openers and interrupted upgrades are
|
||||
//! safe:
|
||||
//!
|
||||
//! | `user_version` | Schema |
|
||||
//! | --- | --- |
|
||||
//! | 0 | empty file |
|
||||
//! | 1 | prototype schema of the original podcast PR (`podcasts`, `podcast_episodes`) |
|
||||
//! | 2 | contract model; the prototype tables are renamed `legacy_*` and imported |
|
||||
//!
|
||||
//! A database written by a newer Furumi ([`PODCAST_SCHEMA_VERSION`] lower
|
||||
//! than its `user_version`) or a file that is not a podcast database is
|
||||
//! refused before anything is written ([`PodcastStoreError`]).
|
||||
//!
|
||||
//! # Personal state and sync
|
||||
//!
|
||||
//! Local actions ([`PodcastStore::subscribe`], [`PodcastStore::seek`], ...)
|
||||
//! draw operation stamps from a [`StampSource`], build the device-sync
|
||||
//! operation, apply it through the same code path as
|
||||
//! [`PodcastStore::apply_remote_op`] (the `SyncOpWire` materializers of
|
||||
//! `music_dht::device_sync` and the merge functions of `music_dht::podcast`)
|
||||
//! and return the [`PodcastLocalOp`]s. The caller appends them to its sync
|
||||
//! log **with exactly these stamps**; applying them again is a no-op.
|
||||
//! Applying operations and snapshot sections is idempotent and independent
|
||||
//! of delivery order.
|
||||
//!
|
||||
//! RSS refreshes ([`PodcastStore::apply_feed`]) only touch the catalog and
|
||||
//! feed aliases: they never write a register, progress or a session.
|
||||
//!
|
||||
//! ```
|
||||
//! use furumi_library::podcasts::{ApplyOutcome, OpStamp, PodcastStore, StampSource};
|
||||
//! use music_dht::podcast::{FeedUrl, PodcastMeta};
|
||||
//!
|
||||
//! /// Stand-in for the client's sync log, which reserves seq and hlc.
|
||||
//! struct Log {
|
||||
//! device: String,
|
||||
//! seq: i64,
|
||||
//! }
|
||||
//!
|
||||
//! impl StampSource for Log {
|
||||
//! fn device_id(&self) -> &str {
|
||||
//! &self.device
|
||||
//! }
|
||||
//! fn next_stamp(&mut self) -> anyhow::Result<OpStamp> {
|
||||
//! self.seq += 1;
|
||||
//! let op_id = format!("{}:{}", self.device, self.seq);
|
||||
//! Ok(OpStamp::new(op_id, self.device.clone(), 1_700_000_000_000 + self.seq))
|
||||
//! }
|
||||
//! }
|
||||
//!
|
||||
//! let store = PodcastStore::open_in_memory()?;
|
||||
//! let mut log = Log { device: "device-a".into(), seq: 0 };
|
||||
//! let meta = PodcastMeta {
|
||||
//! feed_url: FeedUrl::parse("https://example.test/feed.xml")?,
|
||||
//! title: "Show".into(),
|
||||
//! author: None,
|
||||
//! description: None,
|
||||
//! link: None,
|
||||
//! image_url: None,
|
||||
//! };
|
||||
//! let outcome = store.subscribe(&mut log, &meta)?;
|
||||
//! assert_eq!(outcome.ops.len(), 1);
|
||||
//! for op in outcome.ops {
|
||||
//! // Append to the sync log with this stamp; re-applying is a no-op.
|
||||
//! let op = op.into_wire(log.seq);
|
||||
//! assert_eq!(store.apply_remote_op(&op)?, ApplyOutcome::Unchanged);
|
||||
//! }
|
||||
//! # Ok::<(), anyhow::Error>(())
|
||||
//! ```
|
||||
//!
|
||||
//! # Thread safety
|
||||
//!
|
||||
//! All methods take `&self` and serialize on an internal mutex around one
|
||||
//! synchronous `rusqlite` connection, so an `Arc<PodcastStore>` can be shared
|
||||
//! between threads. Calls block on SQLite I/O: run them on a blocking pool
|
||||
//! (for example `tokio::task::spawn_blocking`), never on an async reactor
|
||||
//! thread. Several processes may open the same file; SQLite serializes the
|
||||
//! writers.
|
||||
|
||||
mod actions;
|
||||
mod apply;
|
||||
mod queries;
|
||||
mod refresh;
|
||||
mod schema;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::fmt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Mutex, MutexGuard, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, FeedUrl, ListenSession, PodcastError, PodcastId, PodcastMeta, ProgressRecord,
|
||||
ProgressStatus,
|
||||
};
|
||||
use rusqlite::Connection;
|
||||
|
||||
pub use actions::{CheckpointOutcome, ProgressSession, ProgressWrite, SubscribeOutcome};
|
||||
pub use apply::{ApplyOutcome, SnapshotApplyStats};
|
||||
pub use queries::{
|
||||
EpisodeListItem, EpisodeQuery, EpisodeSort, LegacyQuarantineEntry, PlayedFilter,
|
||||
PodcastListFilter, PodcastListenEntry, PodcastSummary, SeasonFilter, SeasonSummary,
|
||||
};
|
||||
pub use refresh::{FeedRefresh, FetchState, RefreshOutcome};
|
||||
pub use schema::{LEGACY_PROTOTYPE_SCHEMA_VERSION, PODCAST_SCHEMA_VERSION};
|
||||
|
||||
/// Conventional file name of the podcast store, next to `library.db`.
|
||||
pub const PODCAST_DB_FILE: &str = "podcasts.sqlite3";
|
||||
|
||||
/// `op_id` prefix of registers imported from the prototype database.
|
||||
///
|
||||
/// Imported subscriptions use generation 0, `hlc_ms` = the prototype
|
||||
/// `added_at` and `op_id` = this prefix followed by the podcast id, so every
|
||||
/// write made through the contract orders after them.
|
||||
pub const LEGACY_IMPORT_OP_PREFIX: &str = "legacy-import:";
|
||||
|
||||
/// Typed errors of [`PodcastStore`]; other failures are SQLite or I/O errors.
|
||||
///
|
||||
/// They are returned inside `anyhow::Error`; use `downcast_ref` to match.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum PodcastStoreError {
|
||||
/// The database was written by a newer Furumi. It was not modified.
|
||||
NewerSchema {
|
||||
/// `user_version` found in the file.
|
||||
found: i64,
|
||||
/// Newest version this build understands.
|
||||
supported: i64,
|
||||
},
|
||||
/// The file is not a podcast database (or has an unexpected shape). It
|
||||
/// was not modified.
|
||||
UnrecognizedDatabase(String),
|
||||
/// The podcast is unknown to this store.
|
||||
UnknownPodcast(PodcastId),
|
||||
/// The action needs an active subscription.
|
||||
NotSubscribed(PodcastId),
|
||||
/// Input violates the podcast contract.
|
||||
Invalid(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for PodcastStoreError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::NewerSchema { found, supported } => write!(
|
||||
f,
|
||||
"podcast database schema v{found} is newer than this Furumi supports (v{supported}); \
|
||||
update Furumi, the database was left unchanged"
|
||||
),
|
||||
Self::UnrecognizedDatabase(reason) => {
|
||||
write!(f, "not a Furumi podcast database: {reason}")
|
||||
}
|
||||
Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"),
|
||||
Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"),
|
||||
Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for PodcastStoreError {}
|
||||
|
||||
impl From<PodcastError> for PodcastStoreError {
|
||||
fn from(error: PodcastError) -> Self {
|
||||
Self::Invalid(error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Stamp of a device-sync operation: the fields the `SyncOpWire`
|
||||
/// materializers turn into register stamps.
|
||||
///
|
||||
/// `seq` is not part of it: the client's sync log assigns it when it appends
|
||||
/// the operation ([`PodcastLocalOp::into_wire`]).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct OpStamp {
|
||||
/// Unique operation id, usually `<origin_device_id>:<seq>`.
|
||||
pub op_id: String,
|
||||
/// Origin device id (the progress register `device_id`).
|
||||
pub origin_device_id: String,
|
||||
/// Operation timestamp (register `hlc_ms`).
|
||||
pub hlc_ms: i64,
|
||||
}
|
||||
|
||||
impl OpStamp {
|
||||
/// Builds a stamp.
|
||||
pub fn new(op_id: impl Into<String>, origin_device_id: impl Into<String>, hlc_ms: i64) -> Self {
|
||||
Self {
|
||||
op_id: op_id.into(),
|
||||
origin_device_id: origin_device_id.into(),
|
||||
hlc_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// The stamp of an existing operation.
|
||||
pub fn of(op: &SyncOpWire) -> Self {
|
||||
Self::new(op.op_id.clone(), op.origin_device_id.clone(), op.hlc_ms)
|
||||
}
|
||||
|
||||
pub(crate) fn wire(&self, payload: SyncOpPayload) -> SyncOpWire {
|
||||
SyncOpWire {
|
||||
op_id: self.op_id.clone(),
|
||||
origin_device_id: self.origin_device_id.clone(),
|
||||
seq: 0,
|
||||
hlc_ms: self.hlc_ms,
|
||||
payload,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Source of operation stamps for local actions, usually the client's sync
|
||||
/// log (which reserves the next sequence number and timestamp).
|
||||
///
|
||||
/// A stamp is drawn only when an action actually emits an operation.
|
||||
pub trait StampSource {
|
||||
/// The local device id; every stamp must use it as origin.
|
||||
fn device_id(&self) -> &str;
|
||||
/// Reserves the stamp of the next local operation.
|
||||
fn next_stamp(&mut self) -> Result<OpStamp>;
|
||||
}
|
||||
|
||||
/// A local operation produced by a [`PodcastStore`] action.
|
||||
///
|
||||
/// The store has already applied it. Append it to the sync log with exactly
|
||||
/// this stamp: a different `op_id` or `hlc_ms` would make other devices
|
||||
/// materialize a different register than this one.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PodcastLocalOp {
|
||||
/// Stamp drawn from the [`StampSource`].
|
||||
pub stamp: OpStamp,
|
||||
/// Operation payload (one of the `podcast_*` kinds).
|
||||
pub payload: SyncOpPayload,
|
||||
}
|
||||
|
||||
impl PodcastLocalOp {
|
||||
/// The operation with the sequence number assigned by the sync log.
|
||||
pub fn into_wire(self, seq: i64) -> SyncOpWire {
|
||||
let mut op = self.stamp.wire(self.payload);
|
||||
op.seq = seq;
|
||||
op
|
||||
}
|
||||
}
|
||||
|
||||
/// SQLite store of podcast catalog, personal state and fetch state.
|
||||
///
|
||||
/// See the [module documentation](self) for the schema, the sync model and
|
||||
/// thread safety.
|
||||
pub struct PodcastStore {
|
||||
conn: Mutex<Connection>,
|
||||
path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for PodcastStore {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("PodcastStore")
|
||||
.field("path", &self.path)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl PodcastStore {
|
||||
/// Opens (creating and migrating if needed) the podcast store at `path`.
|
||||
///
|
||||
/// A prototype database (`user_version` 1) is imported; see the module
|
||||
/// documentation. Fails without modifying the file when it was written
|
||||
/// by a newer Furumi or is not a podcast database.
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
if let Some(parent) = path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
std::fs::create_dir_all(parent)
|
||||
.with_context(|| format!("creating {}", parent.display()))?;
|
||||
}
|
||||
let conn = Connection::open(path)
|
||||
.with_context(|| format!("opening podcast database {}", path.display()))?;
|
||||
Self::init(conn, Some(path.to_path_buf()))
|
||||
}
|
||||
|
||||
/// Opens a private in-memory store (tests and previews).
|
||||
pub fn open_in_memory() -> Result<Self> {
|
||||
Self::init(Connection::open_in_memory()?, None)
|
||||
}
|
||||
|
||||
fn init(mut conn: Connection, path: Option<PathBuf>) -> Result<Self> {
|
||||
conn.busy_timeout(Duration::from_secs(5))?;
|
||||
// Refuse newer or foreign files before anything (WAL included) is written.
|
||||
schema::check_openable(&conn)?;
|
||||
if path.is_some() {
|
||||
let _: String = conn.query_row("PRAGMA journal_mode=WAL", [], |row| row.get(0))?;
|
||||
}
|
||||
conn.pragma_update(None, "foreign_keys", "ON")?;
|
||||
register_norm_function(&conn)?;
|
||||
schema::migrate(&mut conn)?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
path,
|
||||
})
|
||||
}
|
||||
|
||||
/// Path of the database file (`None` for in-memory stores).
|
||||
pub fn path(&self) -> Option<&Path> {
|
||||
self.path.as_deref()
|
||||
}
|
||||
|
||||
/// Current `PRAGMA user_version` (always [`PODCAST_SCHEMA_VERSION`]
|
||||
/// after a successful open).
|
||||
pub fn schema_version(&self) -> Result<i64> {
|
||||
Ok(self
|
||||
.lock()
|
||||
.query_row("PRAGMA user_version", [], |row| row.get(0))?)
|
||||
}
|
||||
|
||||
pub(crate) fn lock(&self) -> MutexGuard<'_, Connection> {
|
||||
self.conn.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
|
||||
fn register_norm_function(conn: &Connection) -> Result<()> {
|
||||
use rusqlite::functions::FunctionFlags;
|
||||
conn.create_scalar_function(
|
||||
"norm",
|
||||
1,
|
||||
FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
|
||||
|ctx| {
|
||||
let value: Option<String> = ctx.get(0)?;
|
||||
Ok(music_dht::normalize_name(
|
||||
value.as_deref().unwrap_or_default(),
|
||||
))
|
||||
},
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn now_ms() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|duration| i64::try_from(duration.as_millis()).unwrap_or(i64::MAX))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Converts a contract generation to the SQLite column value.
|
||||
///
|
||||
/// Generations above `i64::MAX` cannot be stored; records carrying one are
|
||||
/// rejected (no conforming client produces them).
|
||||
pub(crate) fn generation_to_sql(generation: u64) -> Result<i64, PodcastStoreError> {
|
||||
i64::try_from(generation)
|
||||
.map_err(|_| PodcastStoreError::Invalid("generation exceeds i64::MAX".to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn status_to_sql(status: ProgressStatus) -> &'static str {
|
||||
match status {
|
||||
ProgressStatus::NotStarted => "not_started",
|
||||
ProgressStatus::InProgress => "in_progress",
|
||||
ProgressStatus::Completed => "completed",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn status_from_sql(value: &str) -> ProgressStatus {
|
||||
match value {
|
||||
"in_progress" => ProgressStatus::InProgress,
|
||||
"completed" => ProgressStatus::Completed,
|
||||
_ => ProgressStatus::NotStarted,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a stored id; stored ids are validated on write.
|
||||
pub(crate) fn podcast_id_col(value: String) -> rusqlite::Result<PodcastId> {
|
||||
PodcastId::try_from(value).map_err(|error| conversion_error(error.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn episode_id_col(value: String) -> rusqlite::Result<EpisodeId> {
|
||||
EpisodeId::try_from(value).map_err(|error| conversion_error(error.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn feed_url_col(value: String) -> rusqlite::Result<FeedUrl> {
|
||||
FeedUrl::try_from(value).map_err(|error| conversion_error(error.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn podcast_meta_col(value: &str) -> rusqlite::Result<PodcastMeta> {
|
||||
serde_json::from_str(value).map_err(|error| conversion_error(error.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn conversion_error(message: String) -> rusqlite::Error {
|
||||
rusqlite::Error::FromSqlConversionFailure(
|
||||
0,
|
||||
rusqlite::types::Type::Text,
|
||||
Box::new(PodcastStoreError::Invalid(message)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn progress_from_row(
|
||||
row: &rusqlite::Row<'_>,
|
||||
at: usize,
|
||||
) -> rusqlite::Result<ProgressRecord> {
|
||||
let generation: i64 = row.get(at + 4)?;
|
||||
Ok(ProgressRecord {
|
||||
episode_id: episode_id_col(row.get(at)?)?,
|
||||
podcast_id: podcast_id_col(row.get(at + 1)?)?,
|
||||
position_ms: row.get(at + 2)?,
|
||||
status: status_from_sql(&row.get::<_, String>(at + 3)?),
|
||||
generation: u64::try_from(generation).unwrap_or(0),
|
||||
session_id: row.get(at + 5)?,
|
||||
hlc_ms: row.get(at + 6)?,
|
||||
device_id: row.get(at + 7)?,
|
||||
op_id: row.get(at + 8)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Column list matching [`progress_from_row`], for table alias `p`.
|
||||
pub(crate) const PROGRESS_COLUMNS: &str = "p.episode_id, p.podcast_id, p.position_ms, p.status, \
|
||||
p.generation, p.session_id, p.hlc_ms, p.device_id, p.op_id";
|
||||
|
||||
pub(crate) fn session_from_row(
|
||||
row: &rusqlite::Row<'_>,
|
||||
at: usize,
|
||||
) -> rusqlite::Result<ListenSession> {
|
||||
Ok(ListenSession {
|
||||
session_id: row.get(at)?,
|
||||
episode_id: episode_id_col(row.get(at + 1)?)?,
|
||||
podcast_id: podcast_id_col(row.get(at + 2)?)?,
|
||||
device_id: row.get(at + 3)?,
|
||||
started_at_ms: row.get(at + 4)?,
|
||||
ended_at_ms: row.get(at + 5)?,
|
||||
start_position_ms: row.get(at + 6)?,
|
||||
end_position_ms: row.get(at + 7)?,
|
||||
listened_ms: row.get(at + 8)?,
|
||||
completed: row.get(at + 9)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Column list matching [`session_from_row`], for table alias `s`.
|
||||
pub(crate) const SESSION_COLUMNS: &str = "s.session_id, s.episode_id, s.podcast_id, s.device_id, \
|
||||
s.started_at_ms, s.ended_at_ms, s.start_position_ms, s.end_position_ms, s.listened_ms, \
|
||||
s.completed";
|
||||
@@ -0,0 +1,605 @@
|
||||
//! Read-only queries for clients' podcast UI.
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, EpisodeMeta, EpisodeType, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId,
|
||||
PodcastMeta, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, effective_share,
|
||||
};
|
||||
use rusqlite::types::Value;
|
||||
use rusqlite::{Connection, OptionalExtension as _, params, params_from_iter};
|
||||
|
||||
use super::apply;
|
||||
use super::{
|
||||
PROGRESS_COLUMNS, PodcastStore, SESSION_COLUMNS, episode_id_col, feed_url_col, podcast_id_col,
|
||||
progress_from_row, session_from_row,
|
||||
};
|
||||
use crate::TrackItem;
|
||||
|
||||
/// Which podcasts [`PodcastStore::list_podcasts`] returns.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum PodcastListFilter {
|
||||
/// Podcasts with an active subscription.
|
||||
#[default]
|
||||
Subscribed,
|
||||
/// Every podcast in the catalog, including unsubscribed ones and
|
||||
/// podcasts only known from synced progress or queue items.
|
||||
All,
|
||||
}
|
||||
|
||||
/// A podcast with its personal state and episode counters.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PodcastSummary {
|
||||
/// Podcast id.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Local channel metadata; `feed_url` is the URL this device fetches.
|
||||
pub meta: PodcastMeta,
|
||||
/// Whether the subscription register says subscribed.
|
||||
pub subscribed: bool,
|
||||
/// Whether the podcast must currently be published (`effective_share`).
|
||||
pub shared: bool,
|
||||
/// Episodes in the catalog.
|
||||
pub episode_count: u64,
|
||||
/// Episodes not completed (no progress, not started or in progress).
|
||||
pub unplayed_count: u64,
|
||||
/// Episodes in progress.
|
||||
pub in_progress_count: u64,
|
||||
/// Newest publication time.
|
||||
pub latest_published_at_ms: Option<i64>,
|
||||
/// Numbered seasons (episodes without a season are not counted).
|
||||
pub season_count: u64,
|
||||
}
|
||||
|
||||
/// One season group of a podcast. `season == None` groups the episodes
|
||||
/// without `itunes:season`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SeasonSummary {
|
||||
/// Season number; `None` for episodes without a season.
|
||||
pub season: Option<u32>,
|
||||
/// Episodes in the group.
|
||||
pub episode_count: u64,
|
||||
/// Episodes not completed.
|
||||
pub unplayed_count: u64,
|
||||
/// Oldest publication time.
|
||||
pub first_published_at_ms: Option<i64>,
|
||||
/// Newest publication time.
|
||||
pub last_published_at_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// Season filter of an [`EpisodeQuery`].
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum SeasonFilter {
|
||||
/// All episodes.
|
||||
#[default]
|
||||
Any,
|
||||
/// Episodes of one season.
|
||||
Season(u32),
|
||||
/// Episodes without a season.
|
||||
NoSeason,
|
||||
}
|
||||
|
||||
/// Listening-state filter of an [`EpisodeQuery`].
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum PlayedFilter {
|
||||
/// All episodes.
|
||||
#[default]
|
||||
Any,
|
||||
/// Not completed (not started or in progress).
|
||||
Unplayed,
|
||||
/// Never started or marked unplayed.
|
||||
NotStarted,
|
||||
/// In progress.
|
||||
InProgress,
|
||||
/// Completed.
|
||||
Played,
|
||||
}
|
||||
|
||||
/// Episode ordering of an [`EpisodeQuery`].
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum EpisodeSort {
|
||||
/// Newest publication first; undated episodes last.
|
||||
#[default]
|
||||
NewestFirst,
|
||||
/// Oldest publication first; undated episodes last.
|
||||
OldestFirst,
|
||||
/// By season, then episode number, ascending; unnumbered last.
|
||||
EpisodeNumber,
|
||||
/// By season, then episode number, descending; unnumbered last.
|
||||
EpisodeNumberDesc,
|
||||
}
|
||||
|
||||
/// Parameters of [`PodcastStore::episodes`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct EpisodeQuery {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Season filter.
|
||||
pub season: SeasonFilter,
|
||||
/// Listening-state filter.
|
||||
pub played: PlayedFilter,
|
||||
/// Text searched in titles and descriptions (Unicode-normalized,
|
||||
/// case-insensitive substring match). Blank means no filter.
|
||||
pub search: Option<String>,
|
||||
/// Ordering.
|
||||
pub sort: EpisodeSort,
|
||||
/// Maximum rows (`None`: all).
|
||||
pub limit: Option<usize>,
|
||||
/// Rows to skip.
|
||||
pub offset: usize,
|
||||
}
|
||||
|
||||
impl EpisodeQuery {
|
||||
/// All episodes of a podcast, newest first.
|
||||
pub fn new(podcast_id: PodcastId) -> Self {
|
||||
Self {
|
||||
podcast_id,
|
||||
season: SeasonFilter::Any,
|
||||
played: PlayedFilter::Any,
|
||||
search: None,
|
||||
sort: EpisodeSort::NewestFirst,
|
||||
limit: None,
|
||||
offset: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An episode with its personal state.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct EpisodeListItem {
|
||||
/// Episode metadata (feed or synced reference).
|
||||
pub episode: EpisodeMeta,
|
||||
/// Progress register, if any.
|
||||
pub progress: Option<ProgressRecord>,
|
||||
/// Distinct completed listening sessions (`completed_listen_count`).
|
||||
pub completed_count: u64,
|
||||
/// Whether the latest refresh of the feed listed the episode.
|
||||
pub in_feed: bool,
|
||||
}
|
||||
|
||||
impl EpisodeListItem {
|
||||
/// Listening status (`NotStarted` without a register).
|
||||
pub fn status(&self) -> ProgressStatus {
|
||||
self.progress
|
||||
.as_ref()
|
||||
.map_or(ProgressStatus::NotStarted, |progress| progress.status)
|
||||
}
|
||||
|
||||
/// Position ordinary Play resumes from.
|
||||
pub fn resume_position_ms(&self) -> i64 {
|
||||
self.progress
|
||||
.as_ref()
|
||||
.map_or(0, ProgressRecord::resume_position_ms)
|
||||
}
|
||||
}
|
||||
|
||||
/// A listening session in history, with display titles when known.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PodcastListenEntry {
|
||||
/// The session.
|
||||
pub session: ListenSession,
|
||||
/// Episode title from the catalog.
|
||||
pub episode_title: Option<String>,
|
||||
/// Podcast title from the catalog.
|
||||
pub podcast_title: Option<String>,
|
||||
}
|
||||
|
||||
/// A prototype row that could not be imported (kept in its `legacy_*` table).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct LegacyQuarantineEntry {
|
||||
/// `legacy_podcasts` or `legacy_podcast_episodes`.
|
||||
pub legacy_table: String,
|
||||
/// Row id in that table.
|
||||
pub legacy_id: i64,
|
||||
/// Short reason.
|
||||
pub reason: String,
|
||||
/// Details (the offending value or error).
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
const EPISODE_COLUMNS: &str = "e.episode_id, e.podcast_id, e.guid, e.title, e.description, \
|
||||
e.season, e.episode_number, e.episode_type, e.published_at_ms, e.duration_ms, \
|
||||
e.audio_url, e.mime, e.byte_len, e.image_url, e.link";
|
||||
const EPISODE_COLUMN_COUNT: usize = 15;
|
||||
|
||||
fn episode_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<EpisodeMeta> {
|
||||
Ok(EpisodeMeta {
|
||||
episode_id: episode_id_col(row.get(0)?)?,
|
||||
podcast_id: podcast_id_col(row.get(1)?)?,
|
||||
guid: row.get(2)?,
|
||||
title: row.get(3)?,
|
||||
description: row.get(4)?,
|
||||
season: row.get(5)?,
|
||||
episode_number: row.get(6)?,
|
||||
episode_type: EpisodeType::parse(&row.get::<_, String>(7)?),
|
||||
published_at_ms: row.get(8)?,
|
||||
duration_ms: row.get(9)?,
|
||||
audio_url: row.get(10)?,
|
||||
mime: row.get(11)?,
|
||||
byte_len: row
|
||||
.get::<_, Option<i64>>(12)?
|
||||
.and_then(|len| u64::try_from(len).ok()),
|
||||
image_url: row.get(13)?,
|
||||
link: row.get(14)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn optional_progress(
|
||||
row: &rusqlite::Row<'_>,
|
||||
at: usize,
|
||||
) -> rusqlite::Result<Option<ProgressRecord>> {
|
||||
if row.get::<_, Option<String>>(at)?.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
progress_from_row(row, at).map(Some)
|
||||
}
|
||||
|
||||
pub(crate) fn episode_meta_in(
|
||||
conn: &Connection,
|
||||
episode_id: &EpisodeId,
|
||||
) -> Result<Option<EpisodeMeta>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
&format!("SELECT {EPISODE_COLUMNS} FROM episodes e WHERE e.episode_id = ?1"),
|
||||
[episode_id.as_str()],
|
||||
episode_from_row,
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
pub(crate) fn podcast_meta_in(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Option<PodcastMeta>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT feed_url, title, author, description, link, image_url
|
||||
FROM podcasts WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
Ok(PodcastMeta {
|
||||
feed_url: feed_url_col(row.get(0)?)?,
|
||||
title: row.get(1)?,
|
||||
author: row.get(2)?,
|
||||
description: row.get(3)?,
|
||||
link: row.get(4)?,
|
||||
image_url: row.get(5)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
/// Queue reference of a catalog episode, with `progress_generation` set to
|
||||
/// the generation of its progress register (handoff hint).
|
||||
pub(crate) fn episode_ref_in(
|
||||
conn: &Connection,
|
||||
episode_id: &EpisodeId,
|
||||
) -> Result<Option<PodcastEpisodeRef>> {
|
||||
let Some(episode) = episode_meta_in(conn, episode_id)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(podcast) = podcast_meta_in(conn, &episode.podcast_id)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut reference = episode.to_ref(&podcast);
|
||||
reference.progress_generation =
|
||||
apply::load_progress(conn, episode_id)?.map(|progress| progress.generation);
|
||||
Ok(Some(reference))
|
||||
}
|
||||
|
||||
const NOT_COMPLETED: &str = "IFNULL(g.status, 'not_started') <> 'completed'";
|
||||
|
||||
impl PodcastStore {
|
||||
/// Podcasts with subscription state and counters, sorted by title.
|
||||
pub fn list_podcasts(&self, filter: PodcastListFilter) -> Result<Vec<PodcastSummary>> {
|
||||
let conn = self.lock();
|
||||
let join = match filter {
|
||||
PodcastListFilter::Subscribed => {
|
||||
"JOIN podcast_subscriptions r ON r.podcast_id = p.podcast_id AND r.subscribed = 1"
|
||||
}
|
||||
PodcastListFilter::All => "",
|
||||
};
|
||||
let ids: Vec<PodcastId> = {
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT p.podcast_id FROM podcasts p {join}
|
||||
ORDER BY p.title COLLATE NOCASE, p.podcast_id"
|
||||
))?;
|
||||
statement
|
||||
.query_map([], |row| podcast_id_col(row.get(0)?))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let mut out = Vec::with_capacity(ids.len());
|
||||
for id in ids {
|
||||
if let Some(summary) = summary_in(&conn, &id)? {
|
||||
out.push(summary);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// One podcast with subscription state and counters.
|
||||
pub fn podcast(&self, podcast_id: &PodcastId) -> Result<Option<PodcastSummary>> {
|
||||
summary_in(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// The subscription register (with the stored alias set), if any.
|
||||
pub fn subscription(&self, podcast_id: &PodcastId) -> Result<Option<SubscriptionRecord>> {
|
||||
apply::load_subscription_register(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// The share register, if any.
|
||||
pub fn share(&self, podcast_id: &PodcastId) -> Result<Option<ShareRecord>> {
|
||||
apply::load_share(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// The bounded, synced feed alias set of a podcast.
|
||||
pub fn feed_aliases(&self, podcast_id: &PodcastId) -> Result<Vec<FeedUrl>> {
|
||||
apply::load_aliases(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// Season groups of a podcast: numbered seasons ascending, then the
|
||||
/// group of episodes without a season.
|
||||
pub fn seasons(&self, podcast_id: &PodcastId) -> Result<Vec<SeasonSummary>> {
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT s.season, s.episode_count, s.first_published_at_ms, s.last_published_at_ms,
|
||||
(SELECT COUNT(*) FROM episodes e
|
||||
LEFT JOIN episode_progress g ON g.episode_id = e.episode_id
|
||||
WHERE e.podcast_id = s.podcast_id AND e.season IS s.season
|
||||
AND {NOT_COMPLETED})
|
||||
FROM podcast_seasons s WHERE s.podcast_id = ?1
|
||||
ORDER BY s.season IS NULL, s.season"
|
||||
))?;
|
||||
let seasons = statement
|
||||
.query_map([podcast_id.as_str()], |row| {
|
||||
Ok(SeasonSummary {
|
||||
season: row.get(0)?,
|
||||
episode_count: count(row.get(1)?),
|
||||
first_published_at_ms: row.get(2)?,
|
||||
last_published_at_ms: row.get(3)?,
|
||||
unplayed_count: count(row.get(4)?),
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(seasons)
|
||||
}
|
||||
|
||||
/// Episodes of a podcast or season with their progress.
|
||||
pub fn episodes(&self, query: &EpisodeQuery) -> Result<Vec<EpisodeListItem>> {
|
||||
let mut conditions = vec!["e.podcast_id = ?".to_string()];
|
||||
let mut values = vec![Value::Text(query.podcast_id.as_str().to_string())];
|
||||
match query.season {
|
||||
SeasonFilter::Any => {}
|
||||
SeasonFilter::Season(season) => {
|
||||
conditions.push("e.season = ?".to_string());
|
||||
values.push(Value::Integer(i64::from(season)));
|
||||
}
|
||||
SeasonFilter::NoSeason => conditions.push("e.season IS NULL".to_string()),
|
||||
}
|
||||
match query.played {
|
||||
PlayedFilter::Any => {}
|
||||
PlayedFilter::Unplayed => conditions.push(NOT_COMPLETED.to_string()),
|
||||
PlayedFilter::NotStarted => {
|
||||
conditions.push("IFNULL(g.status, 'not_started') = 'not_started'".to_string());
|
||||
}
|
||||
PlayedFilter::InProgress => conditions.push("g.status = 'in_progress'".to_string()),
|
||||
PlayedFilter::Played => conditions.push("g.status = 'completed'".to_string()),
|
||||
}
|
||||
if let Some(search) = query.search.as_deref() {
|
||||
let needle = music_dht::normalize_name(search);
|
||||
if !needle.is_empty() {
|
||||
conditions.push(
|
||||
"(instr(norm(e.title), ?) > 0 OR instr(norm(e.description), ?) > 0)"
|
||||
.to_string(),
|
||||
);
|
||||
values.push(Value::Text(needle.clone()));
|
||||
values.push(Value::Text(needle));
|
||||
}
|
||||
}
|
||||
let order = match query.sort {
|
||||
EpisodeSort::NewestFirst => {
|
||||
"e.published_at_ms IS NULL, e.published_at_ms DESC, e.episode_id"
|
||||
}
|
||||
EpisodeSort::OldestFirst => {
|
||||
"e.published_at_ms IS NULL, e.published_at_ms, e.episode_id"
|
||||
}
|
||||
EpisodeSort::EpisodeNumber => {
|
||||
"e.season IS NULL, e.season, e.episode_number IS NULL, e.episode_number, \
|
||||
e.published_at_ms IS NULL, e.published_at_ms, e.episode_id"
|
||||
}
|
||||
EpisodeSort::EpisodeNumberDesc => {
|
||||
"e.season IS NULL, e.season DESC, e.episode_number IS NULL, \
|
||||
e.episode_number DESC, e.published_at_ms IS NULL, e.published_at_ms DESC, \
|
||||
e.episode_id"
|
||||
}
|
||||
};
|
||||
values.push(Value::Integer(
|
||||
query
|
||||
.limit
|
||||
.map_or(-1, |limit| i64::try_from(limit).unwrap_or(i64::MAX)),
|
||||
));
|
||||
values.push(Value::Integer(
|
||||
i64::try_from(query.offset).unwrap_or(i64::MAX),
|
||||
));
|
||||
let progress_columns = PROGRESS_COLUMNS.replace("p.", "g.");
|
||||
let sql = format!(
|
||||
"SELECT {EPISODE_COLUMNS}, e.in_feed, {progress_columns},
|
||||
(SELECT COUNT(*) FROM podcast_listen_sessions s
|
||||
WHERE s.episode_id = e.episode_id AND s.completed = 1)
|
||||
FROM episodes e
|
||||
LEFT JOIN episode_progress g ON g.episode_id = e.episode_id
|
||||
WHERE {}
|
||||
ORDER BY {order}
|
||||
LIMIT ? OFFSET ?",
|
||||
conditions.join(" AND ")
|
||||
);
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(&sql)?;
|
||||
let items = statement
|
||||
.query_map(params_from_iter(values), list_item_from_row)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// One episode with its progress.
|
||||
pub fn episode(&self, episode_id: &EpisodeId) -> Result<Option<EpisodeListItem>> {
|
||||
let progress_columns = PROGRESS_COLUMNS.replace("p.", "g.");
|
||||
Ok(self
|
||||
.lock()
|
||||
.query_row(
|
||||
&format!(
|
||||
"SELECT {EPISODE_COLUMNS}, e.in_feed, {progress_columns},
|
||||
(SELECT COUNT(*) FROM podcast_listen_sessions s
|
||||
WHERE s.episode_id = e.episode_id AND s.completed = 1)
|
||||
FROM episodes e
|
||||
LEFT JOIN episode_progress g ON g.episode_id = e.episode_id
|
||||
WHERE e.episode_id = ?1"
|
||||
),
|
||||
[episode_id.as_str()],
|
||||
list_item_from_row,
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
/// Progress register of an episode (also for episodes the catalog does
|
||||
/// not know yet).
|
||||
pub fn progress(&self, episode_id: &EpisodeId) -> Result<Option<ProgressRecord>> {
|
||||
apply::load_progress(&self.lock(), episode_id)
|
||||
}
|
||||
|
||||
/// Position ordinary Play resumes from (`ProgressRecord::resume_position_ms`, 0 without progress).
|
||||
pub fn resume_position_ms(&self, episode_id: &EpisodeId) -> Result<i64> {
|
||||
Ok(self
|
||||
.progress(episode_id)?
|
||||
.map_or(0, |progress| progress.resume_position_ms()))
|
||||
}
|
||||
|
||||
/// A queue item for an episode (see [`TrackItem::podcast_episode`]).
|
||||
pub fn episode_track(&self, episode_id: &EpisodeId) -> Result<Option<TrackItem>> {
|
||||
Ok(self
|
||||
.episode_ref(episode_id)?
|
||||
.map(TrackItem::podcast_episode))
|
||||
}
|
||||
|
||||
/// Listening history (sessions that `should_record`), newest first.
|
||||
pub fn listen_history(
|
||||
&self,
|
||||
podcast_id: Option<&PodcastId>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<PodcastListenEntry>> {
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT {SESSION_COLUMNS}, e.title, p.title
|
||||
FROM podcast_listen_sessions s
|
||||
LEFT JOIN episodes e ON e.episode_id = s.episode_id
|
||||
LEFT JOIN podcasts p ON p.podcast_id = s.podcast_id
|
||||
WHERE (s.completed = 1 OR s.listened_ms >= ?1)
|
||||
AND (?2 IS NULL OR s.podcast_id = ?2)
|
||||
ORDER BY s.ended_at_ms DESC, s.session_id DESC
|
||||
LIMIT ?3"
|
||||
))?;
|
||||
let entries = statement
|
||||
.query_map(
|
||||
params![
|
||||
music_dht::podcast::MIN_RECORDED_EPISODE_LISTEN_MS,
|
||||
podcast_id.map(PodcastId::as_str),
|
||||
i64::try_from(limit).unwrap_or(i64::MAX)
|
||||
],
|
||||
|row| {
|
||||
Ok(PodcastListenEntry {
|
||||
session: session_from_row(row, 0)?,
|
||||
episode_title: row.get(10)?,
|
||||
podcast_title: row.get(11)?,
|
||||
})
|
||||
},
|
||||
)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Number of distinct completed sessions of an episode.
|
||||
pub fn completed_listen_count(&self, episode_id: &EpisodeId) -> Result<u64> {
|
||||
let total: i64 = self.lock().query_row(
|
||||
"SELECT COUNT(*) FROM podcast_listen_sessions WHERE episode_id = ?1 AND completed = 1",
|
||||
[episode_id.as_str()],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
Ok(count(total))
|
||||
}
|
||||
|
||||
/// Prototype rows that could not be imported.
|
||||
pub fn legacy_quarantine(&self) -> Result<Vec<LegacyQuarantineEntry>> {
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT legacy_table, legacy_id, reason, detail FROM legacy_quarantine
|
||||
ORDER BY legacy_table, legacy_id",
|
||||
)?;
|
||||
let entries = statement
|
||||
.query_map([], |row| {
|
||||
Ok(LegacyQuarantineEntry {
|
||||
legacy_table: row.get(0)?,
|
||||
legacy_id: row.get(1)?,
|
||||
reason: row.get(2)?,
|
||||
detail: row.get(3)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(entries)
|
||||
}
|
||||
}
|
||||
|
||||
fn count(value: i64) -> u64 {
|
||||
u64::try_from(value).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn list_item_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<EpisodeListItem> {
|
||||
Ok(EpisodeListItem {
|
||||
episode: episode_from_row(row)?,
|
||||
in_feed: row.get(EPISODE_COLUMN_COUNT)?,
|
||||
progress: optional_progress(row, EPISODE_COLUMN_COUNT + 1)?,
|
||||
completed_count: count(row.get(EPISODE_COLUMN_COUNT + 10)?),
|
||||
})
|
||||
}
|
||||
|
||||
fn summary_in(conn: &Connection, podcast_id: &PodcastId) -> Result<Option<PodcastSummary>> {
|
||||
let Some(meta) = podcast_meta_in(conn, podcast_id)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (episode_count, unplayed_count, in_progress_count, latest, season_count) = conn.query_row(
|
||||
&format!(
|
||||
"SELECT COUNT(*),
|
||||
COUNT(*) FILTER (WHERE {NOT_COMPLETED}),
|
||||
COUNT(*) FILTER (WHERE g.status = 'in_progress'),
|
||||
MAX(e.published_at_ms),
|
||||
(SELECT COUNT(*) FROM podcast_seasons s
|
||||
WHERE s.podcast_id = ?1 AND s.season IS NOT NULL)
|
||||
FROM episodes e
|
||||
LEFT JOIN episode_progress g ON g.episode_id = e.episode_id
|
||||
WHERE e.podcast_id = ?1"
|
||||
),
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get::<_, i64>(0)?,
|
||||
row.get::<_, i64>(1)?,
|
||||
row.get::<_, i64>(2)?,
|
||||
row.get::<_, Option<i64>>(3)?,
|
||||
row.get::<_, i64>(4)?,
|
||||
))
|
||||
},
|
||||
)?;
|
||||
let subscription = apply::load_subscription_register(conn, podcast_id)?;
|
||||
let share = apply::load_share(conn, podcast_id)?;
|
||||
Ok(Some(PodcastSummary {
|
||||
podcast_id: podcast_id.clone(),
|
||||
meta,
|
||||
subscribed: subscription
|
||||
.as_ref()
|
||||
.is_some_and(|record| record.subscribed),
|
||||
shared: effective_share(subscription.as_ref(), share.as_ref()),
|
||||
episode_count: count(episode_count),
|
||||
unplayed_count: count(unplayed_count),
|
||||
in_progress_count: count(in_progress_count),
|
||||
latest_published_at_ms: latest,
|
||||
season_count: count(season_count),
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,618 @@
|
||||
//! Feed refresh ingestion and the local-only fetch state.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::device_sync::SyncOpPayload;
|
||||
use music_dht::podcast::feed::ParsedFeed;
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, EpisodeIdentityHints, EpisodeMeta, FeedUrl, KnownEpisode, PodcastId,
|
||||
merge_feed_aliases, resolve_episode_ids,
|
||||
};
|
||||
use rusqlite::{Connection, OptionalExtension as _, TransactionBehavior, params};
|
||||
|
||||
use super::apply::{self, EpisodeSource};
|
||||
use super::{PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource, now_ms};
|
||||
|
||||
/// A fetched and parsed feed document for [`PodcastStore::apply_feed`].
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct FeedRefresh<'a> {
|
||||
/// Podcast the feed belongs to.
|
||||
pub podcast_id: &'a PodcastId,
|
||||
/// The parsed document (`music_dht::podcast::feed::parse_feed_with_base`
|
||||
/// with `fetched_from` as base).
|
||||
pub feed: &'a ParsedFeed,
|
||||
/// URL the document was finally served from, after redirects.
|
||||
pub fetched_from: &'a FeedUrl,
|
||||
/// Whether every redirect leading to `fetched_from` was permanent
|
||||
/// (301/308). Temporary redirects do not move the feed.
|
||||
pub moved_permanently: bool,
|
||||
}
|
||||
|
||||
/// Result of [`PodcastStore::apply_feed`].
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct RefreshOutcome {
|
||||
/// Episodes seen for the first time.
|
||||
pub new_episodes: usize,
|
||||
/// Known episodes whose metadata changed (title, audio URL, ...).
|
||||
pub updated_episodes: usize,
|
||||
/// Known episodes without metadata changes.
|
||||
pub unchanged_episodes: usize,
|
||||
/// Items without identity or with invalid metadata.
|
||||
pub skipped_items: usize,
|
||||
/// Feed URLs learned from a permanent redirect or `itunes:new-feed-url`.
|
||||
pub learned_aliases: Vec<FeedUrl>,
|
||||
/// URL the podcast is fetched from after this refresh.
|
||||
pub feed_url: Option<FeedUrl>,
|
||||
/// `podcast_feed_aliases_added`, only when the learned URLs changed the
|
||||
/// stored alias set. Append it to the sync log.
|
||||
pub ops: Vec<PodcastLocalOp>,
|
||||
}
|
||||
|
||||
/// Local-only fetch state of a podcast. Never part of sync output.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct FetchState {
|
||||
/// HTTP `ETag` validator of the last successful fetch.
|
||||
pub etag: Option<String>,
|
||||
/// HTTP `Last-Modified` validator of the last successful fetch.
|
||||
pub last_modified: Option<String>,
|
||||
/// Last fetch attempt, unix milliseconds.
|
||||
pub checked_at_ms: Option<i64>,
|
||||
/// Last successful fetch (including "not modified").
|
||||
pub last_success_at_ms: Option<i64>,
|
||||
/// When the scheduler should check again (`None`: due now).
|
||||
pub next_check_at_ms: Option<i64>,
|
||||
/// Error of the last attempt; `None` after a success.
|
||||
pub last_error: Option<String>,
|
||||
/// Failed attempts since the last success.
|
||||
pub consecutive_failures: u32,
|
||||
/// Image URL the cached cover was downloaded from.
|
||||
pub cover_url: Option<String>,
|
||||
/// Local path of the cached cover.
|
||||
pub cover_path: Option<String>,
|
||||
}
|
||||
|
||||
fn podcast_fetch_url(conn: &Connection, podcast_id: &PodcastId) -> Result<Option<FeedUrl>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT feed_url FROM podcasts WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| super::feed_url_col(row.get(0)?),
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
fn fetch_state_in(conn: &Connection, podcast_id: &PodcastId) -> Result<FetchState> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT etag, last_modified, checked_at_ms, last_success_at_ms,
|
||||
next_check_at_ms, last_error, consecutive_failures, cover_url,
|
||||
cover_path
|
||||
FROM podcast_fetch_state WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
Ok(FetchState {
|
||||
etag: row.get(0)?,
|
||||
last_modified: row.get(1)?,
|
||||
checked_at_ms: row.get(2)?,
|
||||
last_success_at_ms: row.get(3)?,
|
||||
next_check_at_ms: row.get(4)?,
|
||||
last_error: row.get(5)?,
|
||||
consecutive_failures: u32::try_from(row.get::<_, i64>(6)?).unwrap_or(u32::MAX),
|
||||
cover_url: row.get(7)?,
|
||||
cover_path: row.get(8)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?
|
||||
.unwrap_or_default())
|
||||
}
|
||||
|
||||
fn has_personal_state(conn: &Connection, episode_id: &EpisodeId) -> Result<bool> {
|
||||
Ok(conn.query_row(
|
||||
"SELECT EXISTS (SELECT 1 FROM episode_progress WHERE episode_id = ?1)
|
||||
OR EXISTS (SELECT 1 FROM podcast_listen_sessions WHERE episode_id = ?1)",
|
||||
[episode_id.as_str()],
|
||||
|row| row.get(0),
|
||||
)?)
|
||||
}
|
||||
|
||||
/// Re-keys episodes imported from the prototype whose stored `guid` was the
|
||||
/// enclosure URL (`legacy_guid`) when the feed shows it is a real GUID.
|
||||
///
|
||||
/// Only episodes without progress or sessions move, so no personal state is
|
||||
/// touched; otherwise the GUID item becomes a separate episode. When another
|
||||
/// episode already holds the GUID id (for example one a remote operation
|
||||
/// created), the imported episode is a duplicate of it and is deleted.
|
||||
fn rekey_legacy_guids(conn: &Connection, podcast_id: &PodcastId, feed: &ParsedFeed) -> Result<()> {
|
||||
for item in &feed.episodes {
|
||||
let Some(guid) = item.guid.as_deref() else {
|
||||
continue;
|
||||
};
|
||||
let Some(guid_id) = EpisodeId::from_guid(podcast_id, guid) else {
|
||||
continue;
|
||||
};
|
||||
let taken = conn
|
||||
.query_row(
|
||||
"SELECT 1 FROM episodes WHERE episode_id = ?1",
|
||||
[guid_id.as_str()],
|
||||
|_| Ok(()),
|
||||
)
|
||||
.optional()?
|
||||
.is_some();
|
||||
let legacy: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT episode_id FROM episodes
|
||||
WHERE podcast_id = ?1 AND legacy_guid = ?2 ORDER BY episode_id LIMIT 1",
|
||||
params![podcast_id.as_str(), guid],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
let Some(legacy) = legacy else {
|
||||
continue;
|
||||
};
|
||||
let legacy = super::episode_id_col(legacy)?;
|
||||
if legacy == guid_id || has_personal_state(conn, &legacy)? {
|
||||
continue;
|
||||
}
|
||||
if taken {
|
||||
conn.execute(
|
||||
"DELETE FROM episodes WHERE episode_id = ?1",
|
||||
[legacy.as_str()],
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
conn.execute(
|
||||
"UPDATE episodes SET episode_id = ?2, guid = ?3, legacy_guid = NULL
|
||||
WHERE episode_id = ?1",
|
||||
params![legacy.as_str(), guid_id.as_str(), guid],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Episodes imported from the prototype without a GUID (`legacy_guid` set)
|
||||
/// that carry no progress and no session.
|
||||
///
|
||||
/// Their ids were resolved over every stored prototype row at once, stale
|
||||
/// rows included, so they may differ from what a fresh device derives from
|
||||
/// the current feed. They are provisional until a refresh confirms them.
|
||||
fn provisional_episode_ids(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<BTreeSet<EpisodeId>> {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT e.episode_id FROM episodes e
|
||||
WHERE e.podcast_id = ?1 AND e.legacy_guid IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM episode_progress p WHERE p.episode_id = e.episode_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM podcast_listen_sessions s
|
||||
WHERE s.episode_id = e.episode_id)",
|
||||
)?;
|
||||
let ids = statement
|
||||
.query_map([podcast_id.as_str()], |row| {
|
||||
super::episode_id_col(row.get(0)?)
|
||||
})?
|
||||
.collect::<rusqlite::Result<BTreeSet<_>>>()?;
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
/// Resolves the ids of the feed items as a device without prototype history
|
||||
/// would, and moves provisional imported episodes onto them.
|
||||
///
|
||||
/// * The ids come from the contract identity rule against every stored
|
||||
/// episode except the provisional ones (episodes from earlier refreshes,
|
||||
/// from sync, and imported ones that already carry personal state).
|
||||
/// * Each provisional episode is matched to the item it was imported from
|
||||
/// with the same rule (enclosure, then link, then date and title; GUIDs
|
||||
/// ignored, since the prototype may have stored the enclosure URL as GUID)
|
||||
/// and re-keyed to that item's id. When another stored episode that stays
|
||||
/// already holds that id, it is the item's episode and the provisional one
|
||||
/// is a duplicate: it is deleted. Unmatched provisional episodes stay as
|
||||
/// they are (not in the feed); the next refresh that lists them re-keys
|
||||
/// them.
|
||||
///
|
||||
/// Provisional episodes have no progress and no session, so no personal
|
||||
/// state moves. The refresh then clears `legacy_guid` on every episode it
|
||||
/// lists, which makes their ids final.
|
||||
fn resolve_and_rekey_provisional(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
feed: &ParsedFeed,
|
||||
) -> Result<Vec<Option<EpisodeId>>> {
|
||||
let known = apply::known_episodes(conn, podcast_id)?;
|
||||
let provisional = provisional_episode_ids(conn, podcast_id)?;
|
||||
if provisional.is_empty() {
|
||||
return Ok(feed.resolve_episode_ids(podcast_id, &known));
|
||||
}
|
||||
let (provisional_known, stable_known): (Vec<KnownEpisode>, Vec<KnownEpisode>) = known
|
||||
.into_iter()
|
||||
.partition(|episode| provisional.contains(&episode.episode_id));
|
||||
let ids = feed.resolve_episode_ids(podcast_id, &stable_known);
|
||||
|
||||
let guidless: Vec<EpisodeIdentityHints> = feed
|
||||
.identity_hints()
|
||||
.into_iter()
|
||||
.map(|hints| EpisodeIdentityHints {
|
||||
guid: None,
|
||||
..hints
|
||||
})
|
||||
.collect();
|
||||
let matches = resolve_episode_ids(
|
||||
podcast_id,
|
||||
feed.link.as_deref(),
|
||||
&guidless,
|
||||
&provisional_known,
|
||||
);
|
||||
let mut moves: Vec<(EpisodeId, EpisodeId)> = matches
|
||||
.into_iter()
|
||||
.zip(&ids)
|
||||
.filter_map(|(matched, target)| {
|
||||
let matched = matched.filter(|id| provisional.contains(id))?;
|
||||
let target = target.clone()?;
|
||||
(matched != target).then_some((matched, target))
|
||||
})
|
||||
.collect();
|
||||
// A target held by an episode that stays (for example a row a remote
|
||||
// operation created before this first refresh) is the item's episode
|
||||
// already: the provisional episode is a duplicate of it. It has no
|
||||
// progress and no session, and its metadata would be overwritten by the
|
||||
// item anyway, so it is removed instead of kept as a second copy. A
|
||||
// removed duplicate frees its own id for other moves.
|
||||
let mut occupied: BTreeSet<EpisodeId> = stable_known
|
||||
.iter()
|
||||
.chain(&provisional_known)
|
||||
.map(|episode| episode.episode_id.clone())
|
||||
.collect();
|
||||
let mut duplicates: Vec<EpisodeId> = Vec::new();
|
||||
loop {
|
||||
let leaving: BTreeSet<&EpisodeId> = moves.iter().map(|(from, _)| from).collect();
|
||||
let blocked: BTreeSet<EpisodeId> = moves
|
||||
.iter()
|
||||
.filter(|(_, to)| occupied.contains(to) && !leaving.contains(to))
|
||||
.map(|(from, _)| from.clone())
|
||||
.collect();
|
||||
if blocked.is_empty() {
|
||||
break;
|
||||
}
|
||||
moves.retain(|(from, _)| !blocked.contains(from));
|
||||
for from in blocked {
|
||||
occupied.remove(&from);
|
||||
duplicates.push(from);
|
||||
}
|
||||
}
|
||||
for duplicate in &duplicates {
|
||||
// Guarded again: only a provisional episode without personal state.
|
||||
conn.execute(
|
||||
"DELETE FROM episodes
|
||||
WHERE episode_id = ?1 AND legacy_guid IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM episode_progress p
|
||||
WHERE p.episode_id = episodes.episode_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM podcast_listen_sessions s
|
||||
WHERE s.episode_id = episodes.episode_id)",
|
||||
[duplicate.as_str()],
|
||||
)?;
|
||||
}
|
||||
// Two phases, so a move may take an id another move vacates.
|
||||
for (index, (from, _)) in moves.iter().enumerate() {
|
||||
conn.execute(
|
||||
"UPDATE episodes SET episode_id = ?2 WHERE episode_id = ?1",
|
||||
params![from.as_str(), format!("rekey-pending:{index}")],
|
||||
)?;
|
||||
}
|
||||
for (index, (_, to)) in moves.iter().enumerate() {
|
||||
conn.execute(
|
||||
"UPDATE episodes SET episode_id = ?2 WHERE episode_id = ?1",
|
||||
params![format!("rekey-pending:{index}"), to.as_str()],
|
||||
)?;
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
fn upsert_feed_episode(conn: &Connection, meta: &EpisodeMeta, now: i64) -> Result<Option<bool>> {
|
||||
let Some(stored) = super::queries::episode_meta_in(conn, &meta.episode_id)? else {
|
||||
apply::insert_episode(conn, meta, EpisodeSource::Feed, true, None, now, now)?;
|
||||
return Ok(None);
|
||||
};
|
||||
let changed = stored != *meta;
|
||||
conn.execute(
|
||||
"UPDATE episodes
|
||||
SET podcast_id = ?2, guid = ?3, title = ?4, description = ?5, season = ?6,
|
||||
episode_number = ?7, episode_type = ?8, published_at_ms = ?9,
|
||||
duration_ms = ?10, audio_url = ?11, mime = ?12, byte_len = ?13,
|
||||
image_url = ?14, link = ?15, source = 'feed', in_feed = 1,
|
||||
legacy_guid = NULL,
|
||||
updated_at_ms = CASE WHEN ?16 THEN ?17 ELSE updated_at_ms END
|
||||
WHERE episode_id = ?1",
|
||||
params![
|
||||
meta.episode_id.as_str(),
|
||||
meta.podcast_id.as_str(),
|
||||
meta.guid,
|
||||
meta.title,
|
||||
meta.description,
|
||||
meta.season,
|
||||
meta.episode_number,
|
||||
apply::episode_type_to_sql(meta.episode_type),
|
||||
meta.published_at_ms,
|
||||
meta.duration_ms,
|
||||
meta.audio_url,
|
||||
meta.mime,
|
||||
meta.byte_len.and_then(|len| i64::try_from(len).ok()),
|
||||
meta.image_url,
|
||||
meta.link,
|
||||
changed,
|
||||
now,
|
||||
],
|
||||
)?;
|
||||
Ok(Some(changed))
|
||||
}
|
||||
|
||||
impl PodcastStore {
|
||||
/// Ingests a fetched feed document for a podcast.
|
||||
///
|
||||
/// * Channel metadata of the catalog row is replaced by the feed's.
|
||||
/// * Episode ids are resolved with the contract identity rule
|
||||
/// (`ParsedFeed::resolve_episode_ids`) against every stored episode of
|
||||
/// the podcast, including episodes learned through sync, so a changed
|
||||
/// title, audio URL or a GUID-less item keeps its id. Known episodes
|
||||
/// get the new metadata (including the audio URL); new ones are added;
|
||||
/// episodes missing from the document are kept (`in_feed = false`).
|
||||
/// * A permanent redirect target and `itunes:new-feed-url` become feed
|
||||
/// aliases and the new fetch URL. When they change the stored alias
|
||||
/// set, one `podcast_feed_aliases_added` operation is emitted.
|
||||
/// * Subscription, share and progress registers and listening sessions
|
||||
/// are never touched, so a refresh cannot resurrect an unsubscribed
|
||||
/// podcast or move anyone's position.
|
||||
///
|
||||
/// Record the HTTP outcome separately with
|
||||
/// [`PodcastStore::record_fetch_success`] and friends.
|
||||
pub fn apply_feed(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
refresh: &FeedRefresh<'_>,
|
||||
) -> Result<RefreshOutcome> {
|
||||
let podcast_id = refresh.podcast_id;
|
||||
let feed = refresh.feed;
|
||||
let now = now_ms();
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let current_url = match podcast_fetch_url(&tx, podcast_id)? {
|
||||
Some(url) => url,
|
||||
None => {
|
||||
let Some(register) = apply::load_subscription_register(&tx, podcast_id)? else {
|
||||
return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into());
|
||||
};
|
||||
apply::apply_subscription_record(&tx, ®ister, now)?;
|
||||
register.podcast.feed_url
|
||||
}
|
||||
};
|
||||
|
||||
rekey_legacy_guids(&tx, podcast_id, feed)?;
|
||||
let ids = resolve_and_rekey_provisional(&tx, podcast_id, feed)?;
|
||||
tx.execute(
|
||||
"UPDATE episodes SET in_feed = 0 WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
)?;
|
||||
let mut outcome = RefreshOutcome::default();
|
||||
for (item, id) in feed.episodes.iter().zip(ids) {
|
||||
let Some(id) = id else {
|
||||
outcome.skipped_items += 1;
|
||||
continue;
|
||||
};
|
||||
let meta = item.to_meta(podcast_id, id);
|
||||
if meta.validate().is_err() {
|
||||
outcome.skipped_items += 1;
|
||||
continue;
|
||||
}
|
||||
match upsert_feed_episode(&tx, &meta, now)? {
|
||||
None => outcome.new_episodes += 1,
|
||||
Some(true) => outcome.updated_episodes += 1,
|
||||
Some(false) => outcome.unchanged_episodes += 1,
|
||||
}
|
||||
}
|
||||
|
||||
let mut fetch_url = current_url.clone();
|
||||
let mut learned = BTreeSet::new();
|
||||
if refresh.moved_permanently && *refresh.fetched_from != fetch_url {
|
||||
learned.insert(fetch_url.clone());
|
||||
learned.insert(refresh.fetched_from.clone());
|
||||
fetch_url = refresh.fetched_from.clone();
|
||||
}
|
||||
if let Some(new_url) = feed
|
||||
.new_feed_url
|
||||
.as_deref()
|
||||
.and_then(|url| FeedUrl::parse(url).ok())
|
||||
&& new_url != fetch_url
|
||||
{
|
||||
learned.insert(fetch_url.clone());
|
||||
learned.insert(new_url.clone());
|
||||
fetch_url = new_url;
|
||||
}
|
||||
let meta = feed.to_podcast_meta(fetch_url.clone());
|
||||
tx.execute(
|
||||
"UPDATE podcasts
|
||||
SET feed_url = ?2, title = ?3, author = ?4, description = ?5, link = ?6,
|
||||
image_url = ?7, meta_from_feed = 1, updated_at_ms = ?8
|
||||
WHERE podcast_id = ?1",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
meta.feed_url.as_str(),
|
||||
meta.title,
|
||||
meta.author,
|
||||
meta.description,
|
||||
meta.link,
|
||||
meta.image_url,
|
||||
now
|
||||
],
|
||||
)?;
|
||||
if !learned.is_empty() {
|
||||
let learned: Vec<FeedUrl> = learned.into_iter().collect();
|
||||
let current = apply::load_aliases(&tx, podcast_id)?;
|
||||
if merge_feed_aliases(podcast_id, ¤t, &learned) != current {
|
||||
let op = super::actions::emit(&tx, stamps, |_| {
|
||||
SyncOpPayload::PodcastFeedAliasesAdded {
|
||||
podcast_id: podcast_id.clone(),
|
||||
feed_aliases: learned.clone(),
|
||||
}
|
||||
})?;
|
||||
outcome.ops.push(op);
|
||||
}
|
||||
outcome.learned_aliases = learned;
|
||||
}
|
||||
apply::refresh_seasons(&tx, podcast_id)?;
|
||||
tx.commit()?;
|
||||
outcome.feed_url = Some(fetch_url);
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Local fetch state of a podcast (defaults when never fetched).
|
||||
pub fn fetch_state(&self, podcast_id: &PodcastId) -> Result<FetchState> {
|
||||
fetch_state_in(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// Read-modify-write of the fetch state under one lock and one immediate
|
||||
/// transaction, so concurrent `record_*` / `set_cover_cache` calls (from
|
||||
/// this or another process) never lose each other's columns or failure
|
||||
/// increments.
|
||||
fn update_fetch_state(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
update: impl FnOnce(&mut FetchState),
|
||||
) -> Result<FetchState> {
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
if podcast_fetch_url(&tx, podcast_id)?.is_none() {
|
||||
return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into());
|
||||
}
|
||||
let mut state = fetch_state_in(&tx, podcast_id)?;
|
||||
update(&mut state);
|
||||
tx.execute(
|
||||
"INSERT INTO podcast_fetch_state
|
||||
(podcast_id, etag, last_modified, checked_at_ms, last_success_at_ms,
|
||||
next_check_at_ms, last_error, consecutive_failures, cover_url, cover_path)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||
ON CONFLICT(podcast_id) DO UPDATE SET
|
||||
etag = excluded.etag, last_modified = excluded.last_modified,
|
||||
checked_at_ms = excluded.checked_at_ms,
|
||||
last_success_at_ms = excluded.last_success_at_ms,
|
||||
next_check_at_ms = excluded.next_check_at_ms,
|
||||
last_error = excluded.last_error,
|
||||
consecutive_failures = excluded.consecutive_failures,
|
||||
cover_url = excluded.cover_url, cover_path = excluded.cover_path",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
state.etag,
|
||||
state.last_modified,
|
||||
state.checked_at_ms,
|
||||
state.last_success_at_ms,
|
||||
state.next_check_at_ms,
|
||||
state.last_error,
|
||||
i64::from(state.consecutive_failures),
|
||||
state.cover_url,
|
||||
state.cover_path,
|
||||
],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
/// Records a successful fetch (HTTP 200) with its new validators.
|
||||
pub fn record_fetch_success(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
now_ms: i64,
|
||||
etag: Option<&str>,
|
||||
last_modified: Option<&str>,
|
||||
next_check_at_ms: Option<i64>,
|
||||
) -> Result<()> {
|
||||
self.update_fetch_state(podcast_id, |state| {
|
||||
state.etag = etag.map(str::to_string);
|
||||
state.last_modified = last_modified.map(str::to_string);
|
||||
state.checked_at_ms = Some(now_ms);
|
||||
state.last_success_at_ms = Some(now_ms);
|
||||
state.next_check_at_ms = next_check_at_ms;
|
||||
state.last_error = None;
|
||||
state.consecutive_failures = 0;
|
||||
})
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Records a conditional fetch answered with "304 Not Modified".
|
||||
pub fn record_fetch_not_modified(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
now_ms: i64,
|
||||
next_check_at_ms: Option<i64>,
|
||||
) -> Result<()> {
|
||||
self.update_fetch_state(podcast_id, |state| {
|
||||
state.checked_at_ms = Some(now_ms);
|
||||
state.last_success_at_ms = Some(now_ms);
|
||||
state.next_check_at_ms = next_check_at_ms;
|
||||
state.last_error = None;
|
||||
state.consecutive_failures = 0;
|
||||
})
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Records a failed fetch; returns the consecutive failure count. The
|
||||
/// stored catalog is kept.
|
||||
pub fn record_fetch_failure(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
now_ms: i64,
|
||||
error: &str,
|
||||
next_check_at_ms: Option<i64>,
|
||||
) -> Result<u32> {
|
||||
self.update_fetch_state(podcast_id, |state| {
|
||||
state.checked_at_ms = Some(now_ms);
|
||||
state.next_check_at_ms = next_check_at_ms;
|
||||
state.last_error = Some(error.to_string());
|
||||
state.consecutive_failures = state.consecutive_failures.saturating_add(1);
|
||||
})
|
||||
.map(|state| state.consecutive_failures)
|
||||
}
|
||||
|
||||
/// Records the locally cached cover image of a podcast.
|
||||
pub fn set_cover_cache(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
cover_url: Option<&str>,
|
||||
cover_path: Option<&str>,
|
||||
) -> Result<()> {
|
||||
self.update_fetch_state(podcast_id, |state| {
|
||||
state.cover_url = cover_url.map(str::to_string);
|
||||
state.cover_path = cover_path.map(str::to_string);
|
||||
})
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Subscribed podcasts due for a refresh at `now_ms` (never checked or
|
||||
/// `next_check_at_ms <= now_ms`), most overdue first, with their fetch URL.
|
||||
pub fn podcasts_due_for_refresh(
|
||||
&self,
|
||||
now_ms: i64,
|
||||
limit: usize,
|
||||
) -> Result<Vec<(PodcastId, FeedUrl)>> {
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT p.podcast_id, p.feed_url
|
||||
FROM podcasts p
|
||||
JOIN podcast_subscriptions s ON s.podcast_id = p.podcast_id AND s.subscribed = 1
|
||||
LEFT JOIN podcast_fetch_state f ON f.podcast_id = p.podcast_id
|
||||
WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= ?1
|
||||
ORDER BY IFNULL(f.next_check_at_ms, 0), p.podcast_id
|
||||
LIMIT ?2",
|
||||
)?;
|
||||
let due = statement
|
||||
.query_map(
|
||||
params![now_ms, i64::try_from(limit).unwrap_or(i64::MAX)],
|
||||
|row| {
|
||||
Ok((
|
||||
super::podcast_id_col(row.get(0)?)?,
|
||||
super::feed_url_col(row.get(1)?)?,
|
||||
))
|
||||
},
|
||||
)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(due)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,796 @@
|
||||
//! Schema, migrations and the import of the prototype database.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, EpisodeIdentityHints, EpisodeMeta, EpisodeType, FeedUrl, KnownEpisode,
|
||||
MAX_DESCRIPTION_CHARS, MAX_TITLE_CHARS, MAX_URL_BYTES, PodcastId, PodcastMeta,
|
||||
SubscriptionRecord, bounded_text, is_http_url, normalize_episode_title, resolve_episode_ids,
|
||||
resolve_podcast_id,
|
||||
};
|
||||
use rusqlite::{Connection, OptionalExtension as _, Transaction, TransactionBehavior, params};
|
||||
|
||||
use super::apply::{self, EpisodeSource};
|
||||
use super::{LEGACY_IMPORT_OP_PREFIX, PodcastStoreError, now_ms};
|
||||
|
||||
/// Newest schema version (`PRAGMA user_version`) this build writes.
|
||||
pub const PODCAST_SCHEMA_VERSION: i64 = 2;
|
||||
|
||||
/// `user_version` of the prototype database of the original podcast PR.
|
||||
pub const LEGACY_PROTOTYPE_SCHEMA_VERSION: i64 = 1;
|
||||
|
||||
/// Migration 1: the prototype schema of the original podcast PR, verbatim
|
||||
/// (`furumi-tui` `src/podcasts/store.rs`, `MIGRATIONS[0]`). It versioned
|
||||
/// itself with `PRAGMA user_version = 1` in the same transaction.
|
||||
const PROTOTYPE_SCHEMA_V1: &str = "
|
||||
CREATE TABLE IF NOT EXISTS podcasts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
feed_url TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
description TEXT,
|
||||
link TEXT,
|
||||
image_url TEXT,
|
||||
cover_url TEXT,
|
||||
cover_path TEXT,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
added_at INTEGER NOT NULL,
|
||||
last_checked_at INTEGER,
|
||||
last_error TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS podcast_episodes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
podcast_id INTEGER NOT NULL REFERENCES podcasts(id) ON DELETE CASCADE,
|
||||
guid TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
published_at INTEGER,
|
||||
duration_seconds REAL,
|
||||
audio_url TEXT NOT NULL,
|
||||
mime_type TEXT,
|
||||
byte_len INTEGER,
|
||||
link TEXT,
|
||||
image_url TEXT,
|
||||
first_seen_at INTEGER NOT NULL,
|
||||
UNIQUE (podcast_id, guid)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS podcast_episodes_by_date
|
||||
ON podcast_episodes (podcast_id, published_at DESC, id DESC);
|
||||
";
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) const PROTOTYPE_SCHEMA_V1_FOR_TESTS: &str = PROTOTYPE_SCHEMA_V1;
|
||||
|
||||
/// Columns every prototype `podcasts` table has.
|
||||
const PROTOTYPE_PODCAST_COLUMNS: &[&str] = &[
|
||||
"id",
|
||||
"feed_url",
|
||||
"title",
|
||||
"author",
|
||||
"description",
|
||||
"link",
|
||||
"image_url",
|
||||
"cover_url",
|
||||
"cover_path",
|
||||
"etag",
|
||||
"last_modified",
|
||||
"added_at",
|
||||
"last_checked_at",
|
||||
"last_error",
|
||||
];
|
||||
|
||||
/// Columns every prototype `podcast_episodes` table has.
|
||||
const PROTOTYPE_EPISODE_COLUMNS: &[&str] = &[
|
||||
"id",
|
||||
"podcast_id",
|
||||
"guid",
|
||||
"title",
|
||||
"description",
|
||||
"published_at",
|
||||
"duration_seconds",
|
||||
"audio_url",
|
||||
"mime_type",
|
||||
"byte_len",
|
||||
"link",
|
||||
"image_url",
|
||||
"first_seen_at",
|
||||
];
|
||||
|
||||
/// Migration 2: the contract model. See `PODCASTS.md`, "SQLite reference store".
|
||||
const SCHEMA_V2: &str = "
|
||||
CREATE TABLE IF NOT EXISTS podcasts (
|
||||
podcast_id TEXT PRIMARY KEY,
|
||||
feed_url TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
description TEXT,
|
||||
link TEXT,
|
||||
image_url TEXT,
|
||||
meta_from_feed INTEGER NOT NULL DEFAULT 0,
|
||||
created_at_ms INTEGER NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS podcast_feed_aliases (
|
||||
podcast_id TEXT NOT NULL,
|
||||
feed_url TEXT NOT NULL,
|
||||
PRIMARY KEY (podcast_id, feed_url)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS podcast_feed_aliases_by_url
|
||||
ON podcast_feed_aliases (feed_url);
|
||||
CREATE TABLE IF NOT EXISTS episodes (
|
||||
episode_id TEXT PRIMARY KEY,
|
||||
podcast_id TEXT NOT NULL REFERENCES podcasts(podcast_id),
|
||||
guid TEXT,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
season INTEGER,
|
||||
episode_number INTEGER,
|
||||
episode_type TEXT NOT NULL DEFAULT 'full',
|
||||
published_at_ms INTEGER,
|
||||
duration_ms INTEGER,
|
||||
audio_url TEXT NOT NULL,
|
||||
mime TEXT,
|
||||
byte_len INTEGER,
|
||||
image_url TEXT,
|
||||
link TEXT,
|
||||
source TEXT NOT NULL CHECK (source IN ('feed', 'sync')),
|
||||
in_feed INTEGER NOT NULL DEFAULT 0,
|
||||
legacy_guid TEXT,
|
||||
first_seen_at_ms INTEGER NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS episodes_by_podcast_date
|
||||
ON episodes (podcast_id, published_at_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS episodes_by_podcast_season
|
||||
ON episodes (podcast_id, season, episode_number);
|
||||
CREATE TABLE IF NOT EXISTS podcast_seasons (
|
||||
podcast_id TEXT NOT NULL REFERENCES podcasts(podcast_id),
|
||||
season INTEGER,
|
||||
episode_count INTEGER NOT NULL,
|
||||
first_published_at_ms INTEGER,
|
||||
last_published_at_ms INTEGER
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS podcast_seasons_key
|
||||
ON podcast_seasons (podcast_id, IFNULL(season, -1));
|
||||
CREATE TABLE IF NOT EXISTS podcast_subscriptions (
|
||||
podcast_id TEXT PRIMARY KEY,
|
||||
subscribed INTEGER NOT NULL,
|
||||
podcast_json TEXT NOT NULL,
|
||||
generation INTEGER NOT NULL CHECK (generation >= 0),
|
||||
hlc_ms INTEGER NOT NULL,
|
||||
op_id TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS podcast_shares (
|
||||
podcast_id TEXT PRIMARY KEY,
|
||||
shared INTEGER NOT NULL,
|
||||
generation INTEGER NOT NULL CHECK (generation >= 0),
|
||||
hlc_ms INTEGER NOT NULL,
|
||||
op_id TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS episode_progress (
|
||||
episode_id TEXT PRIMARY KEY,
|
||||
podcast_id TEXT NOT NULL,
|
||||
position_ms INTEGER NOT NULL,
|
||||
status TEXT NOT NULL
|
||||
CHECK (status IN ('not_started', 'in_progress', 'completed')),
|
||||
generation INTEGER NOT NULL CHECK (generation >= 0),
|
||||
session_id TEXT NOT NULL,
|
||||
hlc_ms INTEGER NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
op_id TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS episode_progress_by_podcast
|
||||
ON episode_progress (podcast_id);
|
||||
CREATE TABLE IF NOT EXISTS podcast_listen_sessions (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
episode_id TEXT NOT NULL,
|
||||
podcast_id TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
started_at_ms INTEGER NOT NULL,
|
||||
ended_at_ms INTEGER NOT NULL,
|
||||
start_position_ms INTEGER NOT NULL,
|
||||
end_position_ms INTEGER NOT NULL,
|
||||
listened_ms INTEGER NOT NULL,
|
||||
completed INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS podcast_listen_sessions_by_episode
|
||||
ON podcast_listen_sessions (episode_id);
|
||||
CREATE INDEX IF NOT EXISTS podcast_listen_sessions_by_end
|
||||
ON podcast_listen_sessions (ended_at_ms DESC);
|
||||
CREATE TABLE IF NOT EXISTS podcast_fetch_state (
|
||||
podcast_id TEXT PRIMARY KEY REFERENCES podcasts(podcast_id),
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
checked_at_ms INTEGER,
|
||||
last_success_at_ms INTEGER,
|
||||
next_check_at_ms INTEGER,
|
||||
last_error TEXT,
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
cover_url TEXT,
|
||||
cover_path TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS legacy_quarantine (
|
||||
legacy_table TEXT NOT NULL,
|
||||
legacy_id INTEGER NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
detail TEXT,
|
||||
PRIMARY KEY (legacy_table, legacy_id)
|
||||
);
|
||||
";
|
||||
|
||||
/// Tables a version-2 store owns (the `legacy_*` tables included).
|
||||
const V2_TABLES: &[&str] = &[
|
||||
"podcasts",
|
||||
"podcast_feed_aliases",
|
||||
"episodes",
|
||||
"podcast_seasons",
|
||||
"podcast_subscriptions",
|
||||
"podcast_shares",
|
||||
"episode_progress",
|
||||
"podcast_listen_sessions",
|
||||
"podcast_fetch_state",
|
||||
"legacy_quarantine",
|
||||
"legacy_podcasts",
|
||||
"legacy_podcast_episodes",
|
||||
];
|
||||
|
||||
fn user_version(conn: &Connection) -> Result<i64> {
|
||||
Ok(conn.query_row("PRAGMA user_version", [], |row| row.get(0))?)
|
||||
}
|
||||
|
||||
fn user_tables(conn: &Connection) -> Result<BTreeSet<String>> {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
|
||||
)?;
|
||||
let names = statement
|
||||
.query_map([], |row| row.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<BTreeSet<_>>>()?;
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
pub(crate) fn table_columns(conn: &Connection, table: &str) -> Result<BTreeSet<String>> {
|
||||
let mut statement = conn.prepare("SELECT name FROM pragma_table_info(?1)")?;
|
||||
let names = statement
|
||||
.query_map([table], |row| row.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<BTreeSet<_>>>()?;
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
fn unrecognized(reason: impl Into<String>) -> anyhow::Error {
|
||||
PodcastStoreError::UnrecognizedDatabase(reason.into()).into()
|
||||
}
|
||||
|
||||
fn has_columns(conn: &Connection, table: &str, required: &[&str]) -> Result<bool> {
|
||||
let columns = table_columns(conn, table)?;
|
||||
Ok(required.iter().all(|column| columns.contains(*column)))
|
||||
}
|
||||
|
||||
/// Checks the prototype tables that exist have the prototype shape.
|
||||
fn check_prototype_shape(conn: &Connection, tables: &BTreeSet<String>) -> Result<()> {
|
||||
if tables.contains("podcasts") && !has_columns(conn, "podcasts", PROTOTYPE_PODCAST_COLUMNS)? {
|
||||
return Err(unrecognized(
|
||||
"table `podcasts` does not have the prototype columns",
|
||||
));
|
||||
}
|
||||
if tables.contains("podcast_episodes")
|
||||
&& !has_columns(conn, "podcast_episodes", PROTOTYPE_EPISODE_COLUMNS)?
|
||||
{
|
||||
return Err(unrecognized(
|
||||
"table `podcast_episodes` does not have the prototype columns",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read-only check run before anything is written to the file.
|
||||
///
|
||||
/// * `user_version` above [`PODCAST_SCHEMA_VERSION`]: written by a newer
|
||||
/// Furumi, refused.
|
||||
/// * 0: must be empty, or hold only the prototype tables (the prototype
|
||||
/// sets its version in the same transaction, so this is defensive).
|
||||
/// * 1: the prototype database; its tables must have the prototype shape.
|
||||
/// * 2: the current schema.
|
||||
///
|
||||
/// Any other table means the path points at a different database (for
|
||||
/// example `library.db`), which is refused instead of being modified.
|
||||
pub(super) fn check_openable(conn: &Connection) -> Result<()> {
|
||||
let version = user_version(conn)?;
|
||||
if version > PODCAST_SCHEMA_VERSION {
|
||||
return Err(PodcastStoreError::NewerSchema {
|
||||
found: version,
|
||||
supported: PODCAST_SCHEMA_VERSION,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
if version < 0 {
|
||||
return Err(unrecognized(format!("negative user_version {version}")));
|
||||
}
|
||||
let tables = user_tables(conn)?;
|
||||
let allowed: &[&str] = if version < PODCAST_SCHEMA_VERSION {
|
||||
&["podcasts", "podcast_episodes"]
|
||||
} else {
|
||||
V2_TABLES
|
||||
};
|
||||
if let Some(table) = tables
|
||||
.iter()
|
||||
.find(|table| !allowed.contains(&table.as_str()))
|
||||
{
|
||||
return Err(unrecognized(format!(
|
||||
"unexpected table `{table}` at schema version {version}"
|
||||
)));
|
||||
}
|
||||
if version < PODCAST_SCHEMA_VERSION {
|
||||
check_prototype_shape(conn, &tables)?;
|
||||
} else if tables.contains("podcasts")
|
||||
&& !has_columns(conn, "podcasts", &["podcast_id", "feed_url"])?
|
||||
{
|
||||
return Err(unrecognized("table `podcasts` is not the version 2 table"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Applies every pending migration, each in its own immediate transaction
|
||||
/// that re-reads the version (another process may have migrated meanwhile).
|
||||
pub(super) fn migrate(conn: &mut Connection) -> Result<()> {
|
||||
for target in 1..=PODCAST_SCHEMA_VERSION {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let version = user_version(&tx)?;
|
||||
if version > PODCAST_SCHEMA_VERSION {
|
||||
return Err(PodcastStoreError::NewerSchema {
|
||||
found: version,
|
||||
supported: PODCAST_SCHEMA_VERSION,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
if version >= target {
|
||||
continue;
|
||||
}
|
||||
match target {
|
||||
1 => tx
|
||||
.execute_batch(PROTOTYPE_SCHEMA_V1)
|
||||
.context("creating the prototype podcast schema (v1)")?,
|
||||
2 => migrate_to_v2(&tx).context("migrating the podcast database to v2")?,
|
||||
_ => unreachable!("no migration for version {target}"),
|
||||
}
|
||||
tx.pragma_update(None, "user_version", target)?;
|
||||
tx.commit()
|
||||
.with_context(|| format!("committing podcast schema v{target}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_to_v2(tx: &Transaction<'_>) -> Result<()> {
|
||||
let tables = user_tables(tx)?;
|
||||
// Rename, never drop: the prototype data stays available as `legacy_*`.
|
||||
// Renaming `podcasts` also rewrites the foreign key of the episode table.
|
||||
if tables.contains("podcasts") && !tables.contains("legacy_podcasts") {
|
||||
tx.execute_batch("ALTER TABLE podcasts RENAME TO legacy_podcasts")?;
|
||||
}
|
||||
if tables.contains("podcast_episodes") && !tables.contains("legacy_podcast_episodes") {
|
||||
tx.execute_batch("ALTER TABLE podcast_episodes RENAME TO legacy_podcast_episodes")?;
|
||||
}
|
||||
tx.execute_batch(SCHEMA_V2)?;
|
||||
let tables = user_tables(tx)?;
|
||||
if tables.contains("legacy_podcasts") {
|
||||
import_prototype(tx, tables.contains("legacy_podcast_episodes"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct LegacyPodcast {
|
||||
id: i64,
|
||||
feed_url: String,
|
||||
title: String,
|
||||
author: Option<String>,
|
||||
description: Option<String>,
|
||||
link: Option<String>,
|
||||
image_url: Option<String>,
|
||||
cover_url: Option<String>,
|
||||
cover_path: Option<String>,
|
||||
etag: Option<String>,
|
||||
last_modified: Option<String>,
|
||||
added_at: i64,
|
||||
last_checked_at: Option<i64>,
|
||||
last_error: Option<String>,
|
||||
}
|
||||
|
||||
struct LegacyEpisode {
|
||||
id: i64,
|
||||
guid: String,
|
||||
title: String,
|
||||
description: Option<String>,
|
||||
published_at: Option<i64>,
|
||||
duration_seconds: Option<f64>,
|
||||
audio_url: String,
|
||||
mime_type: Option<String>,
|
||||
byte_len: Option<i64>,
|
||||
link: Option<String>,
|
||||
image_url: Option<String>,
|
||||
first_seen_at: i64,
|
||||
}
|
||||
|
||||
fn quarantine(
|
||||
tx: &Transaction<'_>,
|
||||
table: &str,
|
||||
id: i64,
|
||||
reason: &str,
|
||||
detail: &str,
|
||||
) -> Result<()> {
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO legacy_quarantine (legacy_table, legacy_id, reason, detail)
|
||||
VALUES (?1, ?2, ?3, ?4)",
|
||||
params![table, id, reason, detail],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn secs_to_ms(secs: i64) -> i64 {
|
||||
secs.saturating_mul(1_000)
|
||||
}
|
||||
|
||||
/// Imports the prototype tables into the contract model.
|
||||
///
|
||||
/// * Every prototype row is an active subscription: it becomes a
|
||||
/// subscription register with `subscribed = true`, generation 0,
|
||||
/// `hlc_ms` = `added_at` and `op_id` = [`LEGACY_IMPORT_OP_PREFIX`] + id,
|
||||
/// so any later contract write orders after it. Other devices learn it
|
||||
/// through the snapshot sections. Nothing is shared (private by default).
|
||||
/// * The podcast id is derived from the canonical feed URL
|
||||
/// ([`FeedUrl::parse`], falling back to [`FeedUrl::from_user_input`]); rows
|
||||
/// whose URL cannot be canonicalized are recorded in `legacy_quarantine`
|
||||
/// and stay in `legacy_podcasts` with their episodes. Rows with the same
|
||||
/// canonical URL merge into one podcast (the first row's subscription and
|
||||
/// fetch state win; the episodes of all of them are imported).
|
||||
/// * Episodes get ids from the contract identity rule over the stored rows.
|
||||
/// The prototype stored the enclosure URL as `guid` when the item had
|
||||
/// none: such rows are imported as GUID-less items and keep the stored
|
||||
/// value in `legacy_guid`. Rows of one GUID-less item seen under several
|
||||
/// enclosure URLs are collapsed into the newest (the others are
|
||||
/// quarantined as duplicates). Because stale rows can still change which
|
||||
/// key is unique, these ids are provisional: the first refresh that lists
|
||||
/// the item re-keys an episode without progress or sessions to the id the
|
||||
/// current feed gives (see `refresh.rs`), including its GUID id when the
|
||||
/// stored value turns out to be a real GUID.
|
||||
/// * ETag, Last-Modified, check time, last error and the cover cache move to
|
||||
/// the local fetch state. The prototype has no progress or history, so no
|
||||
/// progress register or session is created.
|
||||
fn import_prototype(tx: &Transaction<'_>, with_episodes: bool) -> Result<()> {
|
||||
let now = now_ms();
|
||||
let legacy: Vec<LegacyPodcast> = {
|
||||
let mut statement = tx.prepare(
|
||||
"SELECT id, feed_url, title, author, description, link, image_url, cover_url,
|
||||
cover_path, etag, last_modified, added_at, last_checked_at, last_error
|
||||
FROM legacy_podcasts ORDER BY id",
|
||||
)?;
|
||||
statement
|
||||
.query_map([], |row| {
|
||||
Ok(LegacyPodcast {
|
||||
id: row.get(0)?,
|
||||
feed_url: row.get(1)?,
|
||||
title: row.get(2)?,
|
||||
author: row.get(3)?,
|
||||
description: row.get(4)?,
|
||||
link: row.get(5)?,
|
||||
image_url: row.get(6)?,
|
||||
cover_url: row.get(7)?,
|
||||
cover_path: row.get(8)?,
|
||||
etag: row.get(9)?,
|
||||
last_modified: row.get(10)?,
|
||||
added_at: row.get(11)?,
|
||||
last_checked_at: row.get(12)?,
|
||||
last_error: row.get(13)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let mut known_urls: Vec<(PodcastId, FeedUrl)> = Vec::new();
|
||||
for podcast in legacy {
|
||||
let url = match FeedUrl::parse(&podcast.feed_url)
|
||||
.or_else(|_| FeedUrl::from_user_input(&podcast.feed_url))
|
||||
{
|
||||
Ok(url) => url,
|
||||
Err(error) => {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcasts",
|
||||
podcast.id,
|
||||
"invalid feed URL",
|
||||
&error.to_string(),
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let podcast_id = resolve_podcast_id(&url, known_urls.iter().map(|(id, url)| (id, url)));
|
||||
let duplicate = known_urls.iter().any(|(id, _)| *id == podcast_id);
|
||||
known_urls.push((podcast_id.clone(), url.clone()));
|
||||
if duplicate {
|
||||
// Same canonical feed as an earlier row: its episodes join that
|
||||
// podcast; the first row's subscription and fetch state are kept.
|
||||
if with_episodes {
|
||||
import_prototype_episodes(tx, &podcast, &podcast_id, now)?;
|
||||
}
|
||||
apply::refresh_seasons(tx, &podcast_id)?;
|
||||
continue;
|
||||
}
|
||||
let meta = PodcastMeta {
|
||||
feed_url: url.clone(),
|
||||
title: podcast.title.clone(),
|
||||
author: podcast.author.clone(),
|
||||
description: podcast.description.clone(),
|
||||
link: podcast.link.clone(),
|
||||
image_url: podcast.image_url.clone(),
|
||||
}
|
||||
.bounded();
|
||||
let added_ms = secs_to_ms(podcast.added_at);
|
||||
let record = SubscriptionRecord {
|
||||
podcast_id: podcast_id.clone(),
|
||||
subscribed: true,
|
||||
podcast: meta.clone(),
|
||||
feed_aliases: vec![url.clone()],
|
||||
generation: 0,
|
||||
hlc_ms: added_ms,
|
||||
op_id: format!("{LEGACY_IMPORT_OP_PREFIX}{podcast_id}"),
|
||||
};
|
||||
apply::apply_subscription_record(tx, &record, now)?;
|
||||
// The prototype metadata came from the feed itself.
|
||||
tx.execute(
|
||||
"UPDATE podcasts
|
||||
SET title = ?2, author = ?3, description = ?4, link = ?5, image_url = ?6,
|
||||
meta_from_feed = 1, created_at_ms = MIN(created_at_ms, ?7)
|
||||
WHERE podcast_id = ?1 AND meta_from_feed = 0",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
meta.title,
|
||||
meta.author,
|
||||
meta.description,
|
||||
meta.link,
|
||||
meta.image_url,
|
||||
added_ms
|
||||
],
|
||||
)?;
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO podcast_fetch_state
|
||||
(podcast_id, etag, last_modified, checked_at_ms, last_success_at_ms,
|
||||
next_check_at_ms, last_error, consecutive_failures, cover_url, cover_path)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, NULL, ?6, ?7, ?8, ?9)",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
podcast.etag,
|
||||
podcast.last_modified,
|
||||
podcast.last_checked_at.map(secs_to_ms),
|
||||
podcast
|
||||
.last_error
|
||||
.is_none()
|
||||
.then(|| podcast.last_checked_at.map(secs_to_ms))
|
||||
.flatten(),
|
||||
podcast.last_error,
|
||||
i64::from(podcast.last_error.is_some()),
|
||||
podcast.cover_url,
|
||||
podcast.cover_path,
|
||||
],
|
||||
)?;
|
||||
if with_episodes {
|
||||
import_prototype_episodes(tx, &podcast, &podcast_id, now)?;
|
||||
}
|
||||
apply::refresh_seasons(tx, &podcast_id)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn import_prototype_episodes(
|
||||
tx: &Transaction<'_>,
|
||||
podcast: &LegacyPodcast,
|
||||
podcast_id: &PodcastId,
|
||||
now: i64,
|
||||
) -> Result<()> {
|
||||
let rows: Vec<LegacyEpisode> = {
|
||||
let mut statement = tx.prepare(
|
||||
"SELECT id, guid, title, description, published_at, duration_seconds, audio_url,
|
||||
mime_type, byte_len, link, image_url, first_seen_at
|
||||
FROM legacy_podcast_episodes WHERE podcast_id = ?1 ORDER BY id",
|
||||
)?;
|
||||
statement
|
||||
.query_map([podcast.id], |row| {
|
||||
Ok(LegacyEpisode {
|
||||
id: row.get(0)?,
|
||||
guid: row.get(1)?,
|
||||
title: row.get(2)?,
|
||||
description: row.get(3)?,
|
||||
published_at: row.get(4)?,
|
||||
duration_seconds: row.get(5)?,
|
||||
audio_url: row.get(6)?,
|
||||
mime_type: row.get(7)?,
|
||||
byte_len: row.get(8)?,
|
||||
link: row.get(9)?,
|
||||
image_url: row.get(10)?,
|
||||
first_seen_at: row.get(11)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let mut usable = Vec::new();
|
||||
for row in rows {
|
||||
if !is_http_url(&row.audio_url) {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"invalid audio URL",
|
||||
&row.audio_url,
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
usable.push(row);
|
||||
}
|
||||
// The prototype used the enclosure URL as GUID when the item had none.
|
||||
let real_guid = |row: &LegacyEpisode| {
|
||||
let guid = row.guid.trim();
|
||||
(!guid.is_empty() && guid != row.audio_url.trim()).then(|| guid.to_string())
|
||||
};
|
||||
let usable = collapse_moved_guidless_rows(tx, usable, real_guid)?;
|
||||
let hints: Vec<EpisodeIdentityHints> = usable
|
||||
.iter()
|
||||
.map(|row| EpisodeIdentityHints {
|
||||
guid: real_guid(row),
|
||||
enclosure_url: Some(row.audio_url.clone()),
|
||||
link: row.link.clone(),
|
||||
published_at_ms: row.published_at.map(secs_to_ms),
|
||||
title: Some(row.title.clone()),
|
||||
})
|
||||
.collect();
|
||||
let known: Vec<KnownEpisode> = apply::known_episodes(tx, podcast_id)?;
|
||||
let ids = resolve_episode_ids(podcast_id, podcast.link.as_deref(), &hints, &known);
|
||||
let mut seen: BTreeMap<EpisodeId, i64> = BTreeMap::new();
|
||||
for ((row, hint), id) in usable.iter().zip(&hints).zip(ids) {
|
||||
let Some(episode_id) = id else {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"episode cannot be identified",
|
||||
"no usable identity key or duplicate item",
|
||||
)?;
|
||||
continue;
|
||||
};
|
||||
if let Some(first) = seen.insert(episode_id.clone(), row.id) {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"duplicate episode",
|
||||
&format!("same identity as legacy episode {first}"),
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
let meta = EpisodeMeta {
|
||||
episode_id,
|
||||
podcast_id: podcast_id.clone(),
|
||||
guid: hint.guid.clone(),
|
||||
title: bounded_title(&row.title),
|
||||
description: row
|
||||
.description
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|text| !text.is_empty())
|
||||
.map(|text| bounded_text(text, MAX_DESCRIPTION_CHARS)),
|
||||
season: None,
|
||||
episode_number: None,
|
||||
episode_type: EpisodeType::Full,
|
||||
published_at_ms: row.published_at.map(secs_to_ms),
|
||||
duration_ms: row
|
||||
.duration_seconds
|
||||
.filter(|secs| secs.is_finite() && *secs > 0.0)
|
||||
.map(|secs| (secs * 1_000.0).round() as i64),
|
||||
audio_url: row.audio_url.clone(),
|
||||
mime: row.mime_type.clone().filter(|mime| {
|
||||
!mime.is_empty() && mime.len() <= 128 && mime.bytes().all(|b| b.is_ascii_graphic())
|
||||
}),
|
||||
byte_len: row.byte_len.and_then(|len| u64::try_from(len).ok()),
|
||||
image_url: row.image_url.clone().filter(|url| is_http_url(url)),
|
||||
link: row.link.clone().filter(|link| link.len() <= MAX_URL_BYTES),
|
||||
};
|
||||
if let Err(error) = meta.validate() {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"invalid episode metadata",
|
||||
&error.to_string(),
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
let legacy_guid = hint.guid.is_none().then(|| row.guid.clone());
|
||||
let exists: bool = tx
|
||||
.query_row(
|
||||
"SELECT 1 FROM episodes WHERE episode_id = ?1",
|
||||
[meta.episode_id.as_str()],
|
||||
|_| Ok(true),
|
||||
)
|
||||
.optional()?
|
||||
.unwrap_or(false);
|
||||
if exists {
|
||||
continue;
|
||||
}
|
||||
apply::insert_episode(
|
||||
tx,
|
||||
&meta,
|
||||
EpisodeSource::Feed,
|
||||
true,
|
||||
legacy_guid.as_deref(),
|
||||
secs_to_ms(row.first_seen_at),
|
||||
now,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Collapses prototype rows that are one GUID-less item seen under several
|
||||
/// enclosure URLs.
|
||||
///
|
||||
/// The prototype keyed GUID-less items by their enclosure URL, so an item
|
||||
/// whose audio URL changed (new CDN, tracking prefix) left one row per URL.
|
||||
/// Those stale rows are not part of the current feed; resolved together with
|
||||
/// it they would make the item's `<link>` and `(date, title)` look
|
||||
/// non-unique and give it an id a fresh device never derives. Rows with the
|
||||
/// same publication second, normalized title and `<link>` (or none) are the
|
||||
/// same item: the newest (`first_seen_at`, then `id`) is imported, the others
|
||||
/// are quarantined as duplicates. The link must match too, so distinct
|
||||
/// episodes that only share a title and second are never merged.
|
||||
fn collapse_moved_guidless_rows(
|
||||
tx: &Transaction<'_>,
|
||||
rows: Vec<LegacyEpisode>,
|
||||
real_guid: impl Fn(&LegacyEpisode) -> Option<String>,
|
||||
) -> Result<Vec<LegacyEpisode>> {
|
||||
type ItemKey = (i64, String, Option<String>);
|
||||
let key = |row: &LegacyEpisode| -> Option<ItemKey> {
|
||||
if real_guid(row).is_some() {
|
||||
return None;
|
||||
}
|
||||
let title = normalize_episode_title(&row.title);
|
||||
let link = row
|
||||
.link
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|link| !link.is_empty())
|
||||
.map(str::to_string);
|
||||
(!title.is_empty()).then_some((row.published_at?, title, link))
|
||||
};
|
||||
let mut newest: BTreeMap<ItemKey, (i64, i64)> = BTreeMap::new();
|
||||
for row in &rows {
|
||||
if let Some(key) = key(row) {
|
||||
let candidate = (row.first_seen_at, row.id);
|
||||
newest
|
||||
.entry(key)
|
||||
.and_modify(|best| *best = (*best).max(candidate))
|
||||
.or_insert(candidate);
|
||||
}
|
||||
}
|
||||
let mut kept = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let keeper = key(&row).and_then(|key| newest.get(&key).map(|(_, id)| *id));
|
||||
match keeper {
|
||||
Some(keeper) if keeper != row.id => quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"duplicate episode",
|
||||
&format!(
|
||||
"older enclosure URL of the GUID-less item imported from legacy episode {keeper}"
|
||||
),
|
||||
)?,
|
||||
_ => kept.push(row),
|
||||
}
|
||||
}
|
||||
Ok(kept)
|
||||
}
|
||||
|
||||
fn bounded_title(title: &str) -> String {
|
||||
let title = title.trim();
|
||||
if title.is_empty() {
|
||||
"Untitled episode".to_string()
|
||||
} else {
|
||||
bounded_text(title, MAX_TITLE_CHARS)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -250,6 +250,35 @@ reported itself (`profile_protocol_version`; relayed profiles go through
|
||||
`jam::redact_jam_snapshot`, `jam_command_allowed` and
|
||||
`restore_jam_private_items`.
|
||||
|
||||
### Public podcast discovery
|
||||
|
||||
`music_dht::podcast_dht::PodcastDht` is a separate signed overlay (ALPN
|
||||
`furumi-fd/podcast-dht/1`, capability `podcast_dht`) for podcasts the user
|
||||
explicitly shared. Wiring, as for `SimilarityDht`:
|
||||
|
||||
1. declare the ALPN with `schema_independent_stream_protocol(PODCAST_DHT_ALPN)`,
|
||||
open the overlay with its own cache file, take
|
||||
`stream_acceptor(PODCAST_DHT_ALPN)` and spawn `serve` and `maintenance`;
|
||||
2. advertise it with `CapabilityManifest::with_podcast_dht()` (it is not in
|
||||
`CapabilityManifest::frid()`; an application that does not run the overlay
|
||||
simply does not add it);
|
||||
3. pass every peer manifest received to `note_peer_capabilities`, so peers
|
||||
without the overlay are not dialed;
|
||||
4. call `sync_shares` with the merged subscription and share registers on
|
||||
start and after applying sync operations (until the first call, own cards
|
||||
persisted by an earlier run are neither republished nor served): it
|
||||
publishes cards only where `effective_share` holds, tombstones every other
|
||||
own publication, and reports cards it could not build (`failed`) or held
|
||||
back for privacy (`held_back`);
|
||||
5. ask the user before publishing when `PodcastCardSpec::privacy_warnings`
|
||||
(or `feed_url_privacy_warning`) reports a URL that looks private, then
|
||||
call `publish` or set `PodcastShareState::privacy_confirmed`.
|
||||
|
||||
`search` and `lookup_podcast` return `PodcastSearchHit`s; subscribe with the
|
||||
hit's `feed_url` (the URL the id hashes from), never with a
|
||||
`claimed_feed_urls` entry. `merge_search_results` combines hits with
|
||||
`SearchOutcome::network_results` as typed `FederatedSearchHit`s.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
|
||||
@@ -35,6 +35,8 @@ pub const CATALOG_ID: &str = "catalog";
|
||||
pub const SIMILARITY_ID: &str = "similarity";
|
||||
/// Stable protocol identifier for DHT-routed similarity discovery.
|
||||
pub const SIMILARITY_DHT_ID: &str = "similarity_dht";
|
||||
/// Stable protocol identifier for the public podcast-discovery overlay.
|
||||
pub const PODCAST_DHT_ID: &str = "podcast_dht";
|
||||
/// Stable protocol identifier for personal-device synchronization.
|
||||
pub const DEVICE_SYNC_ID: &str = "device_sync";
|
||||
/// Stable protocol identifier for Jam playback control.
|
||||
@@ -54,7 +56,13 @@ pub struct CapabilityManifest {
|
||||
}
|
||||
|
||||
impl CapabilityManifest {
|
||||
/// Creates a manifest containing every protocol owned by Frid.
|
||||
/// Creates a manifest containing every protocol owned by Frid that a
|
||||
/// music-DHT application serves.
|
||||
///
|
||||
/// The public podcast-discovery overlay is opt-in: applications that run
|
||||
/// [`crate::podcast_dht::PodcastDht`] add it with
|
||||
/// [`Self::with_podcast_dht`], so an application that does not serve the
|
||||
/// overlay never advertises it.
|
||||
pub fn frid(application: impl Into<String>, application_version: impl Into<String>) -> Self {
|
||||
let mut protocols = BTreeMap::new();
|
||||
protocols.insert(
|
||||
@@ -105,6 +113,29 @@ impl CapabilityManifest {
|
||||
self
|
||||
}
|
||||
|
||||
/// Advertises the public podcast-discovery overlay
|
||||
/// ([`PODCAST_DHT_ID`] at [`crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||
/// Call it only when the application serves
|
||||
/// [`crate::podcast_dht::PODCAST_DHT_ALPN`].
|
||||
pub fn with_podcast_dht(self) -> Self {
|
||||
self.with_protocol(
|
||||
PODCAST_DHT_ID,
|
||||
crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION,
|
||||
)
|
||||
}
|
||||
|
||||
/// Version of protocol `id` advertised by the peer, if any.
|
||||
pub fn protocol_version(&self, id: &str) -> Option<u16> {
|
||||
self.protocols.get(id).copied()
|
||||
}
|
||||
|
||||
/// Whether the peer advertises the public podcast-discovery overlay at
|
||||
/// the version this crate speaks. Peers without it are never queried.
|
||||
pub fn supports_podcast_dht(&self) -> bool {
|
||||
self.protocol_version(PODCAST_DHT_ID)
|
||||
== Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
|
||||
}
|
||||
|
||||
/// Validates bounds and identifiers received from a peer.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.manifest_version != CAPABILITIES_PROTOCOL_VERSION {
|
||||
@@ -227,6 +258,24 @@ mod tests {
|
||||
manifest.validate().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_dht_capability_is_opt_in_and_removable() {
|
||||
let plain = CapabilityManifest::frid("furumi", "1.2.3");
|
||||
assert_eq!(plain.protocol_version(PODCAST_DHT_ID), None);
|
||||
assert!(!plain.supports_podcast_dht());
|
||||
let manifest = plain.with_podcast_dht();
|
||||
assert_eq!(
|
||||
manifest.protocol_version(PODCAST_DHT_ID),
|
||||
Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
|
||||
);
|
||||
assert!(manifest.supports_podcast_dht());
|
||||
manifest.validate().unwrap();
|
||||
let without = manifest.clone().without_protocol(PODCAST_DHT_ID);
|
||||
assert!(!without.supports_podcast_dht());
|
||||
let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2);
|
||||
assert!(!other_version.supports_podcast_dht());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn application_protocol_is_additive() {
|
||||
let manifest = CapabilityManifest::frid("furumi", "1.2.3").with_protocol("audio", 1);
|
||||
|
||||
@@ -50,6 +50,11 @@ pub enum MusicDhtError {
|
||||
/// The service is shutting down and no longer accepts operations.
|
||||
#[error("service is shutting down")]
|
||||
ShuttingDown,
|
||||
|
||||
/// A podcast card cannot be published, for example because the user did
|
||||
/// not share the podcast (see `music_dht::podcast::effective_share`).
|
||||
#[error("podcast publication refused: {0}")]
|
||||
PodcastPublication(String),
|
||||
}
|
||||
|
||||
impl From<federation_net::NetworkError> for MusicDhtError {
|
||||
|
||||
@@ -27,6 +27,9 @@
|
||||
//! podcast/episode ids, metadata DTOs and convergent merge rules), with an
|
||||
//! optional RSS/Atom parser behind the `podcast-feed` feature;
|
||||
//! [`device_sync`] protocol v4 carries it between trusted devices.
|
||||
//! * [`podcast_dht`] is a separate signed overlay for public podcast
|
||||
//! discovery: cards a user explicitly shared, found by title/author tokens
|
||||
//! or podcast id, retracted by signed tombstones.
|
||||
//! * Records are replicated to the `K` nodes whose ids are XOR-closest to
|
||||
//! each key. Publishers pick the targets from their routing table and send
|
||||
//! batched store requests (one pipeline per peer), so even a large library
|
||||
@@ -97,6 +100,7 @@ mod message;
|
||||
mod node;
|
||||
mod normalization;
|
||||
pub mod podcast;
|
||||
pub mod podcast_dht;
|
||||
mod record;
|
||||
mod request;
|
||||
mod routing;
|
||||
|
||||
@@ -0,0 +1,899 @@
|
||||
//! Signed public podcast cards, tombstones, DHT keys and their canonical
|
||||
//! signing encoding.
|
||||
//!
|
||||
//! Everything here is pure and deterministic: no clock, no I/O. Callers pass
|
||||
//! the current time to the validation functions.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
|
||||
use federation_net::{EndpointId, NetworkId, Signature};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{MusicDhtError, Result};
|
||||
use crate::normalization::{normalize_name, tokenize};
|
||||
use crate::podcast::{
|
||||
FeedUrl, MAX_URL_BYTES, PodcastId, SubscriptionRecord, bounded_text, is_http_url,
|
||||
};
|
||||
|
||||
use super::privacy::{FeedUrlPrivacyWarning, feed_url_privacy_warning};
|
||||
|
||||
/// Current card, tombstone and overlay wire version.
|
||||
pub const PODCAST_DHT_PROTOCOL_VERSION: u16 = 1;
|
||||
/// Signing domain of a [`PodcastCard`]. Changing it invalidates every card.
|
||||
pub const PODCAST_CARD_SIGNING_DOMAIN: &str = "furumi-podcast-card:v1";
|
||||
/// Signing domain of a [`PodcastTombstone`].
|
||||
pub const PODCAST_TOMBSTONE_SIGNING_DOMAIN: &str = "furumi-podcast-card-tombstone:v1";
|
||||
/// BLAKE3 derivation domain of [`PodcastDhtKey`].
|
||||
pub const PODCAST_DHT_KEY_DOMAIN: &str = "furumi-podcast-dht:key:v1";
|
||||
|
||||
/// Maximum lifetime of a card: the main DHT's active-record TTL.
|
||||
pub const PODCAST_CARD_TTL: Duration = crate::record::ACTIVE_RECORD_TTL;
|
||||
/// Maximum lifetime of a tombstone: the main DHT's tombstone TTL. It exceeds
|
||||
/// [`PODCAST_CARD_TTL`], so a card issued before a tombstone always expires
|
||||
/// before the tombstone does.
|
||||
pub const PODCAST_TOMBSTONE_TTL: Duration = crate::record::TOMBSTONE_TTL;
|
||||
/// Tolerated clock skew for `issued_at_ms` in the future.
|
||||
pub const PODCAST_DHT_MAX_CLOCK_SKEW: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
/// Maximum characters of a card title.
|
||||
pub const MAX_CARD_TITLE_CHARS: usize = 300;
|
||||
/// Maximum characters of a card author.
|
||||
pub const MAX_CARD_AUTHOR_CHARS: usize = 300;
|
||||
/// Maximum characters of a card description.
|
||||
pub const MAX_CARD_DESCRIPTION_CHARS: usize = 1_000;
|
||||
/// Maximum bytes of a card language tag (BCP 47 shape: ASCII letters,
|
||||
/// digits and `-`).
|
||||
pub const MAX_CARD_LANGUAGE_BYTES: usize = 35;
|
||||
/// Maximum categories on a card.
|
||||
pub const MAX_CARD_CATEGORIES: usize = 8;
|
||||
/// Maximum characters of one category.
|
||||
pub const MAX_CARD_CATEGORY_CHARS: usize = 64;
|
||||
/// Maximum indexed search tokens of a card (title tokens, then author tokens).
|
||||
pub const MAX_CARD_INDEX_TOKENS: usize = 32;
|
||||
/// Maximum bytes of one indexed or query token. Longer tokens are not indexed.
|
||||
pub const MAX_INDEX_TOKEN_BYTES: usize = 64;
|
||||
/// Maximum DHT keys a record is stored under: the podcast-id key plus one
|
||||
/// key per indexed token.
|
||||
pub const MAX_RECORD_DHT_KEYS: usize = MAX_CARD_INDEX_TOKENS + 1;
|
||||
/// Maximum postcard-encoded size of one signed record.
|
||||
pub const MAX_PODCAST_RECORD_BYTES: usize = 24 * 1024;
|
||||
|
||||
const KEY_KIND_PODCAST: u8 = 1;
|
||||
const KEY_KIND_TOKEN: u8 = 2;
|
||||
|
||||
/// A 256-bit point in the podcast-discovery key space.
|
||||
///
|
||||
/// `BLAKE3("furumi-podcast-dht:key:v1" || 0x00 || network_id || kind || material)`
|
||||
/// with kind `0x01` and the podcast id string for the id key, or kind `0x02`
|
||||
/// and a normalized token for a text key.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub struct PodcastDhtKey([u8; 32]);
|
||||
|
||||
impl PodcastDhtKey {
|
||||
/// Key under which every card and tombstone of `podcast_id` is stored.
|
||||
pub fn podcast(network_id: &NetworkId, podcast_id: &PodcastId) -> Self {
|
||||
Self::derive(network_id, KEY_KIND_PODCAST, podcast_id.as_str())
|
||||
}
|
||||
|
||||
/// Key of one normalized search token (see [`card_index_tokens`]).
|
||||
pub fn token(network_id: &NetworkId, token: &str) -> Self {
|
||||
Self::derive(network_id, KEY_KIND_TOKEN, token)
|
||||
}
|
||||
|
||||
fn derive(network_id: &NetworkId, kind: u8, material: &str) -> Self {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
hasher.update(PODCAST_DHT_KEY_DOMAIN.as_bytes());
|
||||
hasher.update(&[0]);
|
||||
hasher.update(network_id.as_bytes());
|
||||
hasher.update(&[kind]);
|
||||
hasher.update(material.as_bytes());
|
||||
Self(*hasher.finalize().as_bytes())
|
||||
}
|
||||
|
||||
/// Creates a key from raw bytes received on the wire.
|
||||
pub fn from_bytes(bytes: [u8; 32]) -> Self {
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
/// Raw key bytes.
|
||||
pub fn as_bytes(&self) -> &[u8; 32] {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Indexed search tokens of a card: distinct [`normalize_name`] tokens of the
|
||||
/// title, then of the author, in order, skipping tokens longer than
|
||||
/// [`MAX_INDEX_TOKEN_BYTES`], at most [`MAX_CARD_INDEX_TOKENS`].
|
||||
pub fn card_index_tokens(title: &str, author: Option<&str>) -> Vec<String> {
|
||||
let mut seen = HashSet::new();
|
||||
let mut tokens = Vec::new();
|
||||
let sources = std::iter::once(title).chain(author);
|
||||
for source in sources {
|
||||
for token in tokenize(&normalize_name(source)) {
|
||||
if tokens.len() == MAX_CARD_INDEX_TOKENS {
|
||||
return tokens;
|
||||
}
|
||||
if token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone()) {
|
||||
tokens.push(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
tokens
|
||||
}
|
||||
|
||||
/// Unsigned public projection of a podcast prepared by its publisher.
|
||||
///
|
||||
/// It is built from the user's subscription ([`PodcastCardSpec::from_subscription`])
|
||||
/// and contains only publisher metadata: never progress, listening history,
|
||||
/// subscription times, operation ids, device names or episodes.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastCardSpec {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Canonical URL the feed is currently fetched from.
|
||||
pub feed_url: FeedUrl,
|
||||
/// URL `podcast_id` was derived from, when it differs from `feed_url`.
|
||||
pub id_feed_url: Option<FeedUrl>,
|
||||
/// Podcast title.
|
||||
pub title: String,
|
||||
/// Author.
|
||||
pub author: Option<String>,
|
||||
/// Plain-text description.
|
||||
pub description: Option<String>,
|
||||
/// Artwork URL.
|
||||
pub image_url: Option<String>,
|
||||
/// Publisher web page.
|
||||
pub link: Option<String>,
|
||||
/// Feed language tag, for example `en` or `pt-BR`.
|
||||
pub language: Option<String>,
|
||||
/// Feed categories.
|
||||
pub categories: Vec<String>,
|
||||
/// Number of episodes in the feed, when known.
|
||||
pub episode_count: Option<u32>,
|
||||
/// Publication time of the newest episode in unix milliseconds.
|
||||
pub latest_episode_at_ms: Option<i64>,
|
||||
}
|
||||
|
||||
impl PodcastCardSpec {
|
||||
/// Builds the bounded public projection of a subscription.
|
||||
///
|
||||
/// Only `podcast_id`, the feed URLs and the channel metadata are read.
|
||||
/// `id_feed_url` is the alias that hashes to `podcast_id`
|
||||
/// ([`crate::podcast::merge_feed_aliases`] always keeps it). Fails when no
|
||||
/// such URL is known. Callers may then set `language`, `categories`,
|
||||
/// `episode_count` and `latest_episode_at_ms` from their feed cache.
|
||||
pub fn from_subscription(subscription: &SubscriptionRecord) -> Result<Self> {
|
||||
let origin = subscription
|
||||
.feed_urls()
|
||||
.into_iter()
|
||||
.find(|url| PodcastId::from_feed_url(url) == subscription.podcast_id)
|
||||
.ok_or_else(|| {
|
||||
publication_error("the feed URL the podcast id was derived from is unknown")
|
||||
})?;
|
||||
let feed_url = subscription.podcast.feed_url.clone();
|
||||
let meta = &subscription.podcast;
|
||||
Ok(Self {
|
||||
podcast_id: subscription.podcast_id.clone(),
|
||||
id_feed_url: (origin != feed_url).then_some(origin),
|
||||
feed_url,
|
||||
title: meta.title.clone(),
|
||||
author: meta.author.clone(),
|
||||
description: meta.description.clone(),
|
||||
image_url: meta.image_url.clone(),
|
||||
link: meta.link.clone(),
|
||||
language: None,
|
||||
categories: Vec::new(),
|
||||
episode_count: None,
|
||||
latest_episode_at_ms: None,
|
||||
}
|
||||
.bounded())
|
||||
}
|
||||
|
||||
/// Returns a copy that fits the card bounds: text is truncated with
|
||||
/// [`bounded_text`], control characters are removed, invalid URLs and
|
||||
/// language tags are dropped, and categories are trimmed, deduplicated and
|
||||
/// bounded. An empty title becomes `"Untitled podcast"`.
|
||||
pub fn bounded(&self) -> Self {
|
||||
let title = clean_line(&self.title);
|
||||
let mut seen = HashSet::new();
|
||||
let categories = self
|
||||
.categories
|
||||
.iter()
|
||||
.map(|category| bounded_text(&clean_line(category), MAX_CARD_CATEGORY_CHARS))
|
||||
.filter(|category| !category.is_empty() && seen.insert(category.to_lowercase()))
|
||||
.take(MAX_CARD_CATEGORIES)
|
||||
.collect();
|
||||
Self {
|
||||
podcast_id: self.podcast_id.clone(),
|
||||
feed_url: self.feed_url.clone(),
|
||||
id_feed_url: self
|
||||
.id_feed_url
|
||||
.clone()
|
||||
.filter(|origin| *origin != self.feed_url),
|
||||
title: if title.is_empty() {
|
||||
"Untitled podcast".to_string()
|
||||
} else {
|
||||
bounded_text(&title, MAX_CARD_TITLE_CHARS)
|
||||
},
|
||||
author: bounded_line(self.author.as_deref(), MAX_CARD_AUTHOR_CHARS),
|
||||
description: self
|
||||
.description
|
||||
.as_deref()
|
||||
.map(|text| bounded_text(clean_block(text).trim(), MAX_CARD_DESCRIPTION_CHARS))
|
||||
.filter(|text| !text.is_empty()),
|
||||
image_url: self.image_url.clone().filter(|url| is_http_url(url)),
|
||||
link: self.link.clone().filter(|url| is_http_url(url)),
|
||||
language: self
|
||||
.language
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|language| valid_language(language))
|
||||
.map(str::to_string),
|
||||
categories,
|
||||
episode_count: self.episode_count,
|
||||
latest_episode_at_ms: self.latest_episode_at_ms.filter(|at| *at >= 0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the card bounds without bounding anything.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
let id_url = self.id_feed_url.as_ref().unwrap_or(&self.feed_url);
|
||||
if PodcastId::from_feed_url(id_url) != self.podcast_id {
|
||||
return Err(invalid("podcast id does not match its feed URL"));
|
||||
}
|
||||
if self.id_feed_url.as_ref() == Some(&self.feed_url) {
|
||||
return Err(invalid("id_feed_url must differ from feed_url"));
|
||||
}
|
||||
check_line("title", &self.title, MAX_CARD_TITLE_CHARS, true)?;
|
||||
if let Some(author) = &self.author {
|
||||
check_line("author", author, MAX_CARD_AUTHOR_CHARS, false)?;
|
||||
}
|
||||
if let Some(description) = &self.description {
|
||||
check_chars("description", description, MAX_CARD_DESCRIPTION_CHARS)?;
|
||||
if description
|
||||
.chars()
|
||||
.any(|c| c.is_control() && c != '\n' && c != '\t')
|
||||
{
|
||||
return Err(invalid("description contains control characters"));
|
||||
}
|
||||
}
|
||||
for url in [&self.image_url, &self.link].into_iter().flatten() {
|
||||
if !is_http_url(url) {
|
||||
return Err(invalid("card URLs must be absolute http(s) URLs"));
|
||||
}
|
||||
}
|
||||
if self
|
||||
.language
|
||||
.as_deref()
|
||||
.is_some_and(|language| !valid_language(language))
|
||||
{
|
||||
return Err(invalid("invalid language tag"));
|
||||
}
|
||||
if self.categories.len() > MAX_CARD_CATEGORIES {
|
||||
return Err(invalid("too many categories"));
|
||||
}
|
||||
for category in &self.categories {
|
||||
check_line("category", category, MAX_CARD_CATEGORY_CHARS, true)?;
|
||||
}
|
||||
if self.latest_episode_at_ms.is_some_and(|at| at < 0) {
|
||||
return Err(invalid("negative latest_episode_at_ms"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Advisory privacy warnings for both feed URLs of the card (see
|
||||
/// [`feed_url_privacy_warning`]). The UI must ask before publishing a
|
||||
/// card with warnings.
|
||||
pub fn privacy_warnings(&self) -> Vec<FeedUrlPrivacyWarning> {
|
||||
std::iter::once(&self.feed_url)
|
||||
.chain(self.id_feed_url.as_ref())
|
||||
.filter_map(feed_url_privacy_warning)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Indexed search tokens of this card.
|
||||
pub fn index_tokens(&self) -> Vec<String> {
|
||||
card_index_tokens(&self.title, self.author.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed public podcast card: one publisher device's projection of one
|
||||
/// podcast. See `PODCASTS.md`, "Public discovery (podcast_dht)".
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastCard {
|
||||
/// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||
pub version: u16,
|
||||
/// Network scope; prevents replay into another federation.
|
||||
pub network_id: NetworkId,
|
||||
/// Publisher device (its persistent transport identity); signs the card.
|
||||
pub publisher: EndpointId,
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Canonical URL the feed is currently fetched from.
|
||||
pub feed_url: FeedUrl,
|
||||
/// URL `podcast_id` was derived from, when it differs from `feed_url`.
|
||||
pub id_feed_url: Option<FeedUrl>,
|
||||
/// Podcast title.
|
||||
pub title: String,
|
||||
/// Author.
|
||||
pub author: Option<String>,
|
||||
/// Plain-text description.
|
||||
pub description: Option<String>,
|
||||
/// Artwork URL.
|
||||
pub image_url: Option<String>,
|
||||
/// Publisher web page.
|
||||
pub link: Option<String>,
|
||||
/// Feed language tag.
|
||||
pub language: Option<String>,
|
||||
/// Feed categories.
|
||||
pub categories: Vec<String>,
|
||||
/// Number of episodes in the feed, when known.
|
||||
pub episode_count: Option<u32>,
|
||||
/// Publication time of the newest episode in unix milliseconds.
|
||||
pub latest_episode_at_ms: Option<i64>,
|
||||
/// Monotonic per `(podcast_id, publisher)` revision.
|
||||
pub revision: u64,
|
||||
/// Unix time at which this version was signed.
|
||||
pub issued_at_ms: u64,
|
||||
/// Unix time after which the card must be discarded.
|
||||
pub expires_at_ms: u64,
|
||||
}
|
||||
|
||||
impl PodcastCard {
|
||||
/// Builds a card from a spec. `expires_at_ms` is
|
||||
/// `issued_at_ms + PODCAST_CARD_TTL`.
|
||||
pub fn from_spec(
|
||||
spec: PodcastCardSpec,
|
||||
network_id: NetworkId,
|
||||
publisher: EndpointId,
|
||||
revision: u64,
|
||||
issued_at_ms: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
network_id,
|
||||
publisher,
|
||||
podcast_id: spec.podcast_id,
|
||||
feed_url: spec.feed_url,
|
||||
id_feed_url: spec.id_feed_url,
|
||||
title: spec.title,
|
||||
author: spec.author,
|
||||
description: spec.description,
|
||||
image_url: spec.image_url,
|
||||
link: spec.link,
|
||||
language: spec.language,
|
||||
categories: spec.categories,
|
||||
episode_count: spec.episode_count,
|
||||
latest_episode_at_ms: spec.latest_episode_at_ms,
|
||||
revision,
|
||||
issued_at_ms,
|
||||
expires_at_ms: issued_at_ms.saturating_add(PODCAST_CARD_TTL.as_millis() as u64),
|
||||
}
|
||||
}
|
||||
|
||||
/// The publisher-provided part of the card.
|
||||
pub fn spec(&self) -> PodcastCardSpec {
|
||||
PodcastCardSpec {
|
||||
podcast_id: self.podcast_id.clone(),
|
||||
feed_url: self.feed_url.clone(),
|
||||
id_feed_url: self.id_feed_url.clone(),
|
||||
title: self.title.clone(),
|
||||
author: self.author.clone(),
|
||||
description: self.description.clone(),
|
||||
image_url: self.image_url.clone(),
|
||||
link: self.link.clone(),
|
||||
language: self.language.clone(),
|
||||
categories: self.categories.clone(),
|
||||
episode_count: self.episode_count,
|
||||
latest_episode_at_ms: self.latest_episode_at_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// Indexed search tokens.
|
||||
pub fn index_tokens(&self) -> Vec<String> {
|
||||
card_index_tokens(&self.title, self.author.as_deref())
|
||||
}
|
||||
|
||||
/// The URL `podcast_id` is derived from (`id_feed_url`, or `feed_url`
|
||||
/// when the feed has not moved). A valid card's id hashes from it, so it
|
||||
/// is the only URL of the card bound to the podcast.
|
||||
pub fn origin_feed_url(&self) -> &FeedUrl {
|
||||
self.id_feed_url.as_ref().unwrap_or(&self.feed_url)
|
||||
}
|
||||
|
||||
/// Whether `feed_url` is the URL the id is derived from. Otherwise
|
||||
/// `feed_url` is an unverified claim of the publisher that the feed moved.
|
||||
pub fn feed_url_is_origin(&self) -> bool {
|
||||
self.id_feed_url.is_none()
|
||||
}
|
||||
|
||||
/// Canonical signing input (see `PODCASTS.md`).
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN);
|
||||
out.u16(self.version);
|
||||
out.raw(self.network_id.as_bytes());
|
||||
out.raw(self.publisher.as_bytes());
|
||||
out.str(self.podcast_id.as_str());
|
||||
out.str(self.feed_url.as_str());
|
||||
out.opt_str(self.id_feed_url.as_ref().map(FeedUrl::as_str));
|
||||
out.str(&self.title);
|
||||
out.opt_str(self.author.as_deref());
|
||||
out.opt_str(self.description.as_deref());
|
||||
out.opt_str(self.image_url.as_deref());
|
||||
out.opt_str(self.link.as_deref());
|
||||
out.opt_str(self.language.as_deref());
|
||||
out.u32(self.categories.len() as u32);
|
||||
for category in &self.categories {
|
||||
out.str(category);
|
||||
}
|
||||
match self.episode_count {
|
||||
Some(count) => {
|
||||
out.u8(1);
|
||||
out.u32(count);
|
||||
}
|
||||
None => out.u8(0),
|
||||
}
|
||||
match self.latest_episode_at_ms {
|
||||
Some(at) => {
|
||||
out.u8(1);
|
||||
out.i64(at);
|
||||
}
|
||||
None => out.u8(0),
|
||||
}
|
||||
out.u64(self.revision);
|
||||
out.u64(self.issued_at_ms);
|
||||
out.u64(self.expires_at_ms);
|
||||
out.finish()
|
||||
}
|
||||
|
||||
fn validate(&self, expected_network: &NetworkId) -> Result<()> {
|
||||
check_header(self.version, &self.network_id, expected_network)?;
|
||||
self.spec().validate()?;
|
||||
check_lifetime(
|
||||
self.revision,
|
||||
self.issued_at_ms,
|
||||
self.expires_at_ms,
|
||||
PODCAST_CARD_TTL,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed retraction of one publisher's card of one podcast.
|
||||
///
|
||||
/// It supersedes every card of the same `(podcast_id, publisher)` with a
|
||||
/// revision lower than or equal to its own and is retained until it expires.
|
||||
/// `index_keys` lists the text keys of the retracted cards so the tombstone
|
||||
/// can be stored next to them; it carries hashes, not the tokens.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastTombstone {
|
||||
/// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||
pub version: u16,
|
||||
/// Network scope.
|
||||
pub network_id: NetworkId,
|
||||
/// Publisher device; signs the tombstone.
|
||||
pub publisher: EndpointId,
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Revision; greater than that of every retracted card.
|
||||
pub revision: u64,
|
||||
/// Unix time at which the tombstone was signed.
|
||||
pub issued_at_ms: u64,
|
||||
/// Unix time after which the tombstone may be discarded.
|
||||
pub expires_at_ms: u64,
|
||||
/// Sorted, distinct text keys besides the podcast-id key, at most
|
||||
/// [`MAX_CARD_INDEX_TOKENS`].
|
||||
pub index_keys: Vec<PodcastDhtKey>,
|
||||
}
|
||||
|
||||
impl PodcastTombstone {
|
||||
/// Builds a tombstone. `expires_at_ms` is `issued_at_ms + PODCAST_TOMBSTONE_TTL`;
|
||||
/// `index_keys` is sorted, deduplicated and stripped of the podcast-id key.
|
||||
pub fn new(
|
||||
network_id: NetworkId,
|
||||
publisher: EndpointId,
|
||||
podcast_id: PodcastId,
|
||||
revision: u64,
|
||||
issued_at_ms: u64,
|
||||
index_keys: impl IntoIterator<Item = PodcastDhtKey>,
|
||||
) -> Self {
|
||||
let id_key = PodcastDhtKey::podcast(&network_id, &podcast_id);
|
||||
let mut index_keys: Vec<_> = index_keys
|
||||
.into_iter()
|
||||
.filter(|key| *key != id_key)
|
||||
.collect();
|
||||
index_keys.sort();
|
||||
index_keys.dedup();
|
||||
index_keys.truncate(MAX_CARD_INDEX_TOKENS);
|
||||
Self {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
network_id,
|
||||
publisher,
|
||||
podcast_id,
|
||||
revision,
|
||||
issued_at_ms,
|
||||
expires_at_ms: issued_at_ms.saturating_add(PODCAST_TOMBSTONE_TTL.as_millis() as u64),
|
||||
index_keys,
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical signing input (see `PODCASTS.md`).
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
let mut out = Canonical::new(PODCAST_TOMBSTONE_SIGNING_DOMAIN);
|
||||
out.u16(self.version);
|
||||
out.raw(self.network_id.as_bytes());
|
||||
out.raw(self.publisher.as_bytes());
|
||||
out.str(self.podcast_id.as_str());
|
||||
out.u64(self.revision);
|
||||
out.u64(self.issued_at_ms);
|
||||
out.u64(self.expires_at_ms);
|
||||
out.u32(self.index_keys.len() as u32);
|
||||
for key in &self.index_keys {
|
||||
out.raw(key.as_bytes());
|
||||
}
|
||||
out.finish()
|
||||
}
|
||||
|
||||
fn validate(&self, expected_network: &NetworkId) -> Result<()> {
|
||||
check_header(self.version, &self.network_id, expected_network)?;
|
||||
if self.index_keys.len() > MAX_CARD_INDEX_TOKENS {
|
||||
return Err(invalid("tombstone has too many keys"));
|
||||
}
|
||||
let id_key = PodcastDhtKey::podcast(&self.network_id, &self.podcast_id);
|
||||
if self.index_keys.contains(&id_key)
|
||||
|| self.index_keys.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
{
|
||||
return Err(invalid("tombstone keys are not canonical"));
|
||||
}
|
||||
check_lifetime(
|
||||
self.revision,
|
||||
self.issued_at_ms,
|
||||
self.expires_at_ms,
|
||||
PODCAST_TOMBSTONE_TTL,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Body of a signed podcast-discovery record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum PodcastRecordBody {
|
||||
/// An active public card.
|
||||
Card(Box<PodcastCard>),
|
||||
/// A retraction.
|
||||
Tombstone(PodcastTombstone),
|
||||
}
|
||||
|
||||
/// End-to-end signed podcast-discovery record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SignedPodcastRecord {
|
||||
/// Signed body.
|
||||
pub body: PodcastRecordBody,
|
||||
/// Ed25519 signature by `publisher` over [`Self::signing_bytes`].
|
||||
pub signature: Signature,
|
||||
}
|
||||
|
||||
impl SignedPodcastRecord {
|
||||
/// Validates and signs a card.
|
||||
pub fn sign_card(card: PodcastCard, signer: impl FnOnce(&[u8]) -> Signature) -> Result<Self> {
|
||||
card.validate(&card.network_id)?;
|
||||
let signature = signer(&card.signing_bytes());
|
||||
let record = Self {
|
||||
body: PodcastRecordBody::Card(Box::new(card)),
|
||||
signature,
|
||||
};
|
||||
record.check_size()?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// Validates and signs a tombstone.
|
||||
pub fn sign_tombstone(
|
||||
tombstone: PodcastTombstone,
|
||||
signer: impl FnOnce(&[u8]) -> Signature,
|
||||
) -> Result<Self> {
|
||||
tombstone.validate(&tombstone.network_id)?;
|
||||
let signature = signer(&tombstone.signing_bytes());
|
||||
let record = Self {
|
||||
body: PodcastRecordBody::Tombstone(tombstone),
|
||||
signature,
|
||||
};
|
||||
record.check_size()?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// Canonical signing input of the body.
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.signing_bytes(),
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.signing_bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates bounds, network, lifetime shape, encoded size and the
|
||||
/// publisher's signature. Does not look at the clock; see
|
||||
/// [`Self::verify_at`].
|
||||
pub fn verify(&self, expected_network: &NetworkId) -> Result<()> {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.validate(expected_network)?,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.validate(expected_network)?,
|
||||
}
|
||||
self.check_size()?;
|
||||
self.publisher()
|
||||
.verify(&self.signing_bytes(), &self.signature)
|
||||
.map_err(|_| invalid("invalid podcast record signature"))
|
||||
}
|
||||
|
||||
/// [`Self::verify`] plus time checks: not issued more than
|
||||
/// [`PODCAST_DHT_MAX_CLOCK_SKEW`] in the future and not expired at `now_ms`.
|
||||
pub fn verify_at(&self, expected_network: &NetworkId, now_ms: u64) -> Result<()> {
|
||||
self.verify(expected_network)?;
|
||||
let future_limit = now_ms.saturating_add(PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64);
|
||||
if self.issued_at_ms() > future_limit || self.expires_at_ms() <= now_ms {
|
||||
return Err(invalid("podcast record is expired or from the future"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The card, when this is not a tombstone.
|
||||
pub fn card(&self) -> Option<&PodcastCard> {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => Some(card.as_ref()),
|
||||
PodcastRecordBody::Tombstone(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this is a tombstone.
|
||||
pub fn is_tombstone(&self) -> bool {
|
||||
matches!(self.body, PodcastRecordBody::Tombstone(_))
|
||||
}
|
||||
|
||||
/// Podcast.
|
||||
pub fn podcast_id(&self) -> &PodcastId {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => &card.podcast_id,
|
||||
PodcastRecordBody::Tombstone(tombstone) => &tombstone.podcast_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Publisher device.
|
||||
pub fn publisher(&self) -> EndpointId {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.publisher,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.publisher,
|
||||
}
|
||||
}
|
||||
|
||||
/// Network scope.
|
||||
pub fn network_id(&self) -> NetworkId {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.network_id,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.network_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Revision.
|
||||
pub fn revision(&self) -> u64 {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.revision,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.revision,
|
||||
}
|
||||
}
|
||||
|
||||
/// Signing time.
|
||||
pub fn issued_at_ms(&self) -> u64 {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.issued_at_ms,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.issued_at_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// Expiry time.
|
||||
pub fn expires_at_ms(&self) -> u64 {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.expires_at_ms,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.expires_at_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every DHT key this record belongs under: the podcast-id key, then the
|
||||
/// text keys (card tokens or the tombstone's `index_keys`).
|
||||
pub fn dht_keys(&self) -> Vec<PodcastDhtKey> {
|
||||
let network_id = self.network_id();
|
||||
let mut keys = vec![PodcastDhtKey::podcast(&network_id, self.podcast_id())];
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => keys.extend(
|
||||
card.index_tokens()
|
||||
.iter()
|
||||
.map(|token| PodcastDhtKey::token(&network_id, token)),
|
||||
),
|
||||
PodcastRecordBody::Tombstone(tombstone) => {
|
||||
keys.extend(tombstone.index_keys.iter().copied());
|
||||
}
|
||||
}
|
||||
keys
|
||||
}
|
||||
|
||||
/// Whether the record may be stored under `key`.
|
||||
pub fn belongs_under(&self, key: &PodcastDhtKey) -> bool {
|
||||
self.dht_keys().contains(key)
|
||||
}
|
||||
|
||||
/// postcard-encoded size.
|
||||
pub fn encoded_len(&self) -> usize {
|
||||
postcard::to_stdvec(self).map_or(usize::MAX, |bytes| bytes.len())
|
||||
}
|
||||
|
||||
fn check_size(&self) -> Result<()> {
|
||||
if self.encoded_len() > MAX_PODCAST_RECORD_BYTES {
|
||||
Err(invalid("podcast record is too large"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Total order of two records of the same `(podcast_id, publisher)`.
|
||||
///
|
||||
/// Greater wins: higher `revision`; at an equal revision a tombstone beats a
|
||||
/// card; then later `issued_at_ms` (a republish); then later
|
||||
/// `expires_at_ms`; then the lexicographically greater canonical signing
|
||||
/// input. Every node therefore keeps the same winner whatever the delivery
|
||||
/// order.
|
||||
pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord) -> Ordering {
|
||||
left.revision()
|
||||
.cmp(&right.revision())
|
||||
.then_with(|| left.is_tombstone().cmp(&right.is_tombstone()))
|
||||
.then_with(|| left.issued_at_ms().cmp(&right.issued_at_ms()))
|
||||
.then_with(|| left.expires_at_ms().cmp(&right.expires_at_ms()))
|
||||
.then_with(|| left.signing_bytes().cmp(&right.signing_bytes()))
|
||||
}
|
||||
|
||||
struct Canonical(Vec<u8>);
|
||||
|
||||
impl Canonical {
|
||||
fn new(domain: &str) -> Self {
|
||||
let mut bytes = Vec::with_capacity(512);
|
||||
bytes.extend_from_slice(domain.as_bytes());
|
||||
bytes.push(0);
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
fn raw(&mut self, bytes: &[u8]) {
|
||||
self.0.extend_from_slice(bytes);
|
||||
}
|
||||
|
||||
fn u8(&mut self, value: u8) {
|
||||
self.0.push(value);
|
||||
}
|
||||
|
||||
fn u16(&mut self, value: u16) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn u32(&mut self, value: u32) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn u64(&mut self, value: u64) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn i64(&mut self, value: i64) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn str(&mut self, value: &str) {
|
||||
self.u32(value.len() as u32);
|
||||
self.raw(value.as_bytes());
|
||||
}
|
||||
|
||||
fn opt_str(&mut self, value: Option<&str>) {
|
||||
match value {
|
||||
Some(value) => {
|
||||
self.u8(1);
|
||||
self.str(value);
|
||||
}
|
||||
None => self.u8(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(self) -> Vec<u8> {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
fn check_header(version: u16, network_id: &NetworkId, expected: &NetworkId) -> Result<()> {
|
||||
if version != PODCAST_DHT_PROTOCOL_VERSION {
|
||||
return Err(invalid("unsupported podcast record version"));
|
||||
}
|
||||
if network_id != expected {
|
||||
return Err(invalid("podcast record belongs to another network"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_lifetime(
|
||||
revision: u64,
|
||||
issued_at_ms: u64,
|
||||
expires_at_ms: u64,
|
||||
ttl: Duration,
|
||||
) -> Result<()> {
|
||||
if revision == 0
|
||||
|| issued_at_ms == 0
|
||||
|| expires_at_ms <= issued_at_ms
|
||||
|| expires_at_ms - issued_at_ms > ttl.as_millis() as u64
|
||||
{
|
||||
return Err(invalid("invalid podcast record revision or lifetime"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_chars(field: &'static str, value: &str, max_chars: usize) -> Result<()> {
|
||||
if value.len() > max_chars * 4 || value.chars().count() > max_chars {
|
||||
return Err(MusicDhtError::Protocol(format!("card {field} is too long")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_line(field: &'static str, value: &str, max_chars: usize, required: bool) -> Result<()> {
|
||||
if required && value.trim().is_empty() {
|
||||
return Err(MusicDhtError::Protocol(format!("card {field} is empty")));
|
||||
}
|
||||
check_chars(field, value, max_chars)?;
|
||||
if value.chars().any(char::is_control) {
|
||||
return Err(MusicDhtError::Protocol(format!(
|
||||
"card {field} contains control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn valid_language(language: &str) -> bool {
|
||||
!language.is_empty()
|
||||
&& language.len() <= MAX_CARD_LANGUAGE_BYTES
|
||||
&& language
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
}
|
||||
|
||||
fn clean_line(text: &str) -> String {
|
||||
text.split(char::is_control)
|
||||
.filter(|part| !part.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn clean_block(text: &str) -> String {
|
||||
text.chars()
|
||||
.filter(|c| !c.is_control() || *c == '\n' || *c == '\t')
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bounded_line(text: Option<&str>, max_chars: usize) -> Option<String> {
|
||||
text.map(clean_line)
|
||||
.filter(|text| !text.is_empty())
|
||||
.map(|text| bounded_text(&text, max_chars))
|
||||
}
|
||||
|
||||
fn invalid(reason: &str) -> MusicDhtError {
|
||||
MusicDhtError::Protocol(reason.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn publication_error(reason: &str) -> MusicDhtError {
|
||||
MusicDhtError::PodcastPublication(reason.to_string())
|
||||
}
|
||||
|
||||
// Statically bound the worst-case card so `MAX_PODCAST_RECORD_BYTES` is a
|
||||
// defensive limit, not the one users hit first.
|
||||
const _: () = assert!(
|
||||
MAX_CARD_TITLE_CHARS * 4
|
||||
+ MAX_CARD_AUTHOR_CHARS * 4
|
||||
+ MAX_CARD_DESCRIPTION_CHARS * 4
|
||||
+ MAX_CARD_CATEGORIES * MAX_CARD_CATEGORY_CHARS * 4
|
||||
+ MAX_CARD_LANGUAGE_BYTES
|
||||
+ 4 * MAX_URL_BYTES
|
||||
+ 1024
|
||||
<= MAX_PODCAST_RECORD_BYTES
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,339 @@
|
||||
//! Advisory detection of feed URLs that look private.
|
||||
//!
|
||||
//! Premium and member feeds usually carry a personal secret in the URL. A
|
||||
//! card publishes its feed URL verbatim, so the UI must ask the user before
|
||||
//! publishing a URL for which [`feed_url_privacy_warning`] returns a warning.
|
||||
//! The heuristics are deliberately broad and can produce false positives
|
||||
//! (and miss secrets); they never block publication by themselves.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::podcast::FeedUrl;
|
||||
|
||||
/// Query-parameter name words that usually carry a credential. A name is
|
||||
/// split into lowercase words at non-alphanumeric characters and camelCase
|
||||
/// boundaries; any listed word triggers a warning.
|
||||
pub const SECRET_QUERY_WORDS: &[&str] = &[
|
||||
"token",
|
||||
"key",
|
||||
"auth",
|
||||
"secret",
|
||||
"sig",
|
||||
"signature",
|
||||
"uid",
|
||||
"user",
|
||||
"pass",
|
||||
"password",
|
||||
"passwd",
|
||||
"pwd",
|
||||
"session",
|
||||
"sess",
|
||||
"sid",
|
||||
"access",
|
||||
"private",
|
||||
"credential",
|
||||
"credentials",
|
||||
"apikey",
|
||||
"jwt",
|
||||
"hmac",
|
||||
];
|
||||
|
||||
/// Substrings that trigger a warning anywhere in a query-parameter name,
|
||||
/// for names such as `apitoken` or `privatefeed` that do not split.
|
||||
const SECRET_QUERY_SUBSTRINGS: &[&str] = &[
|
||||
"token",
|
||||
"secret",
|
||||
"signature",
|
||||
"password",
|
||||
"passwd",
|
||||
"session",
|
||||
"private",
|
||||
"credential",
|
||||
"apikey",
|
||||
];
|
||||
|
||||
/// Hosts (and their subdomains) known to serve personal member feeds.
|
||||
pub const PRIVATE_FEED_HOSTS: &[&str] = &[
|
||||
"patreon.com",
|
||||
"supercast.com",
|
||||
"supportingcast.fm",
|
||||
"glow.fm",
|
||||
];
|
||||
|
||||
/// Minimum length of a query value considered a possible secret.
|
||||
const MIN_SECRET_VALUE_CHARS: usize = 16;
|
||||
/// Minimum length of a path segment considered a possible secret.
|
||||
const MIN_SECRET_SEGMENT_CHARS: usize = 32;
|
||||
/// Minimum Shannon entropy, in bits per character, of a possible secret.
|
||||
const MIN_SECRET_ENTROPY_BITS: f64 = 3.0;
|
||||
|
||||
/// Why a feed URL may contain a personal secret.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "reason", rename_all = "snake_case")]
|
||||
#[non_exhaustive]
|
||||
pub enum FeedUrlPrivacyWarning {
|
||||
/// A query parameter whose name suggests a credential.
|
||||
SecretQueryParameter {
|
||||
/// Parameter name as it appears in the URL.
|
||||
name: String,
|
||||
},
|
||||
/// The host is known to serve personal member feeds.
|
||||
PrivateFeedHost {
|
||||
/// Lowercase host.
|
||||
host: String,
|
||||
},
|
||||
/// A long, random-looking query value.
|
||||
HighEntropyQueryValue {
|
||||
/// Name of the parameter carrying the value.
|
||||
name: String,
|
||||
},
|
||||
/// A long, random-looking path segment.
|
||||
HighEntropyPathSegment {
|
||||
/// Zero-based index of the segment after the host.
|
||||
index: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl fmt::Display for FeedUrlPrivacyWarning {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::SecretQueryParameter { name } => write!(
|
||||
f,
|
||||
"the feed URL has a query parameter \"{name}\" that looks like a credential"
|
||||
),
|
||||
Self::PrivateFeedHost { host } => {
|
||||
write!(f, "{host} usually serves personal member feeds")
|
||||
}
|
||||
Self::HighEntropyQueryValue { name } => write!(
|
||||
f,
|
||||
"the value of the query parameter \"{name}\" looks like a secret token"
|
||||
),
|
||||
Self::HighEntropyPathSegment { index } => write!(
|
||||
f,
|
||||
"path segment {} of the feed URL looks like a secret token",
|
||||
index + 1
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns why `url` may contain a personal secret, or `None`.
|
||||
///
|
||||
/// Advisory only (see the module documentation). Checks, in order: query
|
||||
/// parameter names ([`SECRET_QUERY_WORDS`]), known member-feed hosts
|
||||
/// ([`PRIVATE_FEED_HOSTS`]), long high-entropy query values, and long
|
||||
/// high-entropy path segments.
|
||||
pub fn feed_url_privacy_warning(url: &FeedUrl) -> Option<FeedUrlPrivacyWarning> {
|
||||
let value = url.as_str();
|
||||
let rest = value.split_once("://").map_or(value, |(_, rest)| rest);
|
||||
let (authority, path_and_query) = match rest.find('/') {
|
||||
Some(index) => rest.split_at(index),
|
||||
None => (rest, "/"),
|
||||
};
|
||||
let host = if authority.starts_with('[') {
|
||||
authority
|
||||
.find(']')
|
||||
.map_or(authority, |end| &authority[..=end])
|
||||
} else {
|
||||
authority.split(':').next().unwrap_or(authority)
|
||||
};
|
||||
let (path, query) = path_and_query
|
||||
.split_once('?')
|
||||
.unwrap_or((path_and_query, ""));
|
||||
let parameters: Vec<(&str, &str)> = query
|
||||
.split('&')
|
||||
.filter(|pair| !pair.is_empty())
|
||||
.map(|pair| pair.split_once('=').unwrap_or((pair, "")))
|
||||
.collect();
|
||||
|
||||
if let Some((name, _)) = parameters
|
||||
.iter()
|
||||
.find(|(name, _)| is_secret_parameter_name(name))
|
||||
{
|
||||
return Some(FeedUrlPrivacyWarning::SecretQueryParameter {
|
||||
name: (*name).to_string(),
|
||||
});
|
||||
}
|
||||
if PRIVATE_FEED_HOSTS
|
||||
.iter()
|
||||
.any(|known| host == *known || host.ends_with(&format!(".{known}")))
|
||||
{
|
||||
return Some(FeedUrlPrivacyWarning::PrivateFeedHost {
|
||||
host: host.to_string(),
|
||||
});
|
||||
}
|
||||
if let Some((name, _)) = parameters
|
||||
.iter()
|
||||
.find(|(_, value)| looks_random(value, MIN_SECRET_VALUE_CHARS))
|
||||
{
|
||||
return Some(FeedUrlPrivacyWarning::HighEntropyQueryValue {
|
||||
name: (*name).to_string(),
|
||||
});
|
||||
}
|
||||
path.split('/')
|
||||
.filter(|segment| !segment.is_empty())
|
||||
.position(|segment| {
|
||||
let stem = segment.rsplit_once('.').map_or(segment, |(stem, _)| stem);
|
||||
looks_random(stem, MIN_SECRET_SEGMENT_CHARS)
|
||||
})
|
||||
.map(|index| FeedUrlPrivacyWarning::HighEntropyPathSegment { index })
|
||||
}
|
||||
|
||||
fn is_secret_parameter_name(name: &str) -> bool {
|
||||
let words = split_words(name);
|
||||
if words
|
||||
.iter()
|
||||
.any(|word| SECRET_QUERY_WORDS.contains(&word.as_str()))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let compact: String = name
|
||||
.chars()
|
||||
.filter(char::is_ascii_alphanumeric)
|
||||
.collect::<String>()
|
||||
.to_ascii_lowercase();
|
||||
SECRET_QUERY_SUBSTRINGS
|
||||
.iter()
|
||||
.any(|needle| compact.contains(needle))
|
||||
}
|
||||
|
||||
/// Splits at non-alphanumeric characters and lower-to-upper case changes.
|
||||
fn split_words(name: &str) -> Vec<String> {
|
||||
let mut words = Vec::new();
|
||||
let mut current = String::new();
|
||||
let mut previous_lower = false;
|
||||
for c in name.chars() {
|
||||
if !c.is_ascii_alphanumeric() {
|
||||
if !current.is_empty() {
|
||||
words.push(std::mem::take(&mut current));
|
||||
}
|
||||
previous_lower = false;
|
||||
continue;
|
||||
}
|
||||
if c.is_ascii_uppercase() && previous_lower && !current.is_empty() {
|
||||
words.push(std::mem::take(&mut current));
|
||||
}
|
||||
previous_lower = c.is_ascii_lowercase() || c.is_ascii_digit();
|
||||
current.push(c.to_ascii_lowercase());
|
||||
}
|
||||
if !current.is_empty() {
|
||||
words.push(current);
|
||||
}
|
||||
words
|
||||
}
|
||||
|
||||
/// A long token-alphabet string mixing letters and digits (or both letter
|
||||
/// cases) with high per-character entropy.
|
||||
fn looks_random(value: &str, min_chars: usize) -> bool {
|
||||
if value.chars().count() < min_chars
|
||||
|| !value
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || "-_.~%=+".contains(c))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let has_digit = value.chars().any(|c| c.is_ascii_digit());
|
||||
let has_lower = value.chars().any(|c| c.is_ascii_lowercase());
|
||||
let has_upper = value.chars().any(|c| c.is_ascii_uppercase());
|
||||
let classes = [has_digit, has_lower, has_upper]
|
||||
.into_iter()
|
||||
.filter(|present| *present)
|
||||
.count();
|
||||
classes >= 2 && shannon_entropy(value) >= MIN_SECRET_ENTROPY_BITS
|
||||
}
|
||||
|
||||
fn shannon_entropy(value: &str) -> f64 {
|
||||
let mut counts: HashMap<char, usize> = HashMap::new();
|
||||
for c in value.chars() {
|
||||
*counts.entry(c).or_default() += 1;
|
||||
}
|
||||
let total = value.chars().count() as f64;
|
||||
counts
|
||||
.values()
|
||||
.map(|count| {
|
||||
let p = *count as f64 / total;
|
||||
-p * p.log2()
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn warn(url: &str) -> Option<FeedUrlPrivacyWarning> {
|
||||
feed_url_privacy_warning(&FeedUrl::parse(url).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_feeds_have_no_warning() {
|
||||
for url in [
|
||||
"https://feeds.example.com/podcast.xml",
|
||||
"https://example.com/feed?format=rss&page=2",
|
||||
"https://anchor.example/s/abc123de/podcast/rss",
|
||||
"https://feeds.example.com/12345/rss",
|
||||
"https://example.com/feed?author=jane",
|
||||
"https://example.com:8443/shows/my-great-show/feed.xml",
|
||||
] {
|
||||
assert_eq!(warn(url), None, "{url}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_query_names_warn() {
|
||||
for (url, name) in [
|
||||
("https://example.com/feed?token=abc", "token"),
|
||||
("https://example.com/feed?auth_token=abc", "auth_token"),
|
||||
(
|
||||
"https://example.com/feed?X-Amz-Signature=abc",
|
||||
"X-Amz-Signature",
|
||||
),
|
||||
("https://example.com/feed?accessKey=abc", "accessKey"),
|
||||
("https://example.com/feed?a=1&uid=42", "uid"),
|
||||
("https://example.com/feed?apitoken=x", "apitoken"),
|
||||
("https://example.com/feed?PRIVATE=1", "PRIVATE"),
|
||||
("https://example.com/feed?session_id=1", "session_id"),
|
||||
] {
|
||||
assert_eq!(
|
||||
warn(url),
|
||||
Some(FeedUrlPrivacyWarning::SecretQueryParameter { name: name.into() }),
|
||||
"{url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_member_hosts_warn() {
|
||||
assert_eq!(
|
||||
warn("https://www.patreon.com/rss/show"),
|
||||
Some(FeedUrlPrivacyWarning::PrivateFeedHost {
|
||||
host: "www.patreon.com".into()
|
||||
})
|
||||
);
|
||||
assert_eq!(warn("https://notpatreon.com/rss"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn high_entropy_values_and_segments_warn() {
|
||||
assert_eq!(
|
||||
warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb"),
|
||||
Some(FeedUrlPrivacyWarning::HighEntropyQueryValue { name: "c".into() })
|
||||
);
|
||||
assert_eq!(warn("https://example.com/feed?c=aaaaaaaaaaaaaaaa1"), None);
|
||||
assert_eq!(
|
||||
warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB4sYc1HdWe9GuJa3F.rss"),
|
||||
Some(FeedUrlPrivacyWarning::HighEntropyPathSegment { index: 1 })
|
||||
);
|
||||
// Below the segment length: an ordinary short id.
|
||||
assert_eq!(warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB.rss"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warnings_render_without_the_secret() {
|
||||
let warning = warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb").unwrap();
|
||||
assert!(!warning.to_string().contains("Zx81"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
//! Podcast search hits and their merge with music search results.
|
||||
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::fmt;
|
||||
|
||||
use federation_net::EndpointId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::normalization::{normalize_name, tokenize};
|
||||
use crate::podcast::{FeedUrl, PodcastId};
|
||||
use crate::record::{ItemKind, LibraryItem};
|
||||
|
||||
use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records};
|
||||
|
||||
/// Maximum distinct query tokens used by a podcast text search.
|
||||
pub const MAX_PODCAST_QUERY_TOKENS: usize = 8;
|
||||
/// Maximum publishers reported per hit and per claimed feed URL.
|
||||
pub const MAX_HIT_PUBLISHERS: usize = 64;
|
||||
/// Maximum claimed moved feed URLs reported per hit.
|
||||
pub const MAX_HIT_FEED_URL_CLAIMS: usize = 8;
|
||||
|
||||
/// Kind of a federated search hit.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum SearchHitKind {
|
||||
/// A musical artist.
|
||||
Artist,
|
||||
/// A release.
|
||||
Release,
|
||||
/// A track.
|
||||
Track,
|
||||
/// A public podcast card.
|
||||
Podcast,
|
||||
}
|
||||
|
||||
impl SearchHitKind {
|
||||
/// Stable lowercase name.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Artist => "artist",
|
||||
Self::Release => "release",
|
||||
Self::Track => "track",
|
||||
Self::Podcast => "podcast",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for SearchHitKind {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ItemKind> for SearchHitKind {
|
||||
fn from(kind: ItemKind) -> Self {
|
||||
match kind {
|
||||
ItemKind::Artist => Self::Artist,
|
||||
ItemKind::Release => Self::Release,
|
||||
ItemKind::Track => Self::Track,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One public podcast found in the discovery overlay.
|
||||
///
|
||||
/// Cards of the same podcast published by several devices are merged by
|
||||
/// `podcast_id`. The hit carries no personal state of any publisher.
|
||||
///
|
||||
/// Only [`Self::feed_url`] is bound to the podcast: the id is its hash. Every
|
||||
/// other field is a claim of some publisher device, and device identities are
|
||||
/// free, so neither the metadata nor the number of publishers is a trust
|
||||
/// signal.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastSearchHit {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// The URL `podcast_id` is derived from
|
||||
/// (`PodcastId::from_feed_url(feed_url) == podcast_id`). Subscribe with
|
||||
/// this URL; follow the feed's own HTTP redirects and
|
||||
/// `itunes:new-feed-url` from there.
|
||||
pub feed_url: FeedUrl,
|
||||
/// Representative card for display: a card whose `feed_url` is the
|
||||
/// origin URL when one exists, then the title most publishers agree on,
|
||||
/// then the smallest publisher key. Its metadata is a publisher's claim.
|
||||
pub card: PodcastCard,
|
||||
/// Sorted distinct publisher devices with a current valid card, at most
|
||||
/// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers:
|
||||
/// one user may publish from several devices, and anyone can mint
|
||||
/// devices.
|
||||
pub publishers: Vec<EndpointId>,
|
||||
/// Feed URLs other than [`Self::feed_url`] that publishers claim the
|
||||
/// podcast moved to, most claimed first, at most
|
||||
/// [`MAX_HIT_FEED_URL_CLAIMS`]. Unverified: never fetch one as this
|
||||
/// podcast or record it as an alias unless the feed at
|
||||
/// [`Self::feed_url`] itself redirects there.
|
||||
pub claimed_feed_urls: Vec<ClaimedFeedUrl>,
|
||||
}
|
||||
|
||||
impl PodcastSearchHit {
|
||||
/// Always [`SearchHitKind::Podcast`].
|
||||
pub fn kind(&self) -> SearchHitKind {
|
||||
SearchHitKind::Podcast
|
||||
}
|
||||
}
|
||||
|
||||
/// An unverified moved-feed URL claimed by some publishers of a podcast.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ClaimedFeedUrl {
|
||||
/// Claimed current feed URL.
|
||||
pub feed_url: FeedUrl,
|
||||
/// Sorted distinct publisher devices claiming it, at most
|
||||
/// [`MAX_HIT_PUBLISHERS`].
|
||||
pub publishers: Vec<EndpointId>,
|
||||
}
|
||||
|
||||
/// A music or podcast result of a federated search.
|
||||
///
|
||||
/// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use
|
||||
/// [`FederatedSearchHit::kind`] for the fine-grained kind.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", content = "hit", rename_all = "snake_case")]
|
||||
pub enum FederatedSearchHit {
|
||||
/// A music-DHT record (artist, release or track).
|
||||
Music(LibraryItem),
|
||||
/// A public podcast.
|
||||
Podcast(PodcastSearchHit),
|
||||
}
|
||||
|
||||
impl FederatedSearchHit {
|
||||
/// Kind of the hit: `artist`, `release`, `track` or `podcast`.
|
||||
pub fn kind(&self) -> SearchHitKind {
|
||||
match self {
|
||||
Self::Music(item) => item.kind.into(),
|
||||
Self::Podcast(_) => SearchHitKind::Podcast,
|
||||
}
|
||||
}
|
||||
|
||||
/// Display name: the item name or the podcast title.
|
||||
pub fn title(&self) -> &str {
|
||||
match self {
|
||||
Self::Music(item) => &item.name,
|
||||
Self::Podcast(hit) => &hit.card.title,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges music results (as returned by `MusicDhtService::search_network`)
|
||||
/// with podcast hits for the same query.
|
||||
///
|
||||
/// Results whose normalized name equals the normalized query come first;
|
||||
/// within each tier music keeps its order and precedes podcasts, which keep
|
||||
/// theirs. Existing music consumers that only read `SearchOutcome` are
|
||||
/// unaffected.
|
||||
pub fn merge_search_results(
|
||||
query: &str,
|
||||
music: Vec<LibraryItem>,
|
||||
podcasts: Vec<PodcastSearchHit>,
|
||||
) -> Vec<FederatedSearchHit> {
|
||||
let normalized = normalize_name(query);
|
||||
let mut exact = Vec::new();
|
||||
let mut rest = Vec::new();
|
||||
let hits = music
|
||||
.into_iter()
|
||||
.map(FederatedSearchHit::Music)
|
||||
.chain(podcasts.into_iter().map(FederatedSearchHit::Podcast));
|
||||
for hit in hits {
|
||||
let is_exact = !normalized.is_empty() && normalize_name(hit.title()) == normalized;
|
||||
if is_exact {
|
||||
exact.push(hit);
|
||||
} else {
|
||||
rest.push(hit);
|
||||
}
|
||||
}
|
||||
exact.extend(rest);
|
||||
exact
|
||||
}
|
||||
|
||||
/// Distinct normalized query tokens, at most [`MAX_PODCAST_QUERY_TOKENS`].
|
||||
pub fn podcast_query_tokens(query: &str) -> Vec<String> {
|
||||
let mut seen = HashSet::new();
|
||||
tokenize(&normalize_name(query))
|
||||
.into_iter()
|
||||
.filter(|token| token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone()))
|
||||
.take(MAX_PODCAST_QUERY_TOKENS)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Keeps the winning record per `(podcast_id, publisher)` and groups the
|
||||
/// cards that are still active by podcast. Records must be verified.
|
||||
pub(crate) fn aggregate_hits(
|
||||
records: impl IntoIterator<Item = SignedPodcastRecord>,
|
||||
) -> Vec<PodcastSearchHit> {
|
||||
let mut winners: HashMap<(PodcastId, EndpointId), SignedPodcastRecord> = HashMap::new();
|
||||
for record in records {
|
||||
let owner = (record.podcast_id().clone(), record.publisher());
|
||||
match winners.get(&owner) {
|
||||
Some(current) if compare_records(current, &record).is_ge() => {}
|
||||
_ => {
|
||||
winners.insert(owner, record);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut by_podcast: BTreeMap<PodcastId, Vec<PodcastCard>> = BTreeMap::new();
|
||||
for record in winners.into_values() {
|
||||
if let Some(card) = record.card() {
|
||||
by_podcast
|
||||
.entry(card.podcast_id.clone())
|
||||
.or_default()
|
||||
.push(card.clone());
|
||||
}
|
||||
}
|
||||
by_podcast
|
||||
.into_iter()
|
||||
.map(|(podcast_id, cards)| {
|
||||
let mut title_votes: HashMap<String, usize> = HashMap::new();
|
||||
for card in &cards {
|
||||
*title_votes.entry(normalize_name(&card.title)).or_default() += 1;
|
||||
}
|
||||
// Publisher-controlled times and revisions never break ties.
|
||||
let card = cards
|
||||
.iter()
|
||||
.max_by_key(|card| {
|
||||
(
|
||||
card.feed_url_is_origin(),
|
||||
title_votes[&normalize_name(&card.title)],
|
||||
Reverse(*card.publisher.as_bytes()),
|
||||
)
|
||||
})
|
||||
.cloned()
|
||||
.expect("a group has at least one card");
|
||||
let mut claims: BTreeMap<&FeedUrl, Vec<EndpointId>> = BTreeMap::new();
|
||||
for card in cards.iter().filter(|card| !card.feed_url_is_origin()) {
|
||||
claims
|
||||
.entry(&card.feed_url)
|
||||
.or_default()
|
||||
.push(card.publisher);
|
||||
}
|
||||
let mut claimed_feed_urls: Vec<ClaimedFeedUrl> = claims
|
||||
.into_iter()
|
||||
.map(|(feed_url, publishers)| ClaimedFeedUrl {
|
||||
feed_url: feed_url.clone(),
|
||||
publishers: sorted_publishers(publishers),
|
||||
})
|
||||
.collect();
|
||||
claimed_feed_urls.sort_by(|left, right| {
|
||||
right
|
||||
.publishers
|
||||
.len()
|
||||
.cmp(&left.publishers.len())
|
||||
.then_with(|| left.feed_url.cmp(&right.feed_url))
|
||||
});
|
||||
claimed_feed_urls.truncate(MAX_HIT_FEED_URL_CLAIMS);
|
||||
PodcastSearchHit {
|
||||
podcast_id,
|
||||
feed_url: card.origin_feed_url().clone(),
|
||||
publishers: sorted_publishers(cards.iter().map(|card| card.publisher).collect()),
|
||||
card,
|
||||
claimed_feed_urls,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn sorted_publishers(mut publishers: Vec<EndpointId>) -> Vec<EndpointId> {
|
||||
publishers.sort_by_key(|publisher| *publisher.as_bytes());
|
||||
publishers.dedup();
|
||||
publishers.truncate(MAX_HIT_PUBLISHERS);
|
||||
publishers
|
||||
}
|
||||
|
||||
/// Keeps hits whose card is indexed under every query token and ranks
|
||||
/// them: exact normalized title first, then hits with a card published under
|
||||
/// the origin feed URL, then title and id. The publisher count is not used:
|
||||
/// anyone can inflate it with new device identities.
|
||||
pub(crate) fn rank_hits(
|
||||
mut hits: Vec<PodcastSearchHit>,
|
||||
query: &str,
|
||||
tokens: &[String],
|
||||
) -> Vec<PodcastSearchHit> {
|
||||
let normalized_query = normalize_name(query);
|
||||
hits.retain(|hit| {
|
||||
let indexed = hit.card.index_tokens();
|
||||
tokens.iter().all(|token| indexed.contains(token))
|
||||
});
|
||||
hits.sort_by_cached_key(|hit| {
|
||||
let title = normalize_name(&hit.card.title);
|
||||
(
|
||||
title != normalized_query,
|
||||
!hit.card.feed_url_is_origin(),
|
||||
title,
|
||||
hit.podcast_id.clone(),
|
||||
)
|
||||
});
|
||||
hits
|
||||
}
|
||||
@@ -0,0 +1,891 @@
|
||||
//! SQLite cache of the podcast-discovery overlay: replicas, routing contacts
|
||||
//! and this device's own publications.
|
||||
//!
|
||||
//! The cache is replaceable federation state, kept in its own database file
|
||||
//! (like the similarity-routing cache). Every record read from it is decoded
|
||||
//! and verified again before it is used or served.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||
|
||||
use federation_net::{EndpointId, NetworkId};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
use crate::error::{MusicDhtError, Result};
|
||||
use crate::podcast::PodcastId;
|
||||
use crate::routing::{NodeContact, NodeId};
|
||||
|
||||
use super::MAX_PODCAST_DHT_RECORDS;
|
||||
use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records};
|
||||
|
||||
/// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1
|
||||
/// (pre-release) stored one payload per key; its replica table is dropped on
|
||||
/// open because the cache is replaceable. Own publications are kept.
|
||||
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2;
|
||||
/// Maximum records (cards and tombstones) indexed under one key. It equals
|
||||
/// [`MAX_PODCAST_DHT_RECORDS`], so one `FindValue` response carries every
|
||||
/// record of a key: nothing admitted is unreachable.
|
||||
///
|
||||
/// A full key admits a newcomer by evicting a random placement from its
|
||||
/// younger half (by first indexing time). The older half is never evicted
|
||||
/// for newcomers, so cards that were there before a flood stay, and the
|
||||
/// younger half stays contestable, so nobody who stored first holds the key
|
||||
/// for good.
|
||||
pub const MAX_PODCAST_RECORDS_PER_KEY: usize = MAX_PODCAST_DHT_RECORDS;
|
||||
/// Maximum tombstones indexed under one key, including tombstones that
|
||||
/// replaced their owner's card in place. Over budget, a random tombstone of
|
||||
/// another owner from the younger half of the key's tombstones is evicted.
|
||||
/// It equals [`MAX_PODCAST_RESPONSE_TOMBSTONES`], so every tombstone of a key
|
||||
/// is served.
|
||||
pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = MAX_PODCAST_RESPONSE_TOMBSTONES;
|
||||
/// Maximum tombstones in one `FindValue` response; cards fill the rest.
|
||||
pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32;
|
||||
/// Maximum `(podcast_id, publisher)` records one publisher device keeps in a
|
||||
/// replica. A publisher over it is refused (nothing is evicted for it).
|
||||
pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256;
|
||||
/// Maximum `(podcast_id, publisher)` records in a replica cache. A full cache
|
||||
/// admits a new record by evicting random records from the younger half of
|
||||
/// the cache (by first indexing time), like a full key.
|
||||
pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096;
|
||||
/// Maximum total encoded size of the records in a replica cache, enforced
|
||||
/// like [`MAX_PODCAST_CACHE_RECORDS`].
|
||||
pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024;
|
||||
/// Maximum persisted overlay contacts; the least recently seen are dropped.
|
||||
pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024;
|
||||
/// Maximum records evicted from the cache to admit one record; when that is
|
||||
/// not enough room, the record is refused and nothing is evicted.
|
||||
const MAX_EVICTIONS_PER_RECORD: usize = 16;
|
||||
|
||||
const _: () = assert!(
|
||||
MAX_PODCAST_RECORDS_PER_KEY <= MAX_PODCAST_DHT_RECORDS
|
||||
&& MAX_PODCAST_TOMBSTONES_PER_KEY <= MAX_PODCAST_RESPONSE_TOMBSTONES
|
||||
&& MAX_PODCAST_RESPONSE_TOMBSTONES <= MAX_PODCAST_DHT_RECORDS,
|
||||
"every record admitted under a key must fit one FindValue response"
|
||||
);
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_records (
|
||||
podcast_id TEXT NOT NULL,
|
||||
publisher TEXT NOT NULL,
|
||||
tombstone INTEGER NOT NULL,
|
||||
expires_at_ms INTEGER NOT NULL,
|
||||
payload_bytes INTEGER NOT NULL,
|
||||
payload BLOB NOT NULL,
|
||||
PRIMARY KEY (podcast_id, publisher)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry
|
||||
ON podcast_dht_records(expires_at_ms);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_publisher
|
||||
ON podcast_dht_records(publisher);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_keys (
|
||||
dht_key BLOB NOT NULL,
|
||||
podcast_id TEXT NOT NULL,
|
||||
publisher TEXT NOT NULL,
|
||||
tombstone INTEGER NOT NULL,
|
||||
expires_at_ms INTEGER NOT NULL,
|
||||
first_stored_at_ms INTEGER NOT NULL,
|
||||
PRIMARY KEY (dht_key, podcast_id, publisher)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_owner
|
||||
ON podcast_dht_keys(podcast_id, publisher);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_expiry
|
||||
ON podcast_dht_keys(expires_at_ms);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_peers (
|
||||
peer_id TEXT PRIMARY KEY,
|
||||
node_id BLOB NOT NULL,
|
||||
ticket TEXT NOT NULL,
|
||||
last_seen_ms INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_local (
|
||||
podcast_id TEXT PRIMARY KEY,
|
||||
revision INTEGER NOT NULL,
|
||||
spec BLOB,
|
||||
record BLOB NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
";
|
||||
|
||||
/// Admission limits of a replica cache.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct StoreLimits {
|
||||
pub per_key: usize,
|
||||
pub tombstones_per_key: usize,
|
||||
pub per_publisher: usize,
|
||||
pub records: usize,
|
||||
pub bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for StoreLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
per_key: MAX_PODCAST_RECORDS_PER_KEY,
|
||||
tombstones_per_key: MAX_PODCAST_TOMBSTONES_PER_KEY,
|
||||
per_publisher: MAX_PODCAST_RECORDS_PER_PUBLISHER,
|
||||
records: MAX_PODCAST_CACHE_RECORDS,
|
||||
bytes: MAX_PODCAST_CACHE_BYTES,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This device's latest publication of one podcast.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct LocalPublication {
|
||||
pub podcast_id: PodcastId,
|
||||
/// Greatest revision ever used for this podcast by this device.
|
||||
pub revision: u64,
|
||||
/// Published card content; `None` once unpublished.
|
||||
pub spec: Option<PodcastCardSpec>,
|
||||
/// Latest signed record (card or tombstone).
|
||||
pub record: SignedPodcastRecord,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct PodcastDhtDatabase {
|
||||
conn: Arc<Mutex<Connection>>,
|
||||
network_id: NetworkId,
|
||||
limits: StoreLimits,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PodcastDhtDatabase {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("PodcastDhtDatabase")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl PodcastDhtDatabase {
|
||||
pub async fn open(path: &Path, network_id: NetworkId) -> Result<Self> {
|
||||
Self::open_with_limits(path, network_id, StoreLimits::default()).await
|
||||
}
|
||||
|
||||
pub async fn open_with_limits(
|
||||
path: &Path,
|
||||
network_id: NetworkId,
|
||||
limits: StoreLimits,
|
||||
) -> Result<Self> {
|
||||
let path = path.to_path_buf();
|
||||
let conn = tokio::task::spawn_blocking(move || -> Result<Connection> {
|
||||
let conn = Connection::open(&path).map_err(|error| {
|
||||
MusicDhtError::Database(format!("failed to open {}: {error}", path.display()))
|
||||
})?;
|
||||
let version: i32 = conn
|
||||
.query_row("PRAGMA user_version", [], |row| row.get(0))
|
||||
.map_err(database_error)?;
|
||||
if version > PODCAST_DHT_CACHE_SCHEMA_VERSION {
|
||||
return Err(MusicDhtError::Database(format!(
|
||||
"podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}"
|
||||
)));
|
||||
}
|
||||
if version == 1 {
|
||||
// Replicas are replaceable; the old layout is dropped.
|
||||
conn.execute_batch("DROP TABLE IF EXISTS podcast_dht_records;")
|
||||
.map_err(database_error)?;
|
||||
}
|
||||
conn.execute_batch(SCHEMA).map_err(|error| {
|
||||
MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}"))
|
||||
})?;
|
||||
conn.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION)
|
||||
.map_err(database_error)?;
|
||||
Ok(conn)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))??;
|
||||
Ok(Self {
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
network_id,
|
||||
limits,
|
||||
})
|
||||
}
|
||||
|
||||
async fn call<F, T>(&self, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce(&Connection, &NetworkId, &StoreLimits) -> rusqlite::Result<T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let conn = Arc::clone(&self.conn);
|
||||
let network_id = self.network_id;
|
||||
let limits = self.limits;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let guard = lock(&conn);
|
||||
operation(&guard, &network_id, &limits).map_err(database_error)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))?
|
||||
}
|
||||
|
||||
/// Stores records under the given keys. The caller has verified every
|
||||
/// record, checked that each key belongs to it and that the sender is its
|
||||
/// publisher. Returns the number of `(key, record)` placements written.
|
||||
pub async fn store(
|
||||
&self,
|
||||
entries: Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>,
|
||||
now_ms: u64,
|
||||
) -> Result<usize> {
|
||||
self.call(move |conn, network_id, limits| {
|
||||
let transaction = conn.unchecked_transaction()?;
|
||||
let mut written = 0;
|
||||
for (keys, record) in entries {
|
||||
written += store_one(&transaction, network_id, limits, &keys, &record, now_ms)?;
|
||||
}
|
||||
transaction.commit()?;
|
||||
Ok(written)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Valid, unexpired records under `key`: at most
|
||||
/// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale
|
||||
/// replicas elsewhere can be overruled) and cards containing every token
|
||||
/// in `filter_tokens` up to `limit` records in all. Records indexed longest
|
||||
/// come first. Cards precede tombstones. With the default limits and
|
||||
/// `limit` = [`MAX_PODCAST_DHT_RECORDS`], every record of the key fits.
|
||||
pub async fn records_by_key(
|
||||
&self,
|
||||
key: PodcastDhtKey,
|
||||
filter_tokens: Vec<String>,
|
||||
now_ms: u64,
|
||||
limit: usize,
|
||||
) -> Result<Vec<SignedPodcastRecord>> {
|
||||
self.call(move |conn, network_id, _| {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT r.payload FROM podcast_dht_keys k
|
||||
JOIN podcast_dht_records r
|
||||
ON r.podcast_id = k.podcast_id AND r.publisher = k.publisher
|
||||
WHERE k.dht_key = ?1 AND k.tombstone = ?2 AND k.expires_at_ms > ?3
|
||||
ORDER BY k.first_stored_at_ms, k.podcast_id, k.publisher",
|
||||
)?;
|
||||
let mut select = |tombstone: bool, limit: usize| -> rusqlite::Result<Vec<_>> {
|
||||
let mut records = Vec::new();
|
||||
if limit == 0 {
|
||||
return Ok(records);
|
||||
}
|
||||
let rows = statement.query_map(
|
||||
params![key.as_bytes().as_slice(), tombstone, clamp_i64(now_ms)],
|
||||
|row| row.get::<_, Vec<u8>>(0),
|
||||
)?;
|
||||
for row in rows {
|
||||
let Some(record) = decode_valid(&row?, network_id, now_ms) else {
|
||||
continue;
|
||||
};
|
||||
if record.is_tombstone() != tombstone || !record.belongs_under(&key) {
|
||||
continue;
|
||||
}
|
||||
let matches = record.card().is_none_or(|card| {
|
||||
let tokens = card.index_tokens();
|
||||
filter_tokens.iter().all(|token| tokens.contains(token))
|
||||
});
|
||||
if matches {
|
||||
records.push(record);
|
||||
if records.len() == limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(records)
|
||||
};
|
||||
let tombstones = select(true, MAX_PODCAST_RESPONSE_TOMBSTONES.min(limit))?;
|
||||
let mut records = select(false, limit - tombstones.len())?;
|
||||
records.extend(tombstones);
|
||||
Ok(records)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Removes expired records and their key placements. Returns the number
|
||||
/// of records removed.
|
||||
pub async fn delete_expired(&self, now_ms: u64) -> Result<usize> {
|
||||
self.call(move |conn, _, _| {
|
||||
let now = clamp_i64(now_ms);
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys WHERE expires_at_ms <= ?1",
|
||||
[now],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1",
|
||||
[now],
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> {
|
||||
self.call(move |conn, _, _| {
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO podcast_dht_peers
|
||||
(peer_id, node_id, ticket, last_seen_ms)
|
||||
VALUES (?1, ?2, ?3, ?4)",
|
||||
params![
|
||||
contact.peer_id.to_string(),
|
||||
contact.node_id.as_bytes().as_slice(),
|
||||
contact.ticket,
|
||||
clamp_i64(contact.last_seen_ms),
|
||||
],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_peers WHERE peer_id IN (
|
||||
SELECT peer_id FROM podcast_dht_peers
|
||||
ORDER BY last_seen_ms DESC, peer_id
|
||||
LIMIT -1 OFFSET ?1)",
|
||||
[MAX_PERSISTED_PODCAST_DHT_PEERS as i64],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> {
|
||||
self.call(move |conn, _, _| {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_peers WHERE peer_id = ?1",
|
||||
[peer.to_string()],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn load_peers(&self) -> Result<Vec<NodeContact>> {
|
||||
self.call(move |conn, _, _| {
|
||||
let mut statement =
|
||||
conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?;
|
||||
let rows = statement.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, i64>(2)?,
|
||||
))
|
||||
})?;
|
||||
let mut peers = Vec::new();
|
||||
for row in rows {
|
||||
let (peer, ticket, last_seen_ms) = row?;
|
||||
let Ok(peer_id) = EndpointId::from_str(&peer) else {
|
||||
continue;
|
||||
};
|
||||
peers.push(NodeContact {
|
||||
node_id: NodeId::from_endpoint(&peer_id),
|
||||
peer_id,
|
||||
ticket,
|
||||
last_seen_ms: last_seen_ms.max(0) as u64,
|
||||
});
|
||||
}
|
||||
Ok(peers)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn local_publication(
|
||||
&self,
|
||||
podcast_id: PodcastId,
|
||||
) -> Result<Option<LocalPublication>> {
|
||||
self.call(move |conn, _, _| {
|
||||
conn.query_row(
|
||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||
WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
local_from_row,
|
||||
)
|
||||
.optional()
|
||||
.map(Option::flatten)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn local_publications(&self) -> Result<Vec<LocalPublication>> {
|
||||
self.call(move |conn, _, _| {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||
ORDER BY podcast_id",
|
||||
)?;
|
||||
let rows = statement.query_map([], local_from_row)?;
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
if let Some(publication) = row? {
|
||||
out.push(publication);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn put_local_publication(
|
||||
&self,
|
||||
publication: LocalPublication,
|
||||
now_ms: u64,
|
||||
) -> Result<()> {
|
||||
self.call(move |conn, _, _| {
|
||||
let spec = publication
|
||||
.spec
|
||||
.as_ref()
|
||||
.map(postcard::to_stdvec)
|
||||
.transpose()
|
||||
.map_err(to_sql_error)?;
|
||||
let record = postcard::to_stdvec(&publication.record).map_err(to_sql_error)?;
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO podcast_dht_local
|
||||
(podcast_id, revision, spec, record, updated_at_ms)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
params![
|
||||
publication.podcast_id.as_str(),
|
||||
publication.revision as i64,
|
||||
spec,
|
||||
record,
|
||||
clamp_i64(now_ms),
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn store_one(
|
||||
conn: &Connection,
|
||||
network_id: &NetworkId,
|
||||
limits: &StoreLimits,
|
||||
keys: &[PodcastDhtKey],
|
||||
record: &SignedPodcastRecord,
|
||||
now_ms: u64,
|
||||
) -> rusqlite::Result<usize> {
|
||||
let podcast_id = record.podcast_id().as_str().to_string();
|
||||
let publisher = record.publisher().to_string();
|
||||
let now = clamp_i64(now_ms);
|
||||
|
||||
// The cached record of this (podcast, publisher), re-verified.
|
||||
let cached: Option<(Vec<u8>, i64)> = conn
|
||||
.query_row(
|
||||
"SELECT payload, payload_bytes FROM podcast_dht_records
|
||||
WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![podcast_id, publisher],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let (existing, existing_bytes) = match cached {
|
||||
Some((payload, bytes)) => match decode_valid(&payload, network_id, now_ms) {
|
||||
Some(existing) => (Some(existing), bytes.max(0) as usize),
|
||||
None => {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
(None, 0)
|
||||
}
|
||||
},
|
||||
None => {
|
||||
// Stray placements without a record.
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
(None, 0)
|
||||
}
|
||||
};
|
||||
|
||||
// Higher revision, tombstone at an equal revision, later republish...
|
||||
let replace = match &existing {
|
||||
Some(existing) => match compare_records(existing, record) {
|
||||
std::cmp::Ordering::Greater => return Ok(0),
|
||||
std::cmp::Ordering::Equal => false,
|
||||
std::cmp::Ordering::Less => true,
|
||||
},
|
||||
None => true,
|
||||
};
|
||||
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
|
||||
if replace
|
||||
&& !make_room_for_record(
|
||||
conn,
|
||||
limits,
|
||||
(&podcast_id, &publisher),
|
||||
existing.is_none(),
|
||||
existing_bytes,
|
||||
payload.len(),
|
||||
now,
|
||||
)?
|
||||
{
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let mut indexed: HashSet<[u8; 32]> = HashSet::new();
|
||||
{
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT dht_key FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
)?;
|
||||
let rows = statement.query_map(params![podcast_id, publisher], |row| {
|
||||
row.get::<_, Vec<u8>>(0)
|
||||
})?;
|
||||
for row in rows {
|
||||
if let Ok(key) = <[u8; 32]>::try_from(row?.as_slice()) {
|
||||
indexed.insert(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut written = 0;
|
||||
if replace {
|
||||
// Placements the new version no longer belongs under are dropped (a
|
||||
// renamed card, or a tombstone that does not list the key); the others
|
||||
// now serve the new version.
|
||||
let belongs: HashSet<[u8; 32]> = record
|
||||
.dht_keys()
|
||||
.iter()
|
||||
.map(|key| *key.as_bytes())
|
||||
.collect();
|
||||
let stale: Vec<[u8; 32]> = indexed
|
||||
.iter()
|
||||
.filter(|key| !belongs.contains(*key))
|
||||
.copied()
|
||||
.collect();
|
||||
for key in stale {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key.as_slice(), podcast_id, publisher],
|
||||
)?;
|
||||
indexed.remove(&key);
|
||||
}
|
||||
written += conn.execute(
|
||||
"UPDATE podcast_dht_keys SET tombstone = ?3, expires_at_ms = ?4
|
||||
WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
],
|
||||
)?;
|
||||
// A card converted to a tombstone in place counts against the
|
||||
// tombstone budget of each key it stays under, like a new one.
|
||||
if record.is_tombstone() {
|
||||
for key in &indexed {
|
||||
let key = PodcastDhtKey::from_bytes(*key);
|
||||
make_room_under_key(conn, limits, &key, true, (&podcast_id, &publisher), now, 0)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// New placements. A tombstone only takes a new place under the podcast-id
|
||||
// key: under a text key it may only replace a record of its owner, so
|
||||
// nobody can index tombstones under someone else's search terms.
|
||||
let id_key = PodcastDhtKey::podcast(network_id, record.podcast_id());
|
||||
for key in keys {
|
||||
if indexed.contains(key.as_bytes()) {
|
||||
continue;
|
||||
}
|
||||
if record.is_tombstone() && *key != id_key {
|
||||
continue;
|
||||
}
|
||||
if !make_room_under_key(
|
||||
conn,
|
||||
limits,
|
||||
key,
|
||||
record.is_tombstone(),
|
||||
(&podcast_id, &publisher),
|
||||
now,
|
||||
1,
|
||||
)? {
|
||||
continue;
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_dht_keys
|
||||
(dht_key, podcast_id, publisher, tombstone, expires_at_ms, first_stored_at_ms)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![
|
||||
key.as_bytes().as_slice(),
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
now,
|
||||
],
|
||||
)?;
|
||||
indexed.insert(*key.as_bytes());
|
||||
written += 1;
|
||||
}
|
||||
|
||||
if indexed.is_empty() {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
return Ok(0);
|
||||
}
|
||||
if replace {
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_dht_records
|
||||
(podcast_id, publisher, tombstone, expires_at_ms, payload_bytes, payload)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(podcast_id, publisher) DO UPDATE SET
|
||||
tombstone = excluded.tombstone,
|
||||
expires_at_ms = excluded.expires_at_ms,
|
||||
payload_bytes = excluded.payload_bytes,
|
||||
payload = excluded.payload",
|
||||
params![
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
payload.len() as i64,
|
||||
payload,
|
||||
],
|
||||
)?;
|
||||
}
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
/// Makes room in the cache for the new version of a record of `owner`.
|
||||
///
|
||||
/// * A publisher at [`StoreLimits::per_publisher`] gets no new record.
|
||||
/// * When the record count (for a new owner) or the byte budget would be
|
||||
/// exceeded, random records of other owners from the younger half of the
|
||||
/// cache (by first indexing time) are evicted, at most
|
||||
/// [`MAX_EVICTIONS_PER_RECORD`]. If that is not enough, nothing is evicted
|
||||
/// and the record is refused.
|
||||
fn make_room_for_record(
|
||||
conn: &Connection,
|
||||
limits: &StoreLimits,
|
||||
owner: (&str, &str),
|
||||
new_owner: bool,
|
||||
previous_bytes: usize,
|
||||
new_bytes: usize,
|
||||
now: i64,
|
||||
) -> rusqlite::Result<bool> {
|
||||
if new_bytes > limits.bytes {
|
||||
return Ok(false);
|
||||
}
|
||||
if new_owner {
|
||||
let own: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2",
|
||||
params![owner.1, now],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if own.max(0) as usize >= limits.per_publisher {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
let (records, bytes): (i64, i64) = conn.query_row(
|
||||
"SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records
|
||||
WHERE expires_at_ms > ?1",
|
||||
[now],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)?;
|
||||
let mut records = records.max(0) as usize;
|
||||
let mut bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes);
|
||||
let fits = |records: usize, bytes: usize| {
|
||||
bytes.saturating_add(new_bytes) <= limits.bytes && (!new_owner || records < limits.records)
|
||||
};
|
||||
if fits(records, bytes) {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
// Candidates: the younger half of the other owners' records, shuffled.
|
||||
let others = records.saturating_sub(usize::from(!new_owner));
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id, publisher, payload_bytes FROM (
|
||||
SELECT r.podcast_id, r.publisher, r.payload_bytes,
|
||||
(SELECT MIN(k.first_stored_at_ms) FROM podcast_dht_keys k
|
||||
WHERE k.podcast_id = r.podcast_id AND k.publisher = r.publisher) AS since
|
||||
FROM podcast_dht_records r
|
||||
WHERE r.expires_at_ms > ?1 AND NOT (r.podcast_id = ?2 AND r.publisher = ?3)
|
||||
ORDER BY since DESC, random()
|
||||
LIMIT ?4)
|
||||
ORDER BY random() LIMIT ?5",
|
||||
)?;
|
||||
let candidates = statement
|
||||
.query_map(
|
||||
params![
|
||||
now,
|
||||
owner.0,
|
||||
owner.1,
|
||||
others.div_ceil(2) as i64,
|
||||
MAX_EVICTIONS_PER_RECORD as i64
|
||||
],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, i64>(2)?,
|
||||
))
|
||||
},
|
||||
)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let mut evict = Vec::new();
|
||||
for (podcast_id, publisher, payload_bytes) in candidates {
|
||||
if fits(records, bytes) {
|
||||
break;
|
||||
}
|
||||
records = records.saturating_sub(1);
|
||||
bytes = bytes.saturating_sub(payload_bytes.max(0) as usize);
|
||||
evict.push((podcast_id, publisher));
|
||||
}
|
||||
if !fits(records, bytes) {
|
||||
return Ok(false);
|
||||
}
|
||||
for (podcast_id, publisher) in evict {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Makes room under `key` for `incoming` more placements (0 when an existing
|
||||
/// placement of `owner` was converted to a tombstone in place) and returns
|
||||
/// whether the key has room.
|
||||
///
|
||||
/// Over [`StoreLimits::per_key`], or over [`StoreLimits::tombstones_per_key`]
|
||||
/// for a tombstone, a random placement of another owner from the younger
|
||||
/// half of the key (by first indexing time; only tombstones for the
|
||||
/// tombstone budget) is evicted. The older half is never evicted for a
|
||||
/// newcomer.
|
||||
fn make_room_under_key(
|
||||
conn: &Connection,
|
||||
limits: &StoreLimits,
|
||||
key: &PodcastDhtKey,
|
||||
tombstone: bool,
|
||||
owner: (&str, &str),
|
||||
now: i64,
|
||||
incoming: usize,
|
||||
) -> rusqlite::Result<bool> {
|
||||
for _ in 0..=MAX_EVICTIONS_PER_RECORD {
|
||||
let (total, tombstones): (i64, i64) = conn.query_row(
|
||||
"SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys
|
||||
WHERE dht_key = ?1 AND expires_at_ms > ?2",
|
||||
params![key.as_bytes().as_slice(), now],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)?;
|
||||
let over_tombstones =
|
||||
tombstone && tombstones.max(0) as usize + incoming > limits.tombstones_per_key;
|
||||
let over_total = total.max(0) as usize + incoming > limits.per_key;
|
||||
if !over_tombstones && !over_total {
|
||||
return Ok(true);
|
||||
}
|
||||
let Some((podcast_id, publisher)) =
|
||||
young_placement(conn, key, over_tombstones, owner, now)?
|
||||
else {
|
||||
return Ok(false);
|
||||
};
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key.as_bytes().as_slice(), podcast_id, publisher],
|
||||
)?;
|
||||
let placed: bool = conn.query_row(
|
||||
"SELECT EXISTS (SELECT 1 FROM podcast_dht_keys
|
||||
WHERE podcast_id = ?1 AND publisher = ?2)",
|
||||
params![podcast_id, publisher],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if !placed {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// A random unexpired placement of another owner than `owner` from the
|
||||
/// younger half of `key` (only tombstones with `tombstones_only`).
|
||||
fn young_placement(
|
||||
conn: &Connection,
|
||||
key: &PodcastDhtKey,
|
||||
tombstones_only: bool,
|
||||
owner: (&str, &str),
|
||||
now: i64,
|
||||
) -> rusqlite::Result<Option<(String, String)>> {
|
||||
let filter = "dht_key = ?1 AND expires_at_ms > ?2 AND (?3 = 0 OR tombstone = 1)
|
||||
AND NOT (podcast_id = ?4 AND publisher = ?5)";
|
||||
let count: i64 = conn.query_row(
|
||||
&format!("SELECT COUNT(*) FROM podcast_dht_keys WHERE {filter}"),
|
||||
params![
|
||||
key.as_bytes().as_slice(),
|
||||
now,
|
||||
tombstones_only,
|
||||
owner.0,
|
||||
owner.1
|
||||
],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if count <= 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
conn.query_row(
|
||||
&format!(
|
||||
"SELECT podcast_id, publisher FROM (
|
||||
SELECT podcast_id, publisher FROM podcast_dht_keys WHERE {filter}
|
||||
ORDER BY first_stored_at_ms DESC, random() LIMIT ?6)
|
||||
ORDER BY random() LIMIT 1"
|
||||
),
|
||||
params![
|
||||
key.as_bytes().as_slice(),
|
||||
now,
|
||||
tombstones_only,
|
||||
owner.0,
|
||||
owner.1,
|
||||
(count as usize).div_ceil(2) as i64
|
||||
],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()
|
||||
}
|
||||
|
||||
fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![podcast_id, publisher],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![podcast_id, publisher],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_valid(
|
||||
payload: &[u8],
|
||||
network_id: &NetworkId,
|
||||
now_ms: u64,
|
||||
) -> Option<SignedPodcastRecord> {
|
||||
let record: SignedPodcastRecord = postcard::from_bytes(payload).ok()?;
|
||||
record.verify_at(network_id, now_ms).ok()?;
|
||||
Some(record)
|
||||
}
|
||||
|
||||
fn local_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Option<LocalPublication>> {
|
||||
let podcast_id: String = row.get(0)?;
|
||||
let revision: i64 = row.get(1)?;
|
||||
let spec: Option<Vec<u8>> = row.get(2)?;
|
||||
let record: Vec<u8> = row.get(3)?;
|
||||
let Ok(podcast_id) = PodcastId::parse(&podcast_id) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let spec = match spec {
|
||||
Some(bytes) => match postcard::from_bytes::<PodcastCardSpec>(&bytes) {
|
||||
Ok(spec) => Some(spec),
|
||||
Err(_) => return Ok(None),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
let Ok(record) = postcard::from_bytes::<SignedPodcastRecord>(&record) else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(LocalPublication {
|
||||
podcast_id,
|
||||
revision: revision as u64,
|
||||
spec,
|
||||
record,
|
||||
}))
|
||||
}
|
||||
|
||||
fn clamp_i64(value: u64) -> i64 {
|
||||
value.min(i64::MAX as u64) as i64
|
||||
}
|
||||
|
||||
fn to_sql_error(error: impl std::error::Error + Send + Sync + 'static) -> rusqlite::Error {
|
||||
rusqlite::Error::ToSqlConversionFailure(Box::new(error))
|
||||
}
|
||||
|
||||
fn database_error(error: rusqlite::Error) -> MusicDhtError {
|
||||
MusicDhtError::Database(error.to_string())
|
||||
}
|
||||
|
||||
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
|
||||
mutex.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,763 @@
|
||||
//! Integration test: public podcast discovery on three real loopback iroh
|
||||
//! nodes.
|
||||
//!
|
||||
//! * A publishes a podcast its user shared. B is another device of the same
|
||||
//! user: it applies the same merged registers (as device sync would deliver
|
||||
//! them) and publishes the same podcast under its own key. C is an
|
||||
//! unrelated peer outside the user's trusted-device group, with no
|
||||
//! device-sync pairing.
|
||||
//! * C finds the podcast by text and by id, merged over publishers {A, B};
|
||||
//! once B unshares and goes offline, C still finds A's card, so B is
|
||||
//! irrelevant to discovery. A podcast the user never shared (held by both
|
||||
//! A and B) is never discoverable; after A unpublishes, the tombstone hides
|
||||
//! the card from C.
|
||||
//! * Forged, foreign, tampered, replayed, malformed, oversized and
|
||||
//! wrong-version requests are rejected by a real peer.
|
||||
//! * D, a music-DHT peer without the overlay, is dialed by bootstrap but
|
||||
//! never answers on the overlay ALPN, so it never becomes an overlay contact
|
||||
//! and is never asked for records or to store them.
|
||||
//! * `sync_shares` stays fail-safe for revocation and holds back URLs that
|
||||
//! look private (single node).
|
||||
//! * Contacts a peer returns are never trusted: fabricated, unreachable
|
||||
//! contacts do not enter the routing table (two nodes).
|
||||
//! * After a restart, own cards persisted by the previous run are neither
|
||||
//! republished nor served until the first `sync_shares`, so a share
|
||||
//! revoked on another device meanwhile is never re-sent (two nodes).
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use music_dht::capabilities::{CapabilityManifest, PODCAST_DHT_ID};
|
||||
use music_dht::podcast::{FeedUrl, PodcastId, PodcastMeta, ShareRecord, SubscriptionRecord};
|
||||
use music_dht::podcast_dht::{
|
||||
MAX_PODCAST_DHT_REQUEST_BYTES, PODCAST_DHT_ALPN, PODCAST_DHT_PROTOCOL_VERSION, PodcastCard,
|
||||
PodcastCardSpec, PodcastDht, PodcastDhtOperation, PodcastDhtRequest, PodcastDhtResponse,
|
||||
PodcastRecordBody, PodcastSearchHit, PodcastShareState, PodcastStoreEntry, SignedPodcastRecord,
|
||||
};
|
||||
use music_dht::{MusicDhtConfig, MusicDhtError, MusicDhtService, NetworkId, NodeContact, NodeId};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const TEST_TIMEOUT: Duration = Duration::from_secs(240);
|
||||
|
||||
struct Node {
|
||||
service: Arc<MusicDhtService>,
|
||||
dht: Arc<PodcastDht>,
|
||||
tasks: Vec<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl Node {
|
||||
async fn start(directory: &Path, network: NetworkId) -> Self {
|
||||
let (service, events) = start_service(directory, network, true).await;
|
||||
let acceptor = service.stream_acceptor(PODCAST_DHT_ALPN).unwrap();
|
||||
let dht = PodcastDht::open(Arc::clone(&service), directory.join("podcast-dht.sqlite3"))
|
||||
.await
|
||||
.unwrap();
|
||||
let tasks = vec![events, tokio::spawn(Arc::clone(&dht).serve(acceptor))];
|
||||
Self {
|
||||
service,
|
||||
dht,
|
||||
tasks,
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop(self) {
|
||||
for task in &self.tasks {
|
||||
task.abort();
|
||||
}
|
||||
self.service.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
async fn contact(&self) -> NodeContact {
|
||||
let peer_id = self.service.endpoint_id();
|
||||
NodeContact {
|
||||
node_id: NodeId::from_endpoint(&peer_id),
|
||||
peer_id,
|
||||
ticket: self.service.ticket().await.unwrap().to_string(),
|
||||
last_seen_ms: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn start_service(
|
||||
directory: &Path,
|
||||
network: NetworkId,
|
||||
overlay: bool,
|
||||
) -> (Arc<MusicDhtService>, tokio::task::JoinHandle<()>) {
|
||||
std::fs::create_dir_all(directory).unwrap();
|
||||
let mut config = MusicDhtConfig::builder()
|
||||
.data_dir(directory)
|
||||
.network_id(network)
|
||||
.request_timeout(Duration::from_secs(2))
|
||||
.lookup_timeout(Duration::from_secs(5))
|
||||
.transport_timeout(Duration::from_secs(5))
|
||||
.dial_timeout(Duration::from_secs(2));
|
||||
if overlay {
|
||||
config = config.schema_independent_stream_protocol(PODCAST_DHT_ALPN);
|
||||
}
|
||||
let (service, mut events) = MusicDhtService::start(config.build().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let events = tokio::spawn(async move { while events.recv().await.is_some() {} });
|
||||
(Arc::new(service), events)
|
||||
}
|
||||
|
||||
fn feed(path: &str) -> FeedUrl {
|
||||
FeedUrl::parse(&format!("https://feeds.example.test/{path}")).unwrap()
|
||||
}
|
||||
|
||||
/// A subscription of A's user (generation 1).
|
||||
fn subscription(path: &str, title: &str) -> SubscriptionRecord {
|
||||
let feed_url = feed(path);
|
||||
SubscriptionRecord {
|
||||
podcast_id: PodcastId::from_feed_url(&feed_url),
|
||||
subscribed: true,
|
||||
podcast: PodcastMeta {
|
||||
feed_url,
|
||||
title: title.to_string(),
|
||||
author: Some("Ferris Crab".to_string()),
|
||||
description: Some("Weekly news.".to_string()),
|
||||
link: None,
|
||||
image_url: None,
|
||||
},
|
||||
feed_aliases: Vec::new(),
|
||||
generation: 1,
|
||||
hlc_ms: 1_000,
|
||||
op_id: "device-a-op-1".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn share(subscription: &SubscriptionRecord, shared: bool, generation: u64) -> ShareRecord {
|
||||
ShareRecord {
|
||||
podcast_id: subscription.podcast_id.clone(),
|
||||
shared,
|
||||
generation,
|
||||
hlc_ms: 2_000 + generation as i64,
|
||||
op_id: format!("device-a-share-{generation}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for<T>(what: &str, mut check: impl AsyncFnMut() -> Option<T>) -> T {
|
||||
let started = std::time::Instant::now();
|
||||
loop {
|
||||
if let Some(value) = check().await {
|
||||
return value;
|
||||
}
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(60),
|
||||
"timed out waiting for {what}"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends raw bytes as one overlay request and decodes the response.
|
||||
async fn raw_request(from: &Node, to: &Node, payload: &[u8]) -> PodcastDhtResponse {
|
||||
let ticket = to.service.ticket().await.unwrap();
|
||||
let mut stream = from
|
||||
.service
|
||||
.open_stream_to(&ticket, PODCAST_DHT_ALPN)
|
||||
.await
|
||||
.unwrap();
|
||||
stream.send.write_all(payload).await.unwrap();
|
||||
stream.send.finish().unwrap();
|
||||
let mut bytes = Vec::new();
|
||||
AsyncReadExt::read_to_end(&mut stream.recv, &mut bytes)
|
||||
.await
|
||||
.unwrap();
|
||||
postcard::from_bytes(&bytes).unwrap()
|
||||
}
|
||||
|
||||
async fn store_request(from: &Node, to: &Node, record: SignedPodcastRecord) -> PodcastDhtResponse {
|
||||
let request = PodcastDhtRequest {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
requester: from.contact().await,
|
||||
operation: PodcastDhtOperation::StoreBatch {
|
||||
records: vec![PodcastStoreEntry {
|
||||
keys: record.dht_keys(),
|
||||
record,
|
||||
}],
|
||||
},
|
||||
};
|
||||
raw_request(from, to, &postcard::to_stdvec(&request).unwrap()).await
|
||||
}
|
||||
|
||||
async fn search(node: &Node, query: &str) -> Vec<PodcastSearchHit> {
|
||||
node.dht.search(query, 10).await.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn shared_podcast_is_discoverable_by_strangers_and_retractable() {
|
||||
tokio::time::timeout(TEST_TIMEOUT, scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn scenario() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let network = NetworkId::from_name("podcast-dht-integration");
|
||||
let a = Node::start(&temp.path().join("a"), network).await;
|
||||
let b = Node::start(&temp.path().join("b"), network).await;
|
||||
let c = Node::start(&temp.path().join("c"), network).await;
|
||||
// D runs the music DHT only.
|
||||
let (d, d_events) = start_service(&temp.path().join("d"), network, false).await;
|
||||
|
||||
// B and C reach A through the ordinary music-DHT transport; C has no
|
||||
// device-sync pairing with A or B.
|
||||
b.service
|
||||
.connect(a.service.ticket().await.unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
c.service
|
||||
.connect(a.service.ticket().await.unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
d.connect(a.service.ticket().await.unwrap()).await.unwrap();
|
||||
wait_for("music DHT contacts", async || {
|
||||
(a.service.known_peers().len() >= 3
|
||||
&& !b.service.known_peers().is_empty()
|
||||
&& !c.service.known_peers().is_empty())
|
||||
.then_some(())
|
||||
})
|
||||
.await;
|
||||
for node in [&a, &b, &c] {
|
||||
node.dht.bootstrap().await.unwrap();
|
||||
}
|
||||
wait_for("podcast overlay contacts", async || {
|
||||
(a.dht.known_peers() >= 2 && c.dht.known_peers() >= 1).then_some(())
|
||||
})
|
||||
.await;
|
||||
// D is dialed by bootstrap but never answers on the overlay ALPN, so it
|
||||
// never becomes an overlay contact: it is never asked for records or to
|
||||
// store them.
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(a.dht.known_peers(), 2);
|
||||
|
||||
// A capability manifest without podcast_dht evicts a peer; one with it
|
||||
// lets the peer be learned again.
|
||||
let manifest = CapabilityManifest::frid("furumi", "test").with_podcast_dht();
|
||||
a.dht
|
||||
.note_peer_capabilities(
|
||||
c.service.endpoint_id(),
|
||||
&manifest.clone().without_protocol(PODCAST_DHT_ID),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(a.dht.known_peers(), 1);
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(
|
||||
a.dht.known_peers(),
|
||||
1,
|
||||
"the evicted peer is not bootstrapped"
|
||||
);
|
||||
a.dht
|
||||
.note_peer_capabilities(c.service.endpoint_id(), &manifest)
|
||||
.await
|
||||
.unwrap();
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(a.dht.known_peers(), 2);
|
||||
|
||||
// A request with another overlay version is refused.
|
||||
let request = PodcastDhtRequest {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION + 1,
|
||||
requester: c.contact().await,
|
||||
operation: PodcastDhtOperation::FindNode { target: [0; 32] },
|
||||
};
|
||||
let response = raw_request(&c, &a, &postcard::to_stdvec(&request).unwrap()).await;
|
||||
assert!(!response.ok);
|
||||
|
||||
// A's user shares "Rust Radio Weekly" but never shares "Secret Garden".
|
||||
let shared = subscription("rust-radio.xml", "Rust Radio Weekly");
|
||||
let private = subscription("secret-garden.xml", "Secret Garden Diaries");
|
||||
let states = vec![
|
||||
PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, true, 2)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
PodcastShareState {
|
||||
subscription: private.clone(),
|
||||
share: None,
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
];
|
||||
let stats = a.dht.sync_shares(states.clone()).await.unwrap();
|
||||
assert_eq!(
|
||||
(stats.published, stats.unchanged, stats.unpublished),
|
||||
(1, 0, 0)
|
||||
);
|
||||
assert!(stats.push.local_replica);
|
||||
assert!(stats.push.remote_nodes >= 1);
|
||||
let error = a
|
||||
.dht
|
||||
.publish(
|
||||
&private,
|
||||
&share(&private, false, 2),
|
||||
PodcastCardSpec::from_subscription(&private).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, MusicDhtError::PodcastPublication(_)));
|
||||
// Idempotent: nothing new is published.
|
||||
let again = a.dht.sync_shares(states).await.unwrap();
|
||||
assert_eq!(
|
||||
(again.published, again.unchanged, again.unpublished),
|
||||
(0, 1, 0)
|
||||
);
|
||||
let published = a.dht.published().await.unwrap();
|
||||
assert_eq!(published.len(), 1);
|
||||
let first_card = published[0].record.clone();
|
||||
|
||||
// B, the same user's other device, holds the same merged registers and
|
||||
// publishes the same podcast under its own key; the private podcast
|
||||
// stays private there too.
|
||||
let b_states = vec![
|
||||
PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, true, 2)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
PodcastShareState {
|
||||
subscription: private.clone(),
|
||||
share: None,
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
];
|
||||
let stats = b.dht.sync_shares(b_states).await.unwrap();
|
||||
assert_eq!((stats.published, stats.unpublished), (1, 0));
|
||||
let mut both = vec![a.service.endpoint_id(), b.service.endpoint_id()];
|
||||
both.sort_by_key(|key| *key.as_bytes());
|
||||
let hits = wait_for("C sees both publishers", async || {
|
||||
let hits = search(&c, "rust radio").await;
|
||||
(hits.len() == 1 && hits[0].publishers == both).then_some(hits)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
|
||||
assert!(hits[0].claimed_feed_urls.is_empty());
|
||||
|
||||
// B unshares (for example a stale register on B) and goes offline: B is
|
||||
// not needed for discovery, A's card is still found.
|
||||
let stats = b
|
||||
.dht
|
||||
.sync_shares(vec![PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, false, 3)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
}])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.unpublished, 1);
|
||||
let a_only = vec![a.service.endpoint_id()];
|
||||
wait_for("B's tombstone reaches C's lookup", async || {
|
||||
let hits = search(&c, "rust radio").await;
|
||||
(hits.len() == 1 && hits[0].publishers == a_only).then_some(())
|
||||
})
|
||||
.await;
|
||||
b.stop().await;
|
||||
|
||||
let hits = wait_for("C finds the card by text", async || {
|
||||
let hits = search(&c, "rust radio").await;
|
||||
(!hits.is_empty()).then_some(hits)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(hits[0].podcast_id, shared.podcast_id);
|
||||
assert_eq!(hits[0].publishers, vec![a.service.endpoint_id()]);
|
||||
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
|
||||
assert_eq!(hits[0].card.feed_url, shared.podcast.feed_url);
|
||||
assert_eq!(hits[0].card.title, "Rust Radio Weekly");
|
||||
assert_eq!(search(&c, "ferris weekly").await.len(), 1, "author tokens");
|
||||
assert!(
|
||||
search(&c, "rust jazz").await.is_empty(),
|
||||
"every token must match"
|
||||
);
|
||||
let by_id = c.dht.lookup_podcast(&shared.podcast_id).await.unwrap();
|
||||
assert_eq!(by_id.unwrap().card.podcast_id, shared.podcast_id);
|
||||
|
||||
// The never-shared podcast leaves no trace anywhere.
|
||||
assert!(search(&c, "secret garden").await.is_empty());
|
||||
assert!(search(&a, "secret garden").await.is_empty());
|
||||
assert!(
|
||||
c.dht
|
||||
.lookup_podcast(&private.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
a.dht
|
||||
.lookup_podcast(&private.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
// Forged: C signs a card that claims A as publisher.
|
||||
let forged_spec =
|
||||
PodcastCardSpec::from_subscription(&subscription("forged.xml", "Forged Show")).unwrap();
|
||||
let forged_card = PodcastCard::from_spec(
|
||||
forged_spec.clone(),
|
||||
network,
|
||||
a.service.endpoint_id(),
|
||||
u64::MAX - 1,
|
||||
first_card.issued_at_ms(),
|
||||
);
|
||||
let forged = SignedPodcastRecord {
|
||||
signature: c.service.sign_identity(&forged_card.signing_bytes()),
|
||||
body: PodcastRecordBody::Card(Box::new(forged_card)),
|
||||
};
|
||||
let response = store_request(&c, &a, forged).await;
|
||||
assert!(!response.ok && response.stored == 0, "{response:?}");
|
||||
|
||||
// Foreign: A's genuine card stored by C. Only publishers store records.
|
||||
let response = store_request(&c, &a, first_card.clone()).await;
|
||||
assert!(!response.ok && response.stored == 0, "{response:?}");
|
||||
|
||||
// Tampered: A's genuine card with a different description.
|
||||
let mut tampered = first_card.clone();
|
||||
if let PodcastRecordBody::Card(card) = &mut tampered.body {
|
||||
card.description = Some("Now with malware".into());
|
||||
}
|
||||
let response = store_request(&c, &a, tampered).await;
|
||||
assert!(!response.ok, "{response:?}");
|
||||
|
||||
// Malformed and oversized requests.
|
||||
let response = raw_request(&c, &a, b"definitely not postcard").await;
|
||||
assert!(!response.ok);
|
||||
let response = raw_request(&c, &a, &vec![0u8; MAX_PODCAST_DHT_REQUEST_BYTES + 1]).await;
|
||||
assert!(!response.ok);
|
||||
assert!(response.error.unwrap().contains("size"));
|
||||
|
||||
// A valid card signed by C is accepted: the overlay is open to anyone
|
||||
// who signs their own cards.
|
||||
let own_card = PodcastCard::from_spec(
|
||||
forged_spec,
|
||||
network,
|
||||
c.service.endpoint_id(),
|
||||
1,
|
||||
first_card.issued_at_ms(),
|
||||
);
|
||||
let own =
|
||||
SignedPodcastRecord::sign_card(own_card, |bytes| c.service.sign_identity(bytes)).unwrap();
|
||||
let response = store_request(&c, &a, own).await;
|
||||
assert!(response.ok && response.stored > 0, "{response:?}");
|
||||
let forged_hits = search(&a, "forged show").await;
|
||||
assert_eq!(forged_hits.len(), 1);
|
||||
assert_eq!(forged_hits[0].publishers, vec![c.service.endpoint_id()]);
|
||||
|
||||
// A's user stops sharing: a tombstone with the next revision replaces
|
||||
// the card and C stops seeing it.
|
||||
let unshare = vec![PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, false, 3)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
}];
|
||||
let stats = a.dht.sync_shares(unshare).await.unwrap();
|
||||
assert_eq!(stats.unpublished, 1);
|
||||
let published = a.dht.published().await.unwrap();
|
||||
assert!(!published[0].active);
|
||||
assert!(published[0].record.is_tombstone());
|
||||
assert_eq!(published[0].revision, first_card.revision() + 1);
|
||||
wait_for("tombstone hides the card from C", async || {
|
||||
search(&c, "rust radio").await.is_empty().then_some(())
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
c.dht
|
||||
.lookup_podcast(&shared.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
// A replaying its own old card to a replica that stores the tombstone
|
||||
// does not bring it back; nobody else may replay it at all.
|
||||
let response = store_request(&a, &c, first_card.clone()).await;
|
||||
assert!(response.ok && response.stored == 0, "{response:?}");
|
||||
let response = store_request(&c, &a, first_card).await;
|
||||
assert!(!response.ok, "{response:?}");
|
||||
assert!(search(&c, "rust radio").await.is_empty());
|
||||
assert!(search(&a, "rust radio").await.is_empty());
|
||||
|
||||
a.stop().await;
|
||||
c.stop().await;
|
||||
d_events.abort();
|
||||
d.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn sync_shares_is_fail_safe_and_holds_back_private_urls() {
|
||||
tokio::time::timeout(TEST_TIMEOUT, single_node_scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn single_node_scenario() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let network = NetworkId::from_name("podcast-dht-sync-shares");
|
||||
let a = Node::start(&temp.path().join("a"), network).await;
|
||||
let state =
|
||||
|subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState {
|
||||
subscription: subscription.clone(),
|
||||
share: Some(share(subscription, shared, generation)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
};
|
||||
let first = subscription("first.xml", "First Show");
|
||||
let second = subscription("second.xml", "Second Show");
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&first, true, 2), state(&second, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.published, 2);
|
||||
assert!(stats.failed.is_empty() && stats.held_back.is_empty());
|
||||
|
||||
// In one call the user unshares the first podcast while the second
|
||||
// one's card cannot be built (the URL its id derives from is unknown).
|
||||
let mut broken = second.clone();
|
||||
broken.podcast.feed_url = feed("elsewhere.xml");
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&first, false, 3), state(&broken, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.failed.len(), 1);
|
||||
assert_eq!(stats.failed[0].podcast_id, second.podcast_id);
|
||||
assert_eq!(stats.unpublished, 2, "the unshared and the broken podcast");
|
||||
let published = a.dht.published().await.unwrap();
|
||||
assert!(published.iter().all(|publication| !publication.active));
|
||||
assert!(
|
||||
published
|
||||
.iter()
|
||||
.all(|publication| publication.record.is_tombstone())
|
||||
);
|
||||
// Republishing re-sends tombstones only; nothing comes back.
|
||||
a.dht.republish().await.unwrap();
|
||||
assert!(search(&a, "first show").await.is_empty());
|
||||
assert!(search(&a, "second show").await.is_empty());
|
||||
|
||||
// A feed URL that looks private is held back until the user confirms.
|
||||
let mut member = subscription("member.xml?token=abc", "Member Show");
|
||||
member.podcast_id = PodcastId::from_feed_url(&member.podcast.feed_url);
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&member, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.published, 0);
|
||||
assert_eq!(stats.held_back.len(), 1);
|
||||
assert_eq!(stats.held_back[0].podcast_id, member.podcast_id);
|
||||
assert!(!stats.held_back[0].warnings.is_empty());
|
||||
assert!(search(&a, "member show").await.is_empty());
|
||||
let mut confirmed = state(&member, true, 2);
|
||||
confirmed.privacy_confirmed = true;
|
||||
let stats = a.dht.sync_shares(vec![confirmed]).await.unwrap();
|
||||
assert_eq!(stats.published, 1);
|
||||
assert_eq!(search(&a, "member show").await.len(), 1);
|
||||
// Once published with the user's consent, the same URLs are kept.
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&member, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((stats.unchanged, stats.held_back.len()), (1, 0));
|
||||
|
||||
a.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn fabricated_contacts_never_join_the_overlay() {
|
||||
tokio::time::timeout(TEST_TIMEOUT, fabricated_contacts_scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// K contacts whose node ids are genuine hashes of their endpoint ids but
|
||||
/// whose tickets carry no address, as a malicious peer could return them.
|
||||
fn fabricated_contacts(network: NetworkId) -> Vec<NodeContact> {
|
||||
(0..music_dht::K)
|
||||
.map(|index| {
|
||||
let peer = federation_net::SecretKey::from_bytes(&[index as u8 + 101; 32]).public();
|
||||
let ticket = federation_net::PeerTicket {
|
||||
ticket_version: federation_net::TICKET_VERSION,
|
||||
protocol_version: 1,
|
||||
network_id: network,
|
||||
schema_id: federation_net::SchemaId::from_bytes([0; 32]),
|
||||
endpoint_addr: federation_net::EndpointAddr::new(peer),
|
||||
};
|
||||
NodeContact {
|
||||
node_id: NodeId::from_endpoint(&peer),
|
||||
peer_id: peer,
|
||||
ticket: ticket.to_string(),
|
||||
last_seen_ms: u64::MAX,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn fabricated_contacts_scenario() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let network = NetworkId::from_name("podcast-dht-fabricated-contacts");
|
||||
let a = Node::start(&temp.path().join("a"), network).await;
|
||||
// M speaks the overlay ALPN but answers every request with fabricated
|
||||
// contacts.
|
||||
let (m, m_events) = start_service(&temp.path().join("m"), network, true).await;
|
||||
let mut acceptor = m.stream_acceptor(PODCAST_DHT_ALPN).unwrap();
|
||||
let fabricated = fabricated_contacts(network);
|
||||
let response = PodcastDhtResponse {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
ok: true,
|
||||
error: None,
|
||||
nodes: fabricated.clone(),
|
||||
records: Vec::new(),
|
||||
stored: 0,
|
||||
};
|
||||
let payload = postcard::to_stdvec(&response).unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
while let Some(mut stream) = acceptor.accept().await {
|
||||
let mut request = Vec::new();
|
||||
let _ = AsyncReadExt::read_to_end(
|
||||
&mut (&mut stream.recv).take(MAX_PODCAST_DHT_REQUEST_BYTES as u64),
|
||||
&mut request,
|
||||
)
|
||||
.await;
|
||||
let _ = stream.send.write_all(&payload).await;
|
||||
let _ = stream.send.finish();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(2), stream.send.stopped()).await;
|
||||
}
|
||||
});
|
||||
m.connect(a.service.ticket().await.unwrap()).await.unwrap();
|
||||
wait_for("music DHT contact", async || {
|
||||
(!a.service.known_peers().is_empty()).then_some(())
|
||||
})
|
||||
.await;
|
||||
|
||||
// Bootstrap learns M, which answered, and none of the contacts it made
|
||||
// up: they are tried once and never answer.
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(a.dht.known_peers(), 1);
|
||||
// A lookup treats them as candidates only.
|
||||
let podcast_id = PodcastId::from_feed_url(&feed("nothing.xml"));
|
||||
assert!(a.dht.lookup_podcast(&podcast_id).await.unwrap().is_none());
|
||||
assert_eq!(a.dht.known_peers(), 1);
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(
|
||||
a.dht.known_peers(),
|
||||
1,
|
||||
"fabricated contacts never count for replica responsibility"
|
||||
);
|
||||
|
||||
server.abort();
|
||||
a.stop().await;
|
||||
m_events.abort();
|
||||
m.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn restarted_device_waits_for_sync_shares_before_resending_cards() {
|
||||
tokio::time::timeout(TEST_TIMEOUT, restart_scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn restart_scenario() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let network = NetworkId::from_name("podcast-dht-restart");
|
||||
let state =
|
||||
|subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState {
|
||||
subscription: subscription.clone(),
|
||||
share: Some(share(subscription, shared, generation)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
};
|
||||
let kept = subscription("kept.xml", "Kept Show");
|
||||
let revoked = subscription("revoked.xml", "Revoked Show");
|
||||
|
||||
// A publishes both podcasts while alone, then goes offline.
|
||||
let a = Node::start(&temp.path().join("a"), network).await;
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&kept, true, 2), state(&revoked, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((stats.published, stats.push.remote_nodes), (2, 0));
|
||||
a.stop().await;
|
||||
|
||||
// Meanwhile another trusted device revokes "Revoked Show". A comes back
|
||||
// with its persisted cards and runs maintenance before its client
|
||||
// reconciled: the first bootstrap finds B.
|
||||
let b = Node::start(&temp.path().join("b"), network).await;
|
||||
let mut a = Node::start(&temp.path().join("a"), network).await;
|
||||
b.service
|
||||
.connect(a.service.ticket().await.unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for("music DHT contacts", async || {
|
||||
(!a.service.known_peers().is_empty()).then_some(())
|
||||
})
|
||||
.await;
|
||||
a.tasks.push(tokio::spawn(Arc::clone(&a.dht).maintenance()));
|
||||
wait_for("overlay contacts", async || {
|
||||
(a.dht.known_peers() >= 1 && b.dht.known_peers() >= 1).then_some(())
|
||||
})
|
||||
.await;
|
||||
// Neither maintenance nor an explicit republish sends a stale card, and
|
||||
// A does not serve them to lookups either.
|
||||
let stats = a.dht.republish().await.unwrap();
|
||||
assert_eq!(stats.records, 0, "stale cards are withheld");
|
||||
for podcast in [&kept, &revoked] {
|
||||
assert!(
|
||||
b.dht
|
||||
.lookup_podcast(&podcast.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(search(&b, "show").await.is_empty());
|
||||
|
||||
// The client's first reconciliation revokes the stale share and
|
||||
// re-sends the unchanged card.
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&kept, true, 2), state(&revoked, false, 3)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(stats.published, stats.unchanged, stats.unpublished),
|
||||
(0, 1, 1)
|
||||
);
|
||||
assert_eq!(stats.push.records, 2, "the kept card and the tombstone");
|
||||
assert!(stats.push.remote_nodes >= 1);
|
||||
let a_only = vec![a.service.endpoint_id()];
|
||||
let hit = b
|
||||
.dht
|
||||
.lookup_podcast(&kept.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("the kept card is found");
|
||||
assert_eq!(hit.publishers, a_only);
|
||||
assert!(
|
||||
b.dht
|
||||
.lookup_podcast(&revoked.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
// From now on republishing covers the kept card and the tombstone.
|
||||
let stats = a.dht.republish().await.unwrap();
|
||||
assert_eq!(stats.records, 2);
|
||||
let hits = search(&b, "show").await;
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(hits[0].podcast_id, kept.podcast_id);
|
||||
|
||||
a.stop().await;
|
||||
b.stop().await;
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Publish missing workspace versions together; Cargo orders their dependencies."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
INDEXES = {
|
||||
"crates-io": "https://index.crates.io/",
|
||||
"gitea": "https://gt.hexor.cy/api/packages/Furumi/cargo/",
|
||||
}
|
||||
|
||||
|
||||
def index_path(name):
|
||||
name = name.lower()
|
||||
if len(name) < 3:
|
||||
return f"{len(name)}/{name}"
|
||||
if len(name) == 3:
|
||||
return f"3/{name[0]}/{name}"
|
||||
return f"{name[:2]}/{name[2:4]}/{name}"
|
||||
|
||||
|
||||
def is_published(name, version, registry="crates-io"):
|
||||
request = urllib.request.Request(
|
||||
INDEXES[registry] + index_path(name),
|
||||
headers={"User-Agent": "frid-release-ci", "Cache-Control": "no-cache"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
versions = [json.loads(line) for line in response.read().splitlines()]
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code == 404:
|
||||
return False
|
||||
raise
|
||||
return any(row["vers"] == version for row in versions)
|
||||
|
||||
|
||||
def unpublished_packages(metadata, registry="crates-io"):
|
||||
members = set(metadata["workspace_members"])
|
||||
packages = []
|
||||
for package in sorted(metadata["packages"], key=lambda item: item["name"]):
|
||||
registries = package["publish"]
|
||||
if package["id"] not in members or (
|
||||
registries is not None and registry not in registries
|
||||
):
|
||||
continue
|
||||
name, version = package["name"], package["version"]
|
||||
if is_published(name, version, registry):
|
||||
print(f"Skip {name} {version}: already published", flush=True)
|
||||
else:
|
||||
print(f"Publish {name} {version}", flush=True)
|
||||
packages.append(name)
|
||||
return packages
|
||||
|
||||
|
||||
def wait_for_mirror_dependencies(metadata, selected):
|
||||
# Mirrored manifests retain their crates.io dependencies. In the parallel
|
||||
# upload job, wait for new workspace dependencies to reach crates.io before
|
||||
# Cargo resolves the mirror's packages (it does not compile them again).
|
||||
members = {p["name"]: p for p in metadata["packages"] if p["id"] in metadata["workspace_members"]}
|
||||
required = {
|
||||
(dep["name"], members[dep["name"]]["version"])
|
||||
for name in selected
|
||||
for dep in members[name]["dependencies"]
|
||||
if dep.get("path") and dep["name"] in members and dep["kind"] != "dev"
|
||||
}
|
||||
deadline = time.monotonic() + 600
|
||||
while required:
|
||||
required = {(name, version) for name, version in required if not is_published(name, version)}
|
||||
if not required:
|
||||
return
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError(f"Workspace dependencies did not reach crates.io: {sorted(required)}")
|
||||
print(f"Waiting for crates.io dependencies before mirroring: {sorted(required)}", flush=True)
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--plan", action="store_true", help="list packages without publishing")
|
||||
parser.add_argument("--dry-run", action="store_true", help="verify packages without uploading")
|
||||
parser.add_argument("--registry", choices=INDEXES, default="crates-io")
|
||||
parser.add_argument("--no-verify", action="store_true", help="publish packages already verified by CI without rebuilding")
|
||||
args = parser.parse_args()
|
||||
token_variable = "CARGO_REGISTRY_TOKEN" if args.registry == "crates-io" else "CARGO_REGISTRIES_GITEA_TOKEN"
|
||||
if not (args.plan or args.dry_run) and not os.environ.get(token_variable):
|
||||
parser.error(f"{token_variable} must be set for publishing")
|
||||
metadata = json.loads(subprocess.check_output(
|
||||
["cargo", "metadata", "--no-deps", "--format-version", "1", "--locked"],
|
||||
text=True,
|
||||
))
|
||||
packages = unpublished_packages(metadata, args.registry)
|
||||
if args.plan or not packages:
|
||||
return
|
||||
if args.registry == "gitea":
|
||||
wait_for_mirror_dependencies(metadata, packages)
|
||||
command = ["cargo", "publish", "--registry", args.registry, "--locked", "--all-features"]
|
||||
for name in packages:
|
||||
command.extend(["--package", name])
|
||||
if args.dry_run:
|
||||
command.append("--dry-run")
|
||||
if args.no_verify:
|
||||
command.append("--no-verify")
|
||||
# Multi-package publishing verifies against a temporary local registry,
|
||||
# then uploads and waits for dependencies to appear before their users.
|
||||
subprocess.run(command, check=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,82 @@
|
||||
import io
|
||||
import json
|
||||
import unittest
|
||||
import urllib.error
|
||||
from contextlib import redirect_stdout
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import publish_crates
|
||||
|
||||
|
||||
class PublishCratesTests(unittest.TestCase):
|
||||
def test_publish_job_does_not_rebuild_verified_packages(self):
|
||||
with patch("sys.argv", ["publish_crates.py", "--no-verify"]):
|
||||
with patch.dict(publish_crates.os.environ, {"CARGO_REGISTRY_TOKEN": "test-only"}):
|
||||
with patch.object(publish_crates.subprocess, "check_output", return_value=json.dumps({})):
|
||||
with patch.object(publish_crates, "unpublished_packages", return_value=["music-dht", "furumi-library"]):
|
||||
with patch.object(publish_crates.subprocess, "run") as run:
|
||||
publish_crates.main()
|
||||
run.assert_called_once_with([
|
||||
"cargo", "publish", "--registry", "crates-io", "--locked", "--all-features",
|
||||
"--package", "music-dht", "--package", "furumi-library", "--no-verify",
|
||||
], check=True)
|
||||
|
||||
def test_existing_and_private_versions_are_excluded_on_retry(self):
|
||||
packages = [
|
||||
{"id": "net", "name": "federation-net", "version": "0.3.1", "publish": None},
|
||||
{"id": "dht", "name": "music-dht", "version": "0.6.0", "publish": None},
|
||||
{"id": "lib", "name": "furumi-library", "version": "0.3.0", "publish": None},
|
||||
{"id": "demo", "name": "demo", "version": "0.1.0", "publish": []},
|
||||
{"id": "internal", "name": "internal", "version": "0.1.0", "publish": ["private"]},
|
||||
{"id": "external", "name": "external", "version": "0.1.0", "publish": None},
|
||||
]
|
||||
metadata = {"packages": packages, "workspace_members": [p["id"] for p in packages[:-1]]}
|
||||
for published, expected in [
|
||||
({"federation-net"}, ["furumi-library", "music-dht"]),
|
||||
({"federation-net", "music-dht"}, ["furumi-library"]),
|
||||
]:
|
||||
with self.subTest(published=published), redirect_stdout(io.StringIO()):
|
||||
with patch.object(publish_crates, "is_published", side_effect=lambda name, *args: name in published):
|
||||
self.assertEqual(publish_crates.unpublished_packages(metadata), expected)
|
||||
|
||||
def test_mirror_waits_for_workspace_dependencies_without_building(self):
|
||||
metadata = {"workspace_members": ["lib", "dht"], "packages": [
|
||||
{"id": "lib", "name": "furumi-library", "version": "0.3.0", "dependencies": [
|
||||
{"name": "music-dht", "path": "/workspace/music-dht", "kind": None},
|
||||
]},
|
||||
{"id": "dht", "name": "music-dht", "version": "0.6.0", "dependencies": []},
|
||||
]}
|
||||
with patch.object(publish_crates, "is_published", side_effect=[False, True]) as published:
|
||||
with patch.object(publish_crates.time, "sleep") as sleep, redirect_stdout(io.StringIO()):
|
||||
publish_crates.wait_for_mirror_dependencies(metadata, ["furumi-library"])
|
||||
self.assertEqual(published.call_count, 2)
|
||||
sleep.assert_called_once_with(10)
|
||||
|
||||
def test_gitea_uses_its_own_index(self):
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = b'{"vers":"0.3.1"}\n'
|
||||
with patch.object(publish_crates.urllib.request, "urlopen", return_value=response) as fetch:
|
||||
self.assertTrue(publish_crates.is_published("federation-net", "0.3.1", "gitea"))
|
||||
self.assertEqual(fetch.call_args.args[0].full_url,
|
||||
"https://gt.hexor.cy/api/packages/Furumi/cargo/fe/de/federation-net")
|
||||
|
||||
def test_registry_errors_do_not_look_like_unpublished_versions(self):
|
||||
for status in (403, 429, 500):
|
||||
error = urllib.error.HTTPError("https://index.crates.io/test", status, "error", {}, None)
|
||||
with self.subTest(status=status), patch.object(publish_crates.urllib.request, "urlopen", side_effect=error):
|
||||
with self.assertRaises(urllib.error.HTTPError):
|
||||
publish_crates.is_published("music-dht", "0.6.0")
|
||||
|
||||
def test_missing_crate_and_existing_version(self):
|
||||
error = urllib.error.HTTPError("https://index.crates.io/test", 404, "missing", {}, None)
|
||||
with patch.object(publish_crates.urllib.request, "urlopen", side_effect=error):
|
||||
self.assertFalse(publish_crates.is_published("new-crate", "0.1.0"))
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = b'{"vers":"0.6.0","yanked":false}\n'
|
||||
with patch.object(publish_crates.urllib.request, "urlopen", return_value=response):
|
||||
self.assertTrue(publish_crates.is_published("music-dht", "0.6.0"))
|
||||
self.assertFalse(publish_crates.is_published("music-dht", "0.7.0"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user