Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e47348b21d | ||
|
|
8af4d2cbbc | ||
|
|
84c5eb571c | ||
|
|
987063d616 | ||
|
|
3046c3f924 | ||
|
|
ddea053e9d | ||
|
|
9c82f6d916 | ||
|
|
912527a671 | ||
|
|
c0def74115 | ||
|
|
71e3f1fa0a | ||
|
|
5a4d418d2b | ||
|
|
81ccd924d0 | ||
|
|
fef155d6ef | ||
|
|
2fcb5f1822 | ||
|
|
d27155b257 | ||
|
|
d2530dca6c | ||
|
|
b8d8b532f2 | ||
|
|
06ac3691d8 | ||
|
|
9709f5e2b5 | ||
|
|
e627be51d4 | ||
|
|
49ae6423f6 | ||
|
|
e22c470ea7 | ||
|
|
35115ecc98 | ||
|
|
65d6ca2f14 | ||
|
|
085bff6191 | ||
|
|
629857dbfe | ||
|
|
4f7e999280 | ||
|
|
2d4782819d | ||
|
|
31d132bb3f | ||
|
|
38e49c4926 | ||
|
|
b191c10ba4 |
No files matched your search
@@ -0,0 +1,3 @@
|
||||
[registries.gitea]
|
||||
index = "sparse+https://gt.hexor.cy/api/packages/Furumi/cargo/"
|
||||
credential-provider = "cargo:token"
|
||||
@@ -2,6 +2,8 @@ name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "**"
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
@@ -20,6 +22,7 @@ jobs:
|
||||
toolchain: "1.97"
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: python3 -m unittest discover -s scripts -p 'test_*.py'
|
||||
- run: cargo fmt --all -- --check
|
||||
- run: cargo check --workspace --all-targets --all-features
|
||||
- run: cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
name: Publish crates
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v[0-9]*"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: crates-io-publish
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_NET_RETRY: 10
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.97"
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: python3 -m unittest discover -s scripts -p 'test_*.py'
|
||||
- run: cargo fmt --all -- --check
|
||||
- run: cargo check --workspace --all-targets --all-features --locked
|
||||
- run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
|
||||
- run: cargo test --workspace --locked
|
||||
- name: Verify release packages once
|
||||
run: python3 scripts/publish_crates.py --dry-run
|
||||
|
||||
publish-crates-io:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.97"
|
||||
- name: Publish to crates.io without rebuilding
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
run: python3 scripts/publish_crates.py --no-verify
|
||||
|
||||
publish-gitea:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.97"
|
||||
- name: Publish mirror to Gitea without rebuilding
|
||||
env:
|
||||
CARGO_REGISTRIES_GITEA_TOKEN: ${{ secrets.CARGO_REGISTRIES_GITEA_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$CARGO_REGISTRIES_GITEA_TOKEN" ]; then
|
||||
echo "::error::Add the CARGO_REGISTRIES_GITEA_TOKEN repository Actions secret, then rerun this workflow."
|
||||
exit 1
|
||||
fi
|
||||
export CARGO_REGISTRIES_GITEA_TOKEN="Bearer $CARGO_REGISTRIES_GITEA_TOKEN"
|
||||
python3 scripts/publish_crates.py --registry gitea --no-verify
|
||||
+111
-1
@@ -12,6 +12,7 @@ application
|
||||
furumi-library
|
||||
├── local catalog, import, migrations
|
||||
├── playlists, likes, history, similarity persistence
|
||||
├── podcast store (podcasts.sqlite3)
|
||||
│
|
||||
music-dht
|
||||
├── Kademlia routing and iterative lookup
|
||||
@@ -41,9 +42,30 @@ metadata import, artwork resolution, playlists, likes, listening history, and
|
||||
similarity index storage. Connections enable foreign keys, WAL, and a bounded
|
||||
busy timeout so independently running clients can safely share the database.
|
||||
|
||||
Podcasts use a separate store, `furumi_library::podcasts::PodcastStore`, in
|
||||
its own `podcasts.sqlite3` file next to the library database. It implements
|
||||
the podcast contract of `music_dht::podcast` for SQLite clients: the episode
|
||||
catalog refreshed from RSS, the subscription, share and progress registers,
|
||||
grow-only listening sessions, feed aliases and local-only fetch state.
|
||||
Local actions, received operations and snapshot sections share one apply
|
||||
path built on the `music-dht` materializers and merge functions, so the store
|
||||
converges independently of delivery order. Migrations follow
|
||||
`PRAGMA user_version`; the prototype podcast database is imported and its
|
||||
tables are kept as `legacy_*`; newer or foreign files are refused unchanged.
|
||||
`TrackItem.podcast` marks episode queue items, which music-only code must
|
||||
skip. An episode queue item is identified by its episode id
|
||||
(`TrackItem::key`, a typed source identity: library row, federated owner and
|
||||
item id, or episode id), never by the numeric id: episodes use the fixed
|
||||
sentinel id, `release_id` and artist id -1, which carries no identity and
|
||||
only keeps code written for library tracks (`id >= 0`) away from them.
|
||||
Received playback items are converted with `TrackItem::from_playback_track`,
|
||||
which never copies the wire ids (0 for episodes and private placeholders). See the "SQLite reference store" section
|
||||
of [PODCASTS.md](PODCASTS.md).
|
||||
|
||||
Filesystem scanning schedules, playback, federation lifecycle, and UI state
|
||||
remain application responsibilities. Database and metadata work must stay off
|
||||
interactive and async reactor threads.
|
||||
interactive and async reactor threads. RSS fetching and refresh scheduling are
|
||||
application work as well.
|
||||
|
||||
## Transport: federation-net
|
||||
|
||||
@@ -166,6 +188,89 @@ node.
|
||||
|
||||
### Extension protocols
|
||||
|
||||
Personal playback coordination is a pure domain state machine in
|
||||
`music_dht::playback`. It owns the common claim ordering, output-heartbeat
|
||||
rules, command fences, configurable startup/failover policy and generic
|
||||
snapshot/command envelopes. Applications supply authenticated events,
|
||||
monotonic time and actual output availability, persist scoped checkpoints,
|
||||
and apply the resolved owner to their UI/audio adapters. They must not elect
|
||||
an owner from an online-device list or reject a winning claim just because
|
||||
their local output is playing. The web gateway uses a passive server profile;
|
||||
browser startup and explicit selection are separate inputs.
|
||||
|
||||
This register provides eventual exclusivity after a partition heals, not a
|
||||
quorum lock during a partition. Device-sync profiles now advertise version 4
|
||||
(version 3 introduced coordination, version 4 adds podcast queue items);
|
||||
the additive coordination envelope is independently versioned and transported
|
||||
over the existing JSON `furumi/sync/2` stream. Legacy library records remain
|
||||
readable, but legacy unfenced personal playback messages cannot control an
|
||||
upgraded output. See [PLAYBACK_PROTOCOL.md](PLAYBACK_PROTOCOL.md) for the
|
||||
contract, adapter obligations, configuration and rollout rules.
|
||||
|
||||
Podcasts are a separate content type with a pure shared contract in
|
||||
`music_dht::podcast` (normative description: [PODCASTS.md](PODCASTS.md)). It
|
||||
owns canonical feed URLs, domain-separated BLAKE3 podcast and episode ids,
|
||||
feed aliases, the no-GUID episode rule, bounded metadata DTOs, the
|
||||
self-contained `PodcastEpisodeRef` queue item, and convergent merge functions
|
||||
for subscriptions, share decisions, progress registers and listening
|
||||
sessions. The tolerant RSS/Atom parser is shared behind the `podcast-feed`
|
||||
feature because episode identity depends on it. HTTP fetching, scheduling,
|
||||
buffering and decoding remain client runtime work, and persistence lives in
|
||||
client storage that implements the same rules. Podcast audio is fetched from
|
||||
the publisher: it is never imported, content-addressed or offered for P2P
|
||||
transfer.
|
||||
|
||||
Device-sync protocol v4 carries podcast state between trusted devices only:
|
||||
five `podcast_*` operations, bounded snapshot sections (the recovery path,
|
||||
because sync vectors never re-request operations hidden from an older peer;
|
||||
past the bounds an upgraded peer recovers only the newest records), and an
|
||||
explicit `PlaybackTrack.source`. The operation enum is closed and v3 clients
|
||||
ignore `source`, so senders gate on the version each peer reported itself
|
||||
(relayed profiles only raise it; pairing requests use the version in the
|
||||
invite); podcast state is never sent to peers below v4, handoff to them is
|
||||
refused, and their playback commands are ignored while the queue contains
|
||||
episodes. Jam snapshots replace episodes with opaque placeholders. Progress,
|
||||
subscription and share registers order by a Lamport generation before
|
||||
wall-clock `hlc_ms`, so a device with a fast clock cannot undo an observed
|
||||
seek, restart, unsubscribe or share decision. The music DHT schema and
|
||||
`federation-net` are unchanged.
|
||||
|
||||
Public podcast discovery is a third schema-independent Kademlia overlay,
|
||||
`music_dht::podcast_dht` (ALPN `furumi-fd/podcast-dht/1`, version 1,
|
||||
capability `podcast_dht`), built like the similarity-routing overlay. A
|
||||
device publishes a card only for podcasts whose merged registers satisfy
|
||||
`effective_share` (SQLite clients read the merged registers from
|
||||
`PodcastStore`); `sync_shares` reconciles publications declaratively and
|
||||
nothing is sent for a podcast the device never published. Own cards persisted
|
||||
by an earlier run are neither republished nor served until the first
|
||||
`sync_shares` of the process, so a stale cache cannot revive a share revoked
|
||||
elsewhere while the device was offline. A card is the
|
||||
public projection of a podcast (feed URL and channel metadata, no personal
|
||||
state), signed by the device's iroh identity over a domain-separated
|
||||
canonical encoding. Only the origin URL the id hashes from is bound to the
|
||||
podcast; a moved `feed_url` is an unverified publisher claim that search never
|
||||
offers as the podcast's feed. Cards live under the podcast-id key and one key
|
||||
per normalized title/author token; unpublishing writes a signed tombstone of
|
||||
the next revision that supersedes lower and equal revisions on replicas that
|
||||
hold it, for the keys it lists, and outlives every older card. Only a record's
|
||||
publisher may store it; replicas keep one payload per publisher and podcast,
|
||||
admit placements only for keys they are close to (counting only contacts that
|
||||
answered or authenticated, never ones another peer suggested), within per-key,
|
||||
per-publisher and total budgets. A key holds no more records than one response
|
||||
carries; a full key or cache admits a newcomer by evicting a random record from
|
||||
its younger half, so the older half survives floods and capacity is never
|
||||
first-come-forever (over their budget, tombstones only displace other tombstones). Records use the music DHT's
|
||||
active/tombstone TTLs and republish cadence, are verified on every receive and
|
||||
cache read, and are kept in a separate, replaceable SQLite cache. Search
|
||||
filters on every replica by all query tokens, merges cards of one podcast
|
||||
across publisher devices, and returns typed `podcast` hits that clients merge
|
||||
with music results; publisher counts are not a trust signal. The publisher key
|
||||
is the device's transport identity, so a card is linkable to the device's
|
||||
network address and music-DHT records. Applications advertise the
|
||||
`podcast_dht` capability only when they run the overlay; capability data only
|
||||
helps skip peers, it never authorizes. See the "Public discovery" section of
|
||||
[PODCASTS.md](PODCASTS.md).
|
||||
|
||||
Music discovery identifies an owner and content; it does not force all domain
|
||||
traffic through DHT messages. `music-dht` exposes the underlying authenticated
|
||||
stream capability and defines shared wire models for Furumi catalog and device
|
||||
@@ -265,3 +370,8 @@ architectural constraint:
|
||||
- Local durable state survives ordinary peer and network failure.
|
||||
- Additional stream protocols reuse authenticated connectivity without
|
||||
coupling bulk data to the typed control channel.
|
||||
- Private personal state (podcast subscriptions, progress, history) stays in
|
||||
the trusted-device group; Jam and public overlays only see explicit public
|
||||
projections or opaque placeholders.
|
||||
- A device's protocol version is what that device reported; relays never
|
||||
substitute their own.
|
||||
@@ -1,5 +1,104 @@
|
||||
# Changelog
|
||||
|
||||
## v0.6.0 — 2026-11-20
|
||||
|
||||
Releases `music-dht 0.6.0` and `furumi-library 0.3.0`.
|
||||
`federation-net` remains at `0.3.1`.
|
||||
|
||||
- Tag-driven CI verifies release packages once, then publishes to crates.io
|
||||
and the Furumi Gitea Cargo mirror in parallel jobs without recompiling.
|
||||
Already published versions are skipped independently in each registry.
|
||||
Authentication uses `CARGO_REGISTRY_TOKEN` and `CARGO_REGISTRIES_GITEA_TOKEN`.
|
||||
|
||||
### Podcast discovery (`podcast_dht`)
|
||||
|
||||
- `music-dht 0.6.0`: new `music_dht::podcast_dht` overlay for public podcast
|
||||
discovery on its own ALPN `furumi-fd/podcast-dht/1` (version 1) and its own
|
||||
SQLite cache (schema 2): signed `PodcastCard`s and `PodcastTombstone`s
|
||||
(`furumi-podcast-card:v1` / `furumi-podcast-card-tombstone:v1` canonical
|
||||
signing input), monotonic revisions with a total tie-break, text and id
|
||||
keys, `PodcastDht::{publish, unpublish, sync_shares, republish, search,
|
||||
lookup_podcast}` gated by `effective_share`, typed search hits
|
||||
(`PodcastSearchHit`, `ClaimedFeedUrl`, `FederatedSearchHit`,
|
||||
`SearchHitKind`, `merge_search_results`) and the advisory
|
||||
`feed_url_privacy_warning`.
|
||||
- Replicas accept records only from their publisher, store one payload per
|
||||
`(podcast_id, publisher)`, admit placements only for keys they are close to
|
||||
(counting verified contacts only) and within per-key, per-publisher and
|
||||
total budgets. A key never holds more than one response carries; full keys
|
||||
and caches admit newcomers by evicting random records from their younger
|
||||
half. Lookups give each response a bounded share of owners. Search offers
|
||||
only the URL the podcast id hashes from; moved-feed URLs are reported as
|
||||
unverified claims.
|
||||
- `sync_shares` is fail-safe for revocation (per-podcast `failed`) and holds
|
||||
back cards with privacy warnings until confirmed (`held_back`,
|
||||
`PodcastShareState::privacy_confirmed`).
|
||||
- Own cards persisted by an earlier run are neither republished (by
|
||||
`maintenance` or `republish`) nor served to lookups until the first
|
||||
`sync_shares` of the process, so a share revoked on another trusted device
|
||||
while this one was offline is not re-sent on startup. That first call also
|
||||
re-sends unchanged cards; tombstones are always re-sent.
|
||||
- `music-dht 0.6.0`: capability manifests can advertise `podcast_dht = 1`
|
||||
(`capabilities::PODCAST_DHT_ID`, `CapabilityManifest::with_podcast_dht`,
|
||||
`CapabilityManifest::protocol_version`,
|
||||
`CapabilityManifest::supports_podcast_dht`). It is opt-in:
|
||||
`CapabilityManifest::frid()` does not include it.
|
||||
- `music-dht 0.6.0`: new `MusicDhtError::PodcastPublication` variant (the enum
|
||||
is `#[non_exhaustive]`).
|
||||
- The music DHT (`music-dht-v5`), `federation-net` and device-sync protocol v4
|
||||
are unchanged. See "Public discovery (podcast_dht)" in `PODCASTS.md`.
|
||||
|
||||
### Podcast contract and storage
|
||||
|
||||
- `music-dht 0.6.0` (breaking): new `music_dht::podcast` module with the shared
|
||||
podcast contract: canonical `FeedUrl`, `PodcastId`/`EpisodeId` derivation
|
||||
(`furumi-podcast:*:v1`), feed aliases, the no-GUID episode identity rule,
|
||||
`PodcastMeta`/`EpisodeMeta`/`PodcastEpisodeRef`, and convergent merge rules
|
||||
for subscriptions, shares, progress registers and listening sessions.
|
||||
- `music-dht 0.6.0`: optional `podcast-feed` feature with the shared tolerant
|
||||
RSS/Atom parser (`quick-xml`, `encoding_rs`, `url`).
|
||||
- `music-dht 0.6.0`: device-sync protocol version 4 (ALPN unchanged): five
|
||||
`podcast_*` operation kinds (subscription, share, feed aliases, progress,
|
||||
listening sessions), bounded podcast snapshot sections,
|
||||
`PlaybackTrack.source`, reserved `PlaybackStateWire.playback_rate`,
|
||||
`DeviceProfileSet.protocol_version`, optional `InviteWire` protocol version
|
||||
(`p`), self-reported vs relayed version helpers, version-gating helpers and
|
||||
Jam redaction. New struct fields and enum variants break exhaustive
|
||||
construction and matching in clients.
|
||||
- `furumi-library 0.3.0`: follows `music-dht 0.6.0`, whose types it re-exposes.
|
||||
- `furumi-library 0.3.0`: new `furumi_library::podcasts::PodcastStore`, the
|
||||
SQLite reference store of the podcast contract in its own
|
||||
`podcasts.sqlite3`: `user_version` migrations that import the prototype
|
||||
podcast database (tables kept as `legacy_*`, unimportable rows in
|
||||
`legacy_quarantine`) and refuse newer or foreign files unchanged; podcasts,
|
||||
feed aliases, seasons, episodes, subscription/share/progress registers,
|
||||
listening sessions and local-only fetch state; local actions that return
|
||||
the `podcast_*` operations for the client's sync log; `apply_remote_op`,
|
||||
bounded snapshot export and merge; RSS refresh ingestion with stable
|
||||
episode ids; UI queries. Enables the `music-dht` `podcast-feed` feature.
|
||||
- `furumi-library 0.3.0` (breaking): `TrackItem.podcast:
|
||||
Option<PodcastEpisodeRef>` and `TrackItem::podcast_episode`/`is_podcast`.
|
||||
Exhaustive `TrackItem` literals in clients need the new field; music-only
|
||||
code must skip podcast items. Podcast items use the fixed sentinel
|
||||
`TrackItem::NOT_A_LIBRARY_ROW` (-1) as `id`, `release_id` and artist id,
|
||||
so existing `id < 0` / `release_id < 0` checks already treat them as
|
||||
non-library items; the sentinel carries no identity. New
|
||||
`TrackItem::key` / `TrackItemKey` (`lib:<id>`, `fed:<owner>:<item_id>`,
|
||||
`ep:<episode id>`), the source identity: episodes must be keyed by it (or
|
||||
`TrackItem.podcast`), never by `id`; content-first music matching stays
|
||||
as it is. New `TrackItem::from_playback_track` / `PlaybackItem` convert
|
||||
received playback items without copying wire ids (0 for episodes and Jam
|
||||
private placeholders). The library database is unchanged.
|
||||
- `furumi-library 0.3.0`: prototype import collapses rows of one GUID-less
|
||||
item under several enclosure URLs, and the first refresh re-keys imported
|
||||
GUID-less episodes without personal state to the ids the current feed
|
||||
gives, so they match devices that never had the prototype; when another
|
||||
episode (for example one created by a remote operation) already holds
|
||||
that id, the imported copy is deleted instead of kept as a duplicate.
|
||||
Fetch-state
|
||||
writes are atomic across threads and processes.
|
||||
- See `PODCASTS.md` for the normative contract.
|
||||
|
||||
## v0.4.1 — 2026-09-02
|
||||
|
||||
- `federation-net 0.3.1`: recover timed-out or failed Mainline-DHT rendezvous clients and expose network health to applications.
|
||||
|
||||
Generated
+32
-2
@@ -389,6 +389,12 @@ version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "core_detect"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
@@ -790,6 +796,20 @@ version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"core_detect",
|
||||
"multiversion_no_op",
|
||||
"rustversion",
|
||||
"scopeguard",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "enum-assoc"
|
||||
version = "1.3.0"
|
||||
@@ -912,7 +932,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "furumi-library"
|
||||
version = "0.1.0"
|
||||
version = "0.3.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"blake3",
|
||||
@@ -922,6 +942,7 @@ dependencies = [
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -2071,18 +2092,26 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multiversion_no_op"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
|
||||
|
||||
[[package]]
|
||||
name = "music-dht"
|
||||
version = "0.4.1"
|
||||
version = "0.6.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"blake3",
|
||||
"data-encoding",
|
||||
"encoding_rs",
|
||||
"federation-net",
|
||||
"futures",
|
||||
"iroh",
|
||||
"postcard",
|
||||
"quick-xml",
|
||||
"rand 0.9.5",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
@@ -2092,6 +2121,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tracing",
|
||||
"unicode-normalization",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+4
-1
@@ -14,7 +14,7 @@ repository = "https://gt.hexor.cy/ab/frid"
|
||||
|
||||
[workspace.dependencies]
|
||||
federation-net = { path = "crates/federation-net", version = "0.3.1" }
|
||||
music-dht = { path = "crates/music-dht", version = "0.4.1" }
|
||||
music-dht = { path = "crates/music-dht", version = "0.6.0" }
|
||||
iroh = "1"
|
||||
iroh-base = "1"
|
||||
iroh-tickets = "1"
|
||||
@@ -38,3 +38,6 @@ lofty = "0.22"
|
||||
data-encoding = "2"
|
||||
rand = "0.9"
|
||||
tempfile = "3"
|
||||
quick-xml = "0.41"
|
||||
encoding_rs = "0.8.35"
|
||||
url = "2.5"
|
||||
@@ -0,0 +1,167 @@
|
||||
# Personal playback coordination
|
||||
|
||||
`music_dht::playback` owns the personal-device state machine, policy parameters,
|
||||
snapshot/command envelopes and conflict rules. It has no networking, database,
|
||||
clock, UI or audio dependency. The existing JSON-lines device-sync transport
|
||||
continues to carry snapshots and logged commands. Jam is a separate authority
|
||||
scope; its capability-based protocol is not a personal ownership claim.
|
||||
|
||||
## Contract
|
||||
|
||||
Each logical device identity must have one coordinator and one selected output
|
||||
adapter. Independent live installations must not reuse a device identity or
|
||||
checkpoint. The web hub is a single authority for its user's browser outputs;
|
||||
running multiple independent hubs for that same identity requires an external
|
||||
actor/leader arrangement, not two copies of this in-memory adapter.
|
||||
|
||||
An owner is selected by an immutable `Claim { counter, issuer, owner }`.
|
||||
Claims are ordered lexicographically in that order. A locally issued claim
|
||||
increments the greatest counter observed so far. Persist the winning claim;
|
||||
never reset it because a peer disappears from an online list. Controllers
|
||||
gossip the winning claim as well. Concurrent startup, transfer or failover
|
||||
claims therefore converge to the same winner regardless of delivery order.
|
||||
|
||||
Only the winning output may render personal audio. A losing player stops its
|
||||
audio even if it was playing; it cannot reject a higher claim on that basis.
|
||||
This is eventual exclusivity, not a quorum lock: partitions can temporarily
|
||||
produce two outputs. Convergence assumes eventual delivery between trusted
|
||||
devices, functioning polling/event loops, and no continuing stream of explicit
|
||||
conflicting user transfers. A disconnected browser cannot be forcibly silenced.
|
||||
|
||||
Availability, actual unpaused playback, ownership and network connectivity are
|
||||
different inputs. Only new reports from the owner's audio endpoint renew its
|
||||
liveness. Duplicate snapshots and relayed claims do not. Gateway timers must
|
||||
not turn cached browser state into new output heartbeats. The engine reserves
|
||||
heartbeat sequence ranges in its checkpoint so a restart cannot roll them back
|
||||
and every UI tick does not require a database write.
|
||||
|
||||
Commands carry `CommandStamp { version, claim, sequence }`. A command can carry
|
||||
its existing claim ahead of the corresponding announcement. An explicit
|
||||
handoff must name that claim's owner and originate from its issuer. In a term,
|
||||
commands are applied in increasing sequence per sender; old, duplicate and
|
||||
incompatible envelopes are ignored. The command-sender map is bounded to 128
|
||||
entries per term. Concurrent controllers are serialized by the selected
|
||||
output, whose snapshots distribute the resulting queue and transport state.
|
||||
This does not claim exactly-once audio effects across process crashes.
|
||||
|
||||
## Adapter obligations
|
||||
|
||||
1. Authenticate the peer and trusted group using the existing transport and
|
||||
operation-log membership checks. Bind snapshot `device_id` to the actual
|
||||
sender. Names and snapshot wall-clock timestamps do not grant ownership.
|
||||
2. Restore `Checkpoint` only for its matching group. Construct `Engine` with a
|
||||
stable device id, local `Config` and a monotonic clock. Do not restore wall
|
||||
clock liveness deadlines.
|
||||
3. Feed real output availability and activity. Use `heartbeat` for a local
|
||||
output or `output_report` with an increasing source-report id for a gateway.
|
||||
4. Feed announcements, tick the engine, and route explicit transfers through
|
||||
`transfer`. Ordinary controls only stamp a command in the current term.
|
||||
5. Persist changed durable state before publishing decisions or executing
|
||||
commands. Serialize transitions with checkpoint commits and restore the
|
||||
previous engine if persistence fails. Recheck queued command fences before
|
||||
their effects reach the output.
|
||||
6. Apply the resolved owner to the existing UI/audio adapter. Gossip its claim
|
||||
even while controlling another output. Online lists are presentation only.
|
||||
|
||||
Startup has a discovery grace period. If a restored/relayed owner has not yet
|
||||
reported, keep the startup decision pending until its heartbeat or timeout.
|
||||
Discovery is necessarily bounded: an entirely unknown peer across a partition
|
||||
cannot prevent a local startup claim. Once the peers communicate, normal claim
|
||||
ordering resolves the conflict.
|
||||
|
||||
## Configuration
|
||||
|
||||
The complete serde `Config` accepts these fields (unknown fields are errors):
|
||||
|
||||
| Field | Default | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `claim_on_startup` | `true` | One startup intent for this application instance |
|
||||
| `automatic_failover` | `true` | Claim when the owner stops reporting |
|
||||
| `take_paused_on_startup` | `true` | Startup may take a paused/idle output |
|
||||
| `discovery_ms` | `3000` | Initial discovery grace |
|
||||
| `owner_timeout_ms` | `120000` | Missing fresh output reports before failover |
|
||||
|
||||
`Config::passive()` disables application-startup claims and automatic failover.
|
||||
Furumusic uses this for the always-on server. A real browser startup supplies a
|
||||
separate one-shot `request_startup`; explicit selection supplies a transfer.
|
||||
These events are not generated by server polling. Local browser selection uses
|
||||
the same `Config::should_claim` policy, with automatic cross-federation claims
|
||||
disabled. Browser presence expiry preserves the current owner and queue.
|
||||
|
||||
TUI exposes the configuration as the `playback` section of its existing settings
|
||||
file and preserves it when saving other settings. The web adapter reads optional
|
||||
`playback_config_json` on the user's `furumusic__fed_device_identity` row at
|
||||
session initialization; null selects the passive profile. Audio availability
|
||||
is a runtime input, not a persisted setting.
|
||||
|
||||
## Versioning and rollout
|
||||
|
||||
The library-sync JSON ALPN remains `furumi/sync/2`; catalog, pairing and library
|
||||
data do not require a new transport. Device profiles advertised protocol
|
||||
version 3 when coordination was introduced (version 4 adds podcast queue items,
|
||||
see below). Coordination envelopes independently carry version 1. Snapshot coordination
|
||||
and operation authority are additive optional JSON fields so library-sync
|
||||
records from older clients still deserialize. Upgraded clients ignore legacy
|
||||
personal playback snapshots/commands without the coordination envelope.
|
||||
The exclusivity guarantee applies after all audio clients have upgraded.
|
||||
|
||||
The shared generic `Snapshot<S>` and `Command<S>` keep state conversion in the
|
||||
applications while preventing divergent control message definitions. These
|
||||
envelopes use JSON, not a backwards-compatible promise for postcard layouts.
|
||||
Do not change Jam's independent authority by interpreting personal stamps there.
|
||||
|
||||
## Protocol v4: podcast queue items
|
||||
|
||||
Device profiles advertise protocol version 4. The JSON ALPN stays
|
||||
`furumi/sync/2` and the coordination envelope stays version 1. Version 4 adds
|
||||
to the portable queue DTO:
|
||||
|
||||
- `PlaybackTrack.source`: `music`, `podcast_episode` with a self-contained
|
||||
`PodcastEpisodeRef`, or `private` (Jam placeholder). It is omitted for
|
||||
music, so music-only state serializes exactly as in v3. Podcast episodes are
|
||||
never encoded as negative ids or URLs in `file_path`. Queue items carry
|
||||
`PodcastEpisodeRef.progress_generation`, the greatest progress generation
|
||||
the sender has seen for the episode.
|
||||
- `PlaybackStateWire.playback_rate`: reserved, omitted when unset. Treat a
|
||||
missing or unsupported value as 1.0.
|
||||
|
||||
A v3 client does not know `source` and ignores it: it would treat an episode
|
||||
as a broken music item (id 0, no content id) and forward the queue that way,
|
||||
losing the episode for every device. `PlaybackSource` is also a closed enum
|
||||
for v4 clients, so a future variant needs a new protocol version. Adapters
|
||||
therefore apply the shared helpers from `music_dht::device_sync`:
|
||||
|
||||
1. Send a playback snapshot to a peer only when
|
||||
`playback_allowed_for_peer(state, peer_version)` holds; otherwise send
|
||||
`None`.
|
||||
2. Refuse an explicit transfer to a device below v4 while the queue contains
|
||||
podcast items and show that the target needs an update. The decision uses
|
||||
the version that device reported itself (`profile_protocol_version`;
|
||||
profiles relayed by other devices only raise it, `relayed_protocol_version`),
|
||||
never the local constant. Do not strip episodes from the queue.
|
||||
3. Ignore `SetState`/`ActiveChanged` from peers below v4 while the local
|
||||
queue contains podcast items (`playback_command_allowed_from_peer`).
|
||||
4. A stamped `playback_command` operation with podcast items is itself gated
|
||||
by `op_allowed_for_peer`.
|
||||
5. A `PairRequest` is sent before the invite consumer hears from the inviter:
|
||||
gate its playback snapshot by `InviteWire::inviter_protocol_version()`
|
||||
(version 1 for invites without one).
|
||||
|
||||
A handoff of a podcast episode records the position and the play/pause intent
|
||||
first, transfers the queue with its sources, and lets the receiver open the
|
||||
publisher's HTTP source and seek. Ownership still changes only through the
|
||||
claim, so a late response cannot make the previous owner resume audio. The
|
||||
durable episode progress is a separate register; see `PODCASTS.md`. Only the
|
||||
receiver writes the handoff's explicit progress action, with
|
||||
`ProgressRecord::explicit_generation(max(local, progress_generation))`; the
|
||||
sender writes a final checkpoint at its session generation.
|
||||
|
||||
Jam hosts send `music_dht::jam::redact_jam_snapshot`, which replaces podcast
|
||||
items with opaque `private` placeholders and drops the personal coordination
|
||||
envelope. They reject participant commands with real episodes
|
||||
(`jam_command_allowed`) and map placeholders back with
|
||||
`restore_jam_private_items` before applying a participant's state.
|
||||
|
||||
For local integration checks, patch the three frid crates with Cargo's external
|
||||
`--config` mechanism. Keep released dependency declarations and lockfiles in
|
||||
the applications; publish frid and then bump client dependencies for rollout.
|
||||
+1043
File diff suppressed because it is too large.
Load diff
@@ -13,7 +13,7 @@ The workspace has three layers:
|
||||
- [`music-dht`](crates/music-dht) builds a Kademlia-style distributed music
|
||||
directory on top of that transport, including catalog discovery, content-id
|
||||
lookup, signed similarity-LSH routing, direct byte streams, and shared Furumi
|
||||
wire types.
|
||||
wire types, including the shared podcast contract ([PODCASTS.md](PODCASTS.md)).
|
||||
- [`furumi-library`](crates/furumi-library) owns the shared local `library.db`
|
||||
contract, migrations, catalog operations, metadata import, artwork,
|
||||
playlists, likes, history, and similarity persistence used by Furumi clients.
|
||||
@@ -50,13 +50,13 @@ crates/
|
||||
|
||||
## Using the libraries
|
||||
|
||||
Until crates are published to a registry, depend on the repository directly:
|
||||
Use the published crates from crates.io:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
federation-net = { git = "https://gt.hexor.cy/ab/frid.git" }
|
||||
music-dht = { git = "https://gt.hexor.cy/ab/frid.git" }
|
||||
furumi-library = { git = "https://gt.hexor.cy/ab/frid.git" }
|
||||
federation-net = "0.3.1"
|
||||
music-dht = "0.6.0"
|
||||
furumi-library = "0.3.0"
|
||||
```
|
||||
|
||||
Start with the crate documentation:
|
||||
@@ -80,6 +80,48 @@ The integration tests open real local sockets and establish iroh connections.
|
||||
Some rendezvous tests also use public discovery infrastructure, so a restricted
|
||||
sandbox may need explicit network permission.
|
||||
|
||||
## Publishing
|
||||
|
||||
Add these repository Actions secrets:
|
||||
|
||||
- `CARGO_REGISTRY_TOKEN`: a crates.io API token with publish access to
|
||||
`federation-net`, `music-dht` and `furumi-library`.
|
||||
- `CARGO_REGISTRIES_GITEA_TOKEN`: a raw Gitea personal access token with
|
||||
`write:package` and permission to publish under the `Furumi` organization.
|
||||
Create it under user Settings → Applications. The workflow adds the
|
||||
required `Bearer` prefix itself; no username secret is needed.
|
||||
|
||||
Update changed crate versions, workspace dependency requirements, `Cargo.lock`
|
||||
and `CHANGELOG.md`, then merge to `main`. Create `<crate>-v<version>` tags for
|
||||
the changed crates and one workspace release tag, for example:
|
||||
|
||||
```bash
|
||||
git tag -a music-dht-v0.6.0 -m 'music-dht 0.6.0'
|
||||
git tag -a furumi-library-v0.3.0 -m 'furumi-library 0.3.0'
|
||||
git tag -a v0.6.0 -m 'Frid podcast release'
|
||||
git push origin music-dht-v0.6.0 furumi-library-v0.3.0 v0.6.0
|
||||
```
|
||||
|
||||
Only the `v*` workspace tag triggers `.github/workflows/publish.yml`; the
|
||||
crate tags identify each package release without starting duplicate uploads.
|
||||
The workflow runs the workspace checks and verifies the release packages in
|
||||
one build job. Two publishing jobs then upload without rebuilding
|
||||
(`cargo publish --no-verify`), selecting missing versions independently in
|
||||
crates.io and the Gitea mirror at
|
||||
`https://gt.hexor.cy/api/packages/Furumi/cargo/`. The mirror includes all three
|
||||
crates, including an unchanged version missing there. Already published
|
||||
versions are skipped on retries; a registry lookup error fails the release
|
||||
instead of assuming absence. Mirrored manifests retain crates.io dependency
|
||||
references, so the mirror waits up to ten minutes for new workspace
|
||||
dependencies to appear in crates.io before packaging its uploads.
|
||||
|
||||
Preview the selection with `python3 scripts/publish_crates.py --plan`, or
|
||||
verify the packages without uploading with `--dry-run`. Add `--registry gitea`
|
||||
to preview or publish to the mirror. If a secret was
|
||||
missing or a publish failed, fix the cause and rerun the existing tag workflow;
|
||||
do not move release tags. Manual dispatch must select the release tag as its
|
||||
ref. Publication is performed by CI, not by the tagging commands above.
|
||||
|
||||
## License
|
||||
|
||||
Frid is released under the
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "furumi-library"
|
||||
version = "0.1.0"
|
||||
version = "0.3.0"
|
||||
description = "Client-independent SQLite music library for Furumi players"
|
||||
readme = "README.md"
|
||||
documentation = "https://docs.rs/furumi-library"
|
||||
@@ -16,8 +16,11 @@ anyhow = { workspace = true }
|
||||
blake3 = { workspace = true }
|
||||
directories = { workspace = true }
|
||||
lofty = { workspace = true }
|
||||
music-dht = { workspace = true }
|
||||
music-dht = { workspace = true, features = ["podcast-feed"] }
|
||||
rusqlite = { workspace = true, features = ["functions"] }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
@@ -19,3 +19,69 @@ publication, clients can depend on the released crate version instead.
|
||||
`Library::open(default_db_path()?)` opens the same platform-native Furumi
|
||||
library used by every client. Connections use WAL, foreign keys, and a busy
|
||||
timeout so independently running clients can safely share the database.
|
||||
|
||||
Version 0.3 follows `music-dht` 0.6, whose device-sync and podcast contract
|
||||
types it re-exposes.
|
||||
|
||||
## Podcasts
|
||||
|
||||
`furumi_library::podcasts::PodcastStore` is the SQLite reference store of the
|
||||
podcast contract (workspace `PODCASTS.md`, section "SQLite reference store").
|
||||
It lives in its own file, conventionally `podcasts.sqlite3`
|
||||
(`PODCAST_DB_FILE`) next to `library.db`; the caller passes the path.
|
||||
|
||||
- **Migrations** run on `PRAGMA user_version`, each in its own transaction.
|
||||
A database of the original podcast prototype (`user_version` 1) is imported:
|
||||
its tables are renamed `legacy_podcasts` / `legacy_podcast_episodes` (never
|
||||
dropped), feeds become subscriptions with contract ids, rows that cannot be
|
||||
imported are listed in `legacy_quarantine`. A database from a newer Furumi,
|
||||
or a file that is not a podcast database, is refused without being touched.
|
||||
- **Local actions** (`subscribe`, `unsubscribe`, `set_shared`,
|
||||
`start_session`, `checkpoint`, `seek`, `restart`, `complete_session`,
|
||||
`mark_played`, `mark_unplayed`, `record_listen_session`) draw stamps from a
|
||||
`StampSource` (usually the client's sync log), apply the operation and
|
||||
return `PodcastLocalOp`s to append to the log with exactly those stamps.
|
||||
- **Sync**: `apply_remote_op` applies the five `podcast_*` operations,
|
||||
`fill_snapshot` / `export_snapshot` produce the bounded snapshot sections
|
||||
and `apply_snapshot` merges them. Everything goes through the `music-dht`
|
||||
materializers and merge functions, so the result does not depend on
|
||||
delivery order and never loses newer local state.
|
||||
- **RSS**: `apply_feed` ingests a document parsed by
|
||||
`music_dht::podcast::feed` (the crate enables the `podcast-feed` feature):
|
||||
channel metadata, episodes with stable ids and current audio URLs, feed
|
||||
aliases from permanent redirects and `itunes:new-feed-url`. It never
|
||||
touches subscriptions, progress or history. HTTP, conditional requests and
|
||||
scheduling stay in the client; `record_fetch_*`, `fetch_state` and
|
||||
`podcasts_due_for_refresh` keep the local-only fetch state, which is never
|
||||
synced.
|
||||
- **Queries**: `list_podcasts`, `seasons`, `episodes` (season, played-state
|
||||
and text filters, date or episode-number order), `resume_position_ms`,
|
||||
`episode_ref` / `episode_track` for queueing, `listen_history`.
|
||||
- **Public discovery**: the store records share decisions only. Build each
|
||||
`music_dht::podcast_dht::PodcastShareState` for `PodcastDht::sync_shares`
|
||||
from `subscription` and `share` (workspace `PODCASTS.md`, section "Public
|
||||
discovery (podcast_dht)"); the client keeps the user's `privacy_confirmed` decision.
|
||||
|
||||
The store is synchronous: methods take `&self` and lock one connection, so
|
||||
`Arc<PodcastStore>` can be shared; run calls on a blocking pool.
|
||||
|
||||
`TrackItem.podcast` marks a queue item as a podcast episode
|
||||
(`TrackItem::podcast_episode`). Music-only code (likes, playlists, history,
|
||||
content ids, federation downloads) must check `TrackItem::is_podcast`; the
|
||||
music library never stores episodes.
|
||||
|
||||
`TrackItem::key` returns the typed source identity of a queue item
|
||||
(`TrackItemKey`: `lib:<id>`, `fed:<owner>:<item_id>` or `ep:<episode id>` as
|
||||
a string). A negative `TrackItem::id` only means "not a library row". Every
|
||||
podcast item has the same sentinel `id`, `release_id` and artist id
|
||||
(`TrackItem::NOT_A_LIBRARY_ROW` = -1). It carries no identity and must not
|
||||
be decoded; it only keeps music-only code that predates podcasts from taking
|
||||
an episode for library track, release or artist 0. The episode's identity is
|
||||
`TrackItem.podcast` alone, so queue, current-track and position matching
|
||||
must check it before any fallback to the id. `TrackItemKey` does not equate
|
||||
a pending federated item with its library copy; clients that match music by
|
||||
content id keep doing so.
|
||||
|
||||
Convert received `PlaybackTrack`s with `TrackItem::from_playback_track`
|
||||
(`PlaybackItem::Music`, `PodcastEpisode` or `Private`): podcast items and
|
||||
Jam private placeholders carry id 0 on the wire, which must never be copied.
|
||||
@@ -6,9 +6,13 @@
|
||||
//!
|
||||
//! All methods take `&self` and lock the connection internally, so an
|
||||
//! `Arc<Library>` can be shared across blocking tasks.
|
||||
//!
|
||||
//! Podcasts live in a separate database, [`podcasts::PodcastStore`]
|
||||
//! (`podcasts.sqlite3`); the music library never stores episodes.
|
||||
|
||||
pub mod import;
|
||||
pub mod models;
|
||||
pub mod podcasts;
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Write as _;
|
||||
@@ -22,8 +26,8 @@ use rusqlite::{Connection, OptionalExtension as _, params};
|
||||
|
||||
pub use models::{
|
||||
ArtistCard, ArtistDetail, ArtistRef, ArtistsPage, Availability, FederatedTrack, LibraryFilters,
|
||||
LibrarySourceMode, PlaylistCard, PlaylistDetail, ReleaseCard, ReleaseDetail, ReleaseEdit,
|
||||
SearchResults, TrackEdit, TrackItem,
|
||||
LibrarySourceMode, PlaybackItem, PlaylistCard, PlaylistDetail, ReleaseCard, ReleaseDetail,
|
||||
ReleaseEdit, SearchResults, TrackEdit, TrackItem, TrackItemKey,
|
||||
};
|
||||
|
||||
/// The virtual "Liked tracks" playlist id, kept from the server API.
|
||||
@@ -3283,6 +3287,7 @@ fn query_tracks(
|
||||
artists: Vec::new(),
|
||||
featured_artists: Vec::new(),
|
||||
fed: None,
|
||||
podcast: None,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
@@ -3348,6 +3353,7 @@ fn pending_track(fed: &FederatedTrack) -> TrackItem {
|
||||
file_size_bytes: None,
|
||||
play_count: 0,
|
||||
fed: Some(fed.clone()),
|
||||
podcast: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
//! Data shapes the views render. They mirror what the furumusic API used to
|
||||
//! return, but every field is now filled from the local SQLite library.
|
||||
|
||||
use music_dht::device_sync::{PlaybackSource, PlaybackTrack};
|
||||
use music_dht::podcast::{EpisodeId, PodcastEpisodeRef};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Which catalog sources a client wants library queries to include.
|
||||
@@ -120,6 +122,56 @@ pub struct ArtistRef {
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// A playable queue item: a library track, a pending federated track or a
|
||||
/// podcast episode.
|
||||
///
|
||||
/// # Identity
|
||||
///
|
||||
/// What an item *is* comes from its explicit source fields, never from the
|
||||
/// numeric `id` alone; [`TrackItem::key`] returns it as a typed
|
||||
/// [`TrackItemKey`]:
|
||||
///
|
||||
/// * a library track is a library row: `id >= 0`, `fed` and `podcast` are
|
||||
/// `None`;
|
||||
/// * a federated track is identified by [`TrackItem::fed`] (owner and item
|
||||
/// id). While it is not in the library its `id` is a negative,
|
||||
/// process-local counter value (pending federated tracks count down from
|
||||
/// -1) and `release_id` / artist ids are -1;
|
||||
/// * a podcast episode is identified **only** by [`TrackItem::podcast`] (its
|
||||
/// [`EpisodeId`]). Its `id`, `release_id` and artist id are the fixed
|
||||
/// sentinel [`TrackItem::NOT_A_LIBRARY_ROW`] (-1), shared by every episode.
|
||||
///
|
||||
/// A negative `id` therefore means "not a library row" and nothing else: it
|
||||
/// carries no identity and must never be decoded, compared across items or
|
||||
/// used as a key. The -1 sentinel only keeps music-only code that predates
|
||||
/// podcasts (`id >= 0` / `release_id < 0` checks) from treating an episode
|
||||
/// as library track, release or artist 0.
|
||||
///
|
||||
/// [`TrackItem::key`] is the *source* identity. It does not merge a pending
|
||||
/// federated item with its imported library copy, which share a content id
|
||||
/// but have different keys (`fed:` and `lib:`). Clients that key playback
|
||||
/// items content-first (content id, then federated source, then id) keep
|
||||
/// doing so for music; what they must add is a podcast branch
|
||||
/// ([`TrackItem::podcast`], or [`TrackItemKey::PodcastEpisode`]) checked
|
||||
/// before any fallback to `id`. An episode must never be keyed by `id`: a
|
||||
/// `local:{id}` key would merge all episodes into one. On the wire the item
|
||||
/// kind is `music_dht::device_sync::PlaybackSource`; convert received items
|
||||
/// with [`TrackItem::from_playback_track`], never by copying wire ids.
|
||||
///
|
||||
/// # Podcast episodes
|
||||
///
|
||||
/// Podcast episodes carry their [`PodcastEpisodeRef`] in
|
||||
/// [`TrackItem::podcast`]: `file_path` is empty, `content_id`, `fed` and
|
||||
/// `cover_path` are `None`, and the display fields mirror the reference
|
||||
/// (episode title, podcast title as artist and release). The audio URL is
|
||||
/// only in the reference: it is never a local path, never content-addressed
|
||||
/// and never offered for P2P transfer.
|
||||
///
|
||||
/// **Music-only code paths must check [`TrackItem::is_podcast`]** (or
|
||||
/// `podcast.is_none()`) before treating an item as music: likes, playlists,
|
||||
/// listening history, content-id lookups, federation downloads, similarity
|
||||
/// and file operations do not apply to episodes. Library queries never
|
||||
/// return podcast items; the library database stores none.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TrackItem {
|
||||
pub id: i64,
|
||||
@@ -149,9 +201,187 @@ pub struct TrackItem {
|
||||
/// carries everything needed to download them from the owning peer.
|
||||
/// With an empty `file_path` the player resolves the track on demand.
|
||||
pub fed: Option<FederatedTrack>,
|
||||
/// Set for podcast episodes (see the type documentation); `None` for music.
|
||||
pub podcast: Option<PodcastEpisodeRef>,
|
||||
}
|
||||
|
||||
/// Typed source identity of a [`TrackItem`], from [`TrackItem::key`].
|
||||
///
|
||||
/// Two items with equal keys are the same track or episode; the same track
|
||||
/// may still appear more than once in a queue. The key has no content-id
|
||||
/// equivalence: a pending federated item and its library copy differ. Use it
|
||||
/// where the source matters (and always for podcast episodes, which have no
|
||||
/// other identity); code that matches music by content id keeps doing that
|
||||
/// and checks the podcast variant first (see [`TrackItem`]).
|
||||
///
|
||||
/// The [`Display`](std::fmt::Display) form is stable and suitable as a map
|
||||
/// key or in logs:
|
||||
///
|
||||
/// | Variant | String form |
|
||||
/// | --- | --- |
|
||||
/// | `Library(123)` | `lib:123` |
|
||||
/// | `Federated { owner, item_id }` | `fed:<owner>:<item_id>` |
|
||||
/// | `PodcastEpisode(id)` | `ep:<episode id>`, e.g. `ep:ep1:...` |
|
||||
/// | `Transient(-5)` | `tmp:-5` |
|
||||
///
|
||||
/// `owner` is a peer endpoint id and contains no `:`, so the federated form
|
||||
/// is unambiguous. Keys are not persisted by this crate; a `Transient` key is
|
||||
/// only meaningful inside the process that built the item.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum TrackItemKey {
|
||||
/// A row of the local music library (`TrackItem::id >= 0`).
|
||||
Library(i64),
|
||||
/// A federated track that is not a library row, identified by its owner
|
||||
/// peer and DHT item id ([`FederatedTrack::owner`] /
|
||||
/// [`FederatedTrack::item_id`]), not by its process-local negative `id`.
|
||||
Federated { owner: String, item_id: String },
|
||||
/// A podcast episode, identified by its [`EpisodeId`] only.
|
||||
PodcastEpisode(EpisodeId),
|
||||
/// A non-library item with no explicit source (negative `id`, no `fed`,
|
||||
/// no `podcast`). furumi-library never builds one; clients that do must
|
||||
/// keep its `id` unique within the process.
|
||||
Transient(i64),
|
||||
}
|
||||
|
||||
/// What a received `music_dht::device_sync::PlaybackTrack` is locally, from
|
||||
/// [`TrackItem::from_playback_track`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum PlaybackItem {
|
||||
/// A music track. Convert it with the client's music path (content id,
|
||||
/// then federated source); its wire ids are the sender's.
|
||||
Music,
|
||||
/// A podcast episode, built with [`TrackItem::podcast_episode`].
|
||||
PodcastEpisode(Box<TrackItem>),
|
||||
/// A Jam private placeholder: not playable and without identity. Show it
|
||||
/// as `music_dht::device_sync::PRIVATE_ITEM_TITLE` where the queue is
|
||||
/// displayed; never insert it as a library or queue track, never key it.
|
||||
Private,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for TrackItemKey {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Library(id) => write!(f, "lib:{id}"),
|
||||
Self::Federated { owner, item_id } => write!(f, "fed:{owner}:{item_id}"),
|
||||
Self::PodcastEpisode(episode_id) => write!(f, "ep:{}", episode_id.as_str()),
|
||||
Self::Transient(id) => write!(f, "tmp:{id}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrackItem {
|
||||
/// The `id`, `release_id` and artist id of items that are not library
|
||||
/// rows and have no numeric identity: every podcast episode uses it.
|
||||
///
|
||||
/// It is a sentinel, not an identifier: it is shared by all episodes
|
||||
/// (and can coincide with the first pending federated track's id), so it
|
||||
/// must never be decoded or used as a key. See [`TrackItem::key`].
|
||||
pub const NOT_A_LIBRARY_ROW: i64 = -1;
|
||||
|
||||
/// The typed identity of this item; see [`TrackItemKey`].
|
||||
///
|
||||
/// Precedence: [`TrackItem::podcast`] makes it a podcast episode; else a
|
||||
/// non-negative `id` is a library row; else [`TrackItem::fed`] makes it
|
||||
/// a federated track; anything else is [`TrackItemKey::Transient`].
|
||||
pub fn key(&self) -> TrackItemKey {
|
||||
if let Some(episode) = &self.podcast {
|
||||
return TrackItemKey::PodcastEpisode(episode.episode_id.clone());
|
||||
}
|
||||
if self.id >= 0 {
|
||||
return TrackItemKey::Library(self.id);
|
||||
}
|
||||
match &self.fed {
|
||||
Some(fed) => TrackItemKey::Federated {
|
||||
owner: fed.owner.clone(),
|
||||
item_id: fed.item_id.clone(),
|
||||
},
|
||||
None => TrackItemKey::Transient(self.id),
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue item for a podcast episode streamed from the publisher.
|
||||
///
|
||||
/// Title, artist and release follow `PlaybackTrack::podcast_episode` of
|
||||
/// `music_dht::device_sync` (the episode title, the podcast title as
|
||||
/// artist and release, the announced duration; no track or disc number).
|
||||
/// The other fields differ from that wire form on purpose:
|
||||
///
|
||||
/// * `id`, `release_id` and the artist id are the sentinel
|
||||
/// [`TrackItem::NOT_A_LIBRARY_ROW`] (-1) for every episode, where the
|
||||
/// wire form carries `0`; the item's identity is [`TrackItem::podcast`]
|
||||
/// alone ([`TrackItem::key`]);
|
||||
/// * `release_year` is the publication year (`None` on the wire);
|
||||
/// * `audio_format` is `None` (the wire form carries the MIME type, which
|
||||
/// stays in the reference).
|
||||
///
|
||||
/// Season and episode number stay in the reference.
|
||||
pub fn podcast_episode(episode: PodcastEpisodeRef) -> Self {
|
||||
Self {
|
||||
id: Self::NOT_A_LIBRARY_ROW,
|
||||
title: episode.episode_title.clone(),
|
||||
track_number: None,
|
||||
disc_number: None,
|
||||
duration_seconds: episode
|
||||
.duration_ms
|
||||
.map_or(0.0, |ms| ms.max(0) as f64 / 1_000.0),
|
||||
artists: vec![ArtistRef {
|
||||
id: Self::NOT_A_LIBRARY_ROW,
|
||||
name: episode.podcast_title.clone(),
|
||||
}],
|
||||
featured_artists: Vec::new(),
|
||||
release_id: Self::NOT_A_LIBRARY_ROW,
|
||||
release_title: episode.podcast_title.clone(),
|
||||
release_year: episode.published_at_ms.map(year_of_unix_ms),
|
||||
file_path: String::new(),
|
||||
content_id: None,
|
||||
cover_path: None,
|
||||
audio_format: None,
|
||||
audio_bitrate: None,
|
||||
audio_sample_rate: None,
|
||||
audio_bit_depth: None,
|
||||
file_size_bytes: None,
|
||||
play_count: 0,
|
||||
fed: None,
|
||||
podcast: Some(episode),
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies a received `music_dht::device_sync::PlaybackTrack` and
|
||||
/// builds the local queue item for non-music sources.
|
||||
///
|
||||
/// The wire `id` and `release_id` of podcast episodes and of Jam private
|
||||
/// placeholders are `0`, and those of music items are the sender's local
|
||||
/// ids. None of them identifies anything on this device, so they are
|
||||
/// never copied:
|
||||
///
|
||||
/// * a podcast episode becomes [`TrackItem::podcast_episode`] of its
|
||||
/// [`PlaybackSource::PodcastEpisode`] reference (sentinel ids, key
|
||||
/// `ep:`), whatever the legacy display fields say;
|
||||
/// * a private placeholder ([`PlaybackSource::Private`]) is not playable
|
||||
/// and has no identity, so no `TrackItem` exists for it;
|
||||
/// * music (no source, or [`PlaybackSource::Music`]) is left to the
|
||||
/// client's music path, which resolves it by content id or federated
|
||||
/// source.
|
||||
pub fn from_playback_track(track: &PlaybackTrack) -> PlaybackItem {
|
||||
match &track.source {
|
||||
None | Some(PlaybackSource::Music) => PlaybackItem::Music,
|
||||
Some(PlaybackSource::PodcastEpisode(episode)) => PlaybackItem::PodcastEpisode(
|
||||
Box::new(Self::podcast_episode(episode.as_ref().clone())),
|
||||
),
|
||||
Some(PlaybackSource::Private) => PlaybackItem::Private,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this item is a podcast episode (not music).
|
||||
pub fn is_podcast(&self) -> bool {
|
||||
self.podcast.is_some()
|
||||
}
|
||||
|
||||
/// The episode reference of a podcast item.
|
||||
pub fn podcast_episode_ref(&self) -> Option<&PodcastEpisodeRef> {
|
||||
self.podcast.as_ref()
|
||||
}
|
||||
|
||||
/// A federated track that still needs downloading before playback.
|
||||
pub fn is_fed_pending(&self) -> bool {
|
||||
self.fed.is_some() && self.file_path.is_empty()
|
||||
@@ -202,6 +432,21 @@ impl TrackItem {
|
||||
}
|
||||
}
|
||||
|
||||
/// Proleptic Gregorian year of a unix timestamp in milliseconds (UTC).
|
||||
fn year_of_unix_ms(ms: i64) -> i32 {
|
||||
// Howard Hinnant's civil_from_days.
|
||||
let days = ms.div_euclid(86_400_000);
|
||||
let z = days + 719_468;
|
||||
let era = z.div_euclid(146_097);
|
||||
let doe = z - era * 146_097;
|
||||
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let month = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let year = yoe + era * 400 + i64::from(month <= 2);
|
||||
i32::try_from(year).unwrap_or(if year < 0 { i32::MIN } else { i32::MAX })
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReleaseCard {
|
||||
pub id: i64,
|
||||
@@ -340,8 +585,228 @@ mod tests {
|
||||
file_size_bytes: None,
|
||||
play_count: 0,
|
||||
fed: None,
|
||||
podcast: None,
|
||||
};
|
||||
|
||||
assert_eq!(track.artist_line(), "Main feat. Guest, Other");
|
||||
assert!(!track.is_podcast());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn year_of_unix_ms_handles_epoch_and_leap_years() {
|
||||
assert_eq!(year_of_unix_ms(0), 1970);
|
||||
assert_eq!(year_of_unix_ms(-1), 1969);
|
||||
// 2024-02-29T12:00:00Z and 2024-12-31T23:59:59Z.
|
||||
assert_eq!(year_of_unix_ms(1_709_208_000_000), 2024);
|
||||
assert_eq!(year_of_unix_ms(1_735_689_599_000), 2024);
|
||||
assert_eq!(year_of_unix_ms(1_735_689_600_000), 2025);
|
||||
}
|
||||
|
||||
fn episode_ref(guid: &str) -> PodcastEpisodeRef {
|
||||
use music_dht::podcast::{FeedUrl, PodcastId};
|
||||
let feed_url = FeedUrl::parse("https://example.test/feed.xml").unwrap();
|
||||
let podcast_id = PodcastId::from_feed_url(&feed_url);
|
||||
PodcastEpisodeRef {
|
||||
episode_id: EpisodeId::from_guid(&podcast_id, guid).unwrap(),
|
||||
podcast_id,
|
||||
feed_url,
|
||||
podcast_title: "Show".into(),
|
||||
episode_title: "Pilot".into(),
|
||||
audio_url: "https://cdn.example.test/1.mp3".into(),
|
||||
mime: Some("audio/mpeg".into()),
|
||||
duration_ms: Some(90_500),
|
||||
published_at_ms: Some(1_709_208_000_000),
|
||||
season: Some(2),
|
||||
episode_number: Some(7),
|
||||
image_url: None,
|
||||
progress_generation: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn library_track(id: i64) -> TrackItem {
|
||||
TrackItem {
|
||||
id,
|
||||
title: "Track".into(),
|
||||
track_number: None,
|
||||
disc_number: None,
|
||||
duration_seconds: 1.0,
|
||||
artists: vec![artist("Main")],
|
||||
featured_artists: Vec::new(),
|
||||
release_id: 1,
|
||||
release_title: "Release".into(),
|
||||
release_year: None,
|
||||
file_path: "/tmp/track.mp3".into(),
|
||||
content_id: None,
|
||||
cover_path: None,
|
||||
audio_format: None,
|
||||
audio_bitrate: None,
|
||||
audio_sample_rate: None,
|
||||
audio_bit_depth: None,
|
||||
file_size_bytes: None,
|
||||
play_count: 0,
|
||||
fed: None,
|
||||
podcast: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn federated(owner: &str, item_id: &str) -> FederatedTrack {
|
||||
FederatedTrack {
|
||||
item_id: item_id.into(),
|
||||
owner: owner.into(),
|
||||
own: false,
|
||||
title: "Remote".into(),
|
||||
artist_names: vec!["Someone".into()],
|
||||
featured_artist_names: Vec::new(),
|
||||
year: None,
|
||||
duration_seconds: None,
|
||||
content_id: None,
|
||||
release_title: None,
|
||||
track_number: None,
|
||||
disc_number: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_track_mirrors_the_episode_reference() {
|
||||
let episode = episode_ref("e1");
|
||||
let track = TrackItem::podcast_episode(episode.clone());
|
||||
assert!(track.is_podcast());
|
||||
assert_eq!(track.podcast_episode_ref(), Some(&episode));
|
||||
// The sentinel marks "not a library row, release or artist" and
|
||||
// carries no identity.
|
||||
assert_eq!(TrackItem::NOT_A_LIBRARY_ROW, -1);
|
||||
assert_eq!(track.id, -1);
|
||||
assert_eq!(track.release_id, -1);
|
||||
assert!(track.artists.iter().all(|artist| artist.id == -1));
|
||||
assert!(track.featured_artists.is_empty());
|
||||
assert!(track.file_path.is_empty());
|
||||
assert!(track.content_id.is_none() && track.fed.is_none());
|
||||
assert!(!track.is_fed_pending());
|
||||
assert_eq!(track.title, "Pilot");
|
||||
assert_eq!(track.artist_line(), "Show");
|
||||
assert_eq!(track.release_title, "Show");
|
||||
assert_eq!(track.release_year, Some(2024));
|
||||
assert_eq!(track.duration_label(), "1:31");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_is_typed_by_source() {
|
||||
let library = library_track(123);
|
||||
assert_eq!(library.key(), TrackItemKey::Library(123));
|
||||
assert_eq!(library.key().to_string(), "lib:123");
|
||||
assert_eq!(library_track(0).key(), TrackItemKey::Library(0));
|
||||
|
||||
let mut pending = library_track(-5);
|
||||
pending.release_id = -1;
|
||||
pending.file_path.clear();
|
||||
pending.fed = Some(federated("peer1", "item-9"));
|
||||
let fed_key = TrackItemKey::Federated {
|
||||
owner: "peer1".into(),
|
||||
item_id: "item-9".into(),
|
||||
};
|
||||
assert_eq!(pending.key(), fed_key);
|
||||
assert_eq!(pending.key().to_string(), "fed:peer1:item-9");
|
||||
// The process-local negative id is not part of a federated identity.
|
||||
let mut renumbered = pending.clone();
|
||||
renumbered.id = -6;
|
||||
assert_eq!(renumbered.key(), pending.key());
|
||||
|
||||
let episode = episode_ref("e1");
|
||||
let track = TrackItem::podcast_episode(episode.clone());
|
||||
let ep_key = TrackItemKey::PodcastEpisode(episode.episode_id.clone());
|
||||
assert_eq!(track.key(), ep_key);
|
||||
assert_eq!(
|
||||
track.key().to_string(),
|
||||
format!("ep:{}", episode.episode_id.as_str())
|
||||
);
|
||||
assert!(track.key().to_string().starts_with("ep:ep1:"));
|
||||
|
||||
let transient = library_track(-7);
|
||||
assert_eq!(transient.key(), TrackItemKey::Transient(-7));
|
||||
assert_eq!(transient.key().to_string(), "tmp:-7");
|
||||
|
||||
// The source wins over the numeric id: a podcast item is never keyed
|
||||
// as a library row, a federated item never by its counter value.
|
||||
let mut odd = TrackItem::podcast_episode(episode);
|
||||
odd.id = 42;
|
||||
assert_eq!(odd.key(), ep_key);
|
||||
let mut pending_minus_one = pending.clone();
|
||||
pending_minus_one.id = TrackItem::NOT_A_LIBRARY_ROW;
|
||||
assert_eq!(pending_minus_one.key(), fed_key);
|
||||
assert_ne!(pending_minus_one.key(), track.key());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn received_playback_tracks_never_copy_wire_ids() {
|
||||
// F1 wire form of an episode: id 0, release_id 0.
|
||||
let episode = episode_ref("e1");
|
||||
let wire = PlaybackTrack::podcast_episode(episode.clone());
|
||||
assert_eq!((wire.id, wire.release_id), (0, 0));
|
||||
let PlaybackItem::PodcastEpisode(item) = TrackItem::from_playback_track(&wire) else {
|
||||
panic!("an episode is not music");
|
||||
};
|
||||
assert_eq!(item.id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
assert_eq!(item.release_id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
assert!(item.artists.iter().all(|artist| artist.id < 0));
|
||||
assert_eq!(
|
||||
item.key(),
|
||||
TrackItemKey::PodcastEpisode(episode.episode_id.clone())
|
||||
);
|
||||
assert_eq!(item.podcast_episode_ref(), Some(&episode));
|
||||
// Legacy display fields an older sender filled in do not matter.
|
||||
let mut odd = wire.clone();
|
||||
odd.id = 42;
|
||||
odd.release_id = 7;
|
||||
odd.title = "Something else".into();
|
||||
let PlaybackItem::PodcastEpisode(odd) = TrackItem::from_playback_track(&odd) else {
|
||||
panic!("the source decides");
|
||||
};
|
||||
assert_eq!(odd.key(), item.key());
|
||||
assert_eq!(odd.id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
|
||||
// A private placeholder (also id 0) never becomes library row 0.
|
||||
let private = PlaybackTrack::private_placeholder();
|
||||
assert_eq!(private.id, 0);
|
||||
assert!(matches!(
|
||||
TrackItem::from_playback_track(&private),
|
||||
PlaybackItem::Private
|
||||
));
|
||||
|
||||
// Music is left to the client's music path.
|
||||
let mut music = wire;
|
||||
music.source = None;
|
||||
assert!(matches!(
|
||||
TrackItem::from_playback_track(&music),
|
||||
PlaybackItem::Music
|
||||
));
|
||||
music.source = Some(PlaybackSource::Music);
|
||||
assert!(matches!(
|
||||
TrackItem::from_playback_track(&music),
|
||||
PlaybackItem::Music
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn episodes_share_the_sentinel_id_but_not_the_key() {
|
||||
let first = TrackItem::podcast_episode(episode_ref("e1"));
|
||||
let second = TrackItem::podcast_episode(episode_ref("e2"));
|
||||
assert_eq!(first.id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
assert_eq!(second.id, TrackItem::NOT_A_LIBRARY_ROW);
|
||||
assert_eq!(first.id, second.id);
|
||||
assert_eq!(first.release_id, second.release_id);
|
||||
assert_ne!(first.key(), second.key());
|
||||
assert_ne!(first.key().to_string(), second.key().to_string());
|
||||
assert_eq!(
|
||||
TrackItem::podcast_episode(episode_ref("e1")).key(),
|
||||
first.key()
|
||||
);
|
||||
let keys: std::collections::HashSet<TrackItemKey> =
|
||||
[first.key(), second.key(), first.key()]
|
||||
.into_iter()
|
||||
.collect();
|
||||
assert_eq!(keys.len(), 2);
|
||||
let mut ordered = [second.key(), first.key(), TrackItemKey::Library(1)];
|
||||
ordered.sort();
|
||||
assert_eq!(ordered[0], TrackItemKey::Library(1));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,544 @@
|
||||
//! Local user actions: they build the device-sync operation, apply it through
|
||||
//! the shared code path and return it for the client's sync log.
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::device_sync::SyncOpPayload;
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord,
|
||||
ProgressStatus, effective_share, merge_feed_aliases, merge_listen_session,
|
||||
next_podcast_generation,
|
||||
};
|
||||
use rusqlite::{Connection, TransactionBehavior};
|
||||
|
||||
use super::apply::{self, ApplyOutcome};
|
||||
use super::{OpStamp, PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource};
|
||||
|
||||
/// Result of [`PodcastStore::subscribe`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct SubscribeOutcome {
|
||||
/// Podcast the URL resolved to (an existing one when the URL is a known
|
||||
/// feed URL or alias).
|
||||
pub podcast_id: PodcastId,
|
||||
/// The podcast was already subscribed; no operation was emitted.
|
||||
pub already_subscribed: bool,
|
||||
/// Operations to append to the sync log.
|
||||
pub ops: Vec<PodcastLocalOp>,
|
||||
}
|
||||
|
||||
/// A listening session of this device as seen by the progress register.
|
||||
///
|
||||
/// Returned by explicit actions; periodic checkpoints of the session use its
|
||||
/// `generation` ([`PodcastStore::checkpoint`]).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProgressSession {
|
||||
/// Episode being played.
|
||||
pub episode: PodcastEpisodeRef,
|
||||
/// Session id (also the `ListenSession::session_id`).
|
||||
pub session_id: String,
|
||||
/// Generation of the session's latest explicit action.
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
/// A progress register written by a local action.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ProgressWrite {
|
||||
/// The register as written.
|
||||
pub record: ProgressRecord,
|
||||
/// The session after the action (its generation for later checkpoints).
|
||||
pub session: ProgressSession,
|
||||
/// Operation to append to the sync log.
|
||||
pub op: PodcastLocalOp,
|
||||
}
|
||||
|
||||
/// Result of [`PodcastStore::checkpoint`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum CheckpointOutcome {
|
||||
/// The checkpoint was written.
|
||||
Written(Box<ProgressWrite>),
|
||||
/// The register holds a greater generation (another device or an
|
||||
/// explicit action took over). Nothing was written: stop writing at the
|
||||
/// old generation, or continue locally with an explicit action
|
||||
/// ([`PodcastStore::start_session`] / [`PodcastStore::seek`]).
|
||||
Superseded {
|
||||
/// Current register.
|
||||
current: ProgressRecord,
|
||||
},
|
||||
/// This session already completed the episode at its generation; a
|
||||
/// checkpoint would turn it back into "in progress". Nothing was written.
|
||||
AlreadyCompleted {
|
||||
/// Current register.
|
||||
current: ProgressRecord,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum ExplicitAction {
|
||||
Start(i64),
|
||||
Seek(i64),
|
||||
Restart,
|
||||
Complete(i64),
|
||||
MarkPlayed,
|
||||
MarkUnplayed,
|
||||
}
|
||||
|
||||
fn invalid(reason: impl ToString) -> anyhow::Error {
|
||||
PodcastStoreError::Invalid(reason.to_string()).into()
|
||||
}
|
||||
|
||||
/// Validates the payload (with a provisional stamp), draws the real stamp,
|
||||
/// and applies the operation through [`apply::apply_op`].
|
||||
pub(crate) fn emit(
|
||||
conn: &Connection,
|
||||
stamps: &mut dyn StampSource,
|
||||
make: impl Fn(&OpStamp) -> SyncOpPayload,
|
||||
) -> Result<PodcastLocalOp> {
|
||||
let device_id = stamps.device_id().to_string();
|
||||
let provisional = OpStamp::new("provisional", device_id.clone(), 0);
|
||||
provisional
|
||||
.wire(make(&provisional))
|
||||
.validate_podcast()
|
||||
.map_err(invalid)?;
|
||||
let stamp = stamps.next_stamp()?;
|
||||
if stamp.origin_device_id != device_id {
|
||||
return Err(invalid("stamp origin differs from the local device id"));
|
||||
}
|
||||
let payload = make(&stamp);
|
||||
let op = stamp.wire(payload.clone());
|
||||
match apply::apply_op(conn, &op)? {
|
||||
ApplyOutcome::Rejected(reason) => Err(invalid(reason)),
|
||||
_ => Ok(PodcastLocalOp { stamp, payload }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves a feed URL against every known URL of every podcast.
|
||||
pub(crate) fn resolve_podcast(conn: &Connection, url: &FeedUrl) -> Result<PodcastId> {
|
||||
let mut statement = conn.prepare_cached(
|
||||
"SELECT podcast_id FROM podcast_feed_aliases WHERE feed_url = ?1
|
||||
UNION SELECT podcast_id FROM podcasts WHERE feed_url = ?1
|
||||
UNION SELECT podcast_id FROM podcast_subscriptions
|
||||
WHERE json_extract(podcast_json, '$.feed_url') = ?1",
|
||||
)?;
|
||||
let ids = statement
|
||||
.query_map([url.as_str()], |row| super::podcast_id_col(row.get(0)?))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let pairs: Vec<(PodcastId, FeedUrl)> = ids.into_iter().map(|id| (id, url.clone())).collect();
|
||||
Ok(music_dht::podcast::resolve_podcast_id(
|
||||
url,
|
||||
pairs.iter().map(|(id, url)| (id, url)),
|
||||
))
|
||||
}
|
||||
|
||||
fn catalog_meta(conn: &Connection, podcast_id: &PodcastId) -> Result<Option<PodcastMeta>> {
|
||||
use rusqlite::OptionalExtension as _;
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT feed_url, title, author, description, link, image_url
|
||||
FROM podcasts WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
Ok(PodcastMeta {
|
||||
feed_url: super::feed_url_col(row.get(0)?)?,
|
||||
title: row.get(1)?,
|
||||
author: row.get(2)?,
|
||||
description: row.get(3)?,
|
||||
link: row.get(4)?,
|
||||
image_url: row.get(5)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
fn share_payload(conn: &Connection, podcast_id: &PodcastId, shared: bool) -> Result<SyncOpPayload> {
|
||||
let subscription = apply::load_subscription_register(conn, podcast_id)?;
|
||||
let share = apply::load_share(conn, podcast_id)?;
|
||||
Ok(SyncOpPayload::PodcastShareSet {
|
||||
podcast_id: podcast_id.clone(),
|
||||
shared,
|
||||
generation: next_podcast_generation(subscription.as_ref(), share.as_ref()),
|
||||
})
|
||||
}
|
||||
|
||||
/// The reference carried by progress operations: identity and display only.
|
||||
fn op_ref(episode: &PodcastEpisodeRef) -> PodcastEpisodeRef {
|
||||
let mut episode = episode.clone();
|
||||
episode.progress_generation = None;
|
||||
episode
|
||||
}
|
||||
|
||||
impl PodcastStore {
|
||||
/// Subscribes to the podcast of `podcast.feed_url` (explicit user action).
|
||||
///
|
||||
/// The URL is resolved against known feed URLs and aliases first, so
|
||||
/// subscribing again, through a redirect target or from someone else's
|
||||
/// card reuses the existing podcast with its progress and history. When
|
||||
/// the podcast is already subscribed nothing is emitted. Otherwise one
|
||||
/// `podcast_subscription_set` with generation
|
||||
/// `next_podcast_generation(..)` and the known aliases is applied and
|
||||
/// returned. The new subscription is private until
|
||||
/// [`PodcastStore::set_shared`].
|
||||
///
|
||||
/// Pass the channel metadata of the fetched feed
|
||||
/// (`ParsedFeed::to_podcast_meta(url)`), then ingest the episodes with
|
||||
/// [`PodcastStore::apply_feed`].
|
||||
pub fn subscribe(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
podcast: &PodcastMeta,
|
||||
) -> Result<SubscribeOutcome> {
|
||||
let meta = podcast.bounded();
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let podcast_id = resolve_podcast(&tx, &meta.feed_url)?;
|
||||
let current = apply::load_subscription_register(&tx, &podcast_id)?;
|
||||
if current.as_ref().is_some_and(|record| record.subscribed) {
|
||||
return Ok(SubscribeOutcome {
|
||||
podcast_id,
|
||||
already_subscribed: true,
|
||||
ops: Vec::new(),
|
||||
});
|
||||
}
|
||||
let share = apply::load_share(&tx, &podcast_id)?;
|
||||
let generation = next_podcast_generation(current.as_ref(), share.as_ref());
|
||||
let aliases = merge_feed_aliases(
|
||||
&podcast_id,
|
||||
&apply::load_aliases(&tx, &podcast_id)?,
|
||||
std::slice::from_ref(&meta.feed_url),
|
||||
);
|
||||
let op = emit(&tx, stamps, |_| SyncOpPayload::PodcastSubscriptionSet {
|
||||
podcast_id: podcast_id.clone(),
|
||||
subscribed: true,
|
||||
podcast: meta.clone(),
|
||||
feed_aliases: aliases.clone(),
|
||||
generation,
|
||||
})?;
|
||||
tx.commit()?;
|
||||
Ok(SubscribeOutcome {
|
||||
podcast_id,
|
||||
already_subscribed: false,
|
||||
ops: vec![op],
|
||||
})
|
||||
}
|
||||
|
||||
/// Unsubscribes (explicit user action).
|
||||
///
|
||||
/// Emits `podcast_subscription_set { subscribed: false }` followed by
|
||||
/// `podcast_share_set { shared: false }`, each with the next podcast
|
||||
/// generation. Progress, sessions and the catalog are kept, so
|
||||
/// subscribing again continues where the user stopped. Does nothing when
|
||||
/// the podcast is not subscribed.
|
||||
pub fn unsubscribe(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Vec<PodcastLocalOp>> {
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let Some(current) = apply::load_subscription_register(&tx, podcast_id)? else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
if !current.subscribed {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let share = apply::load_share(&tx, podcast_id)?;
|
||||
let generation = next_podcast_generation(Some(¤t), share.as_ref());
|
||||
let mut meta = catalog_meta(&tx, podcast_id)?
|
||||
.map(|meta| meta.bounded())
|
||||
.unwrap_or_else(|| current.podcast.clone());
|
||||
meta.feed_url = current.podcast.feed_url.clone();
|
||||
let aliases = current.feed_aliases.clone();
|
||||
let unsubscribe = emit(&tx, stamps, |_| SyncOpPayload::PodcastSubscriptionSet {
|
||||
podcast_id: podcast_id.clone(),
|
||||
subscribed: false,
|
||||
podcast: meta.clone(),
|
||||
feed_aliases: aliases.clone(),
|
||||
generation,
|
||||
})?;
|
||||
let revoke_payload = share_payload(&tx, podcast_id, false)?;
|
||||
let revoke = emit(&tx, stamps, |_| revoke_payload.clone())?;
|
||||
tx.commit()?;
|
||||
Ok(vec![unsubscribe, revoke])
|
||||
}
|
||||
|
||||
/// Sets the decision to publish the podcast to the federation.
|
||||
///
|
||||
/// Sharing needs an active subscription and is a no-op when the podcast
|
||||
/// is already effectively shared (`effective_share`). Revoking is a no-op
|
||||
/// when no share decision says `true`. The write uses the next podcast
|
||||
/// generation, so it orders after the subscription it observed.
|
||||
pub fn set_shared(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
podcast_id: &PodcastId,
|
||||
shared: bool,
|
||||
) -> Result<Vec<PodcastLocalOp>> {
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let subscription = apply::load_subscription_register(&tx, podcast_id)?;
|
||||
let share = apply::load_share(&tx, podcast_id)?;
|
||||
if shared {
|
||||
if !subscription
|
||||
.as_ref()
|
||||
.is_some_and(|record| record.subscribed)
|
||||
{
|
||||
return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into());
|
||||
}
|
||||
if effective_share(subscription.as_ref(), share.as_ref()) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
} else if !share.as_ref().is_some_and(|record| record.shared) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let payload = share_payload(&tx, podcast_id, shared)?;
|
||||
let op = emit(&tx, stamps, |_| payload.clone())?;
|
||||
tx.commit()?;
|
||||
Ok(vec![op])
|
||||
}
|
||||
|
||||
fn explicit_progress(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
session_id: Option<&str>,
|
||||
carried_generation: Option<u64>,
|
||||
action: ExplicitAction,
|
||||
) -> Result<ProgressWrite> {
|
||||
episode.validate().map_err(invalid)?;
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let current = apply::load_progress(&tx, &episode.episode_id)?;
|
||||
let max_seen = current
|
||||
.as_ref()
|
||||
.map(|record| record.generation)
|
||||
.into_iter()
|
||||
.chain(carried_generation)
|
||||
.max();
|
||||
let generation = ProgressRecord::explicit_generation(max_seen);
|
||||
let current_position = current.as_ref().map_or(0, |record| record.position_ms);
|
||||
let (position_ms, status) = match action {
|
||||
ExplicitAction::Start(position) | ExplicitAction::Seek(position) => {
|
||||
(position, ProgressStatus::InProgress)
|
||||
}
|
||||
ExplicitAction::Restart => (0, ProgressStatus::InProgress),
|
||||
ExplicitAction::Complete(position) => (position, ProgressStatus::Completed),
|
||||
ExplicitAction::MarkPlayed => (current_position, ProgressStatus::Completed),
|
||||
ExplicitAction::MarkUnplayed => (0, ProgressStatus::NotStarted),
|
||||
};
|
||||
let reference = op_ref(episode);
|
||||
let op = emit(&tx, stamps, |stamp| SyncOpPayload::PodcastProgressSet {
|
||||
episode_id: episode.episode_id.clone(),
|
||||
podcast_id: episode.podcast_id.clone(),
|
||||
position_ms,
|
||||
status,
|
||||
generation,
|
||||
session_id: session_id.unwrap_or(&stamp.op_id).to_string(),
|
||||
episode: Some(reference.clone()),
|
||||
})?;
|
||||
tx.commit()?;
|
||||
let wire = op.stamp.wire(op.payload.clone());
|
||||
let record = wire.podcast_progress().expect("progress payload");
|
||||
Ok(ProgressWrite {
|
||||
session: ProgressSession {
|
||||
episode: episode.clone(),
|
||||
session_id: record.session_id.clone(),
|
||||
generation,
|
||||
},
|
||||
record,
|
||||
op,
|
||||
})
|
||||
}
|
||||
|
||||
/// Starts a listening session at `position_ms` (explicit action).
|
||||
///
|
||||
/// The generation is `explicit_generation(max(local, carried_generation))`;
|
||||
/// pass `PodcastEpisodeRef::progress_generation` of a queue item received
|
||||
/// through a handoff as `carried_generation`. Ordinary Play resumes from
|
||||
/// [`PodcastStore::resume_position_ms`].
|
||||
pub fn start_session(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
session_id: &str,
|
||||
position_ms: i64,
|
||||
carried_generation: Option<u64>,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(
|
||||
stamps,
|
||||
episode,
|
||||
Some(session_id),
|
||||
carried_generation,
|
||||
ExplicitAction::Start(position_ms),
|
||||
)
|
||||
}
|
||||
|
||||
/// Periodic checkpoint (or pause/stop save) of a running session.
|
||||
///
|
||||
/// Keeps the session generation. When the register already holds a
|
||||
/// greater generation nothing is written ([`CheckpointOutcome::Superseded`]).
|
||||
pub fn checkpoint(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
session: &ProgressSession,
|
||||
position_ms: i64,
|
||||
) -> Result<CheckpointOutcome> {
|
||||
let episode = &session.episode;
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
if let Some(current) = apply::load_progress(&tx, &episode.episode_id)? {
|
||||
if current.generation > session.generation {
|
||||
return Ok(CheckpointOutcome::Superseded { current });
|
||||
}
|
||||
if current.generation == session.generation
|
||||
&& current.session_id == session.session_id
|
||||
&& current.status == ProgressStatus::Completed
|
||||
{
|
||||
return Ok(CheckpointOutcome::AlreadyCompleted { current });
|
||||
}
|
||||
}
|
||||
let reference = op_ref(episode);
|
||||
let op = emit(&tx, stamps, |_| SyncOpPayload::PodcastProgressSet {
|
||||
episode_id: episode.episode_id.clone(),
|
||||
podcast_id: episode.podcast_id.clone(),
|
||||
position_ms,
|
||||
status: ProgressStatus::InProgress,
|
||||
generation: session.generation,
|
||||
session_id: session.session_id.clone(),
|
||||
episode: Some(reference.clone()),
|
||||
})?;
|
||||
tx.commit()?;
|
||||
let record = op
|
||||
.stamp
|
||||
.wire(op.payload.clone())
|
||||
.podcast_progress()
|
||||
.expect("progress payload");
|
||||
Ok(CheckpointOutcome::Written(Box::new(ProgressWrite {
|
||||
record,
|
||||
session: session.clone(),
|
||||
op,
|
||||
})))
|
||||
}
|
||||
|
||||
/// User seek during a session (explicit action; the returned session
|
||||
/// carries the new generation for later checkpoints).
|
||||
pub fn seek(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
session: &ProgressSession,
|
||||
position_ms: i64,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(
|
||||
stamps,
|
||||
&session.episode,
|
||||
Some(&session.session_id),
|
||||
Some(session.generation),
|
||||
ExplicitAction::Seek(position_ms),
|
||||
)
|
||||
}
|
||||
|
||||
/// The session reached the end (`reaches_completion` or end of stream):
|
||||
/// writes `completed` with an explicit generation. Record the session
|
||||
/// itself with [`PodcastStore::record_listen_session`].
|
||||
pub fn complete_session(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
session: &ProgressSession,
|
||||
position_ms: i64,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(
|
||||
stamps,
|
||||
&session.episode,
|
||||
Some(&session.session_id),
|
||||
Some(session.generation),
|
||||
ExplicitAction::Complete(position_ms),
|
||||
)
|
||||
}
|
||||
|
||||
/// "Start over": position 0, in progress, explicit generation.
|
||||
pub fn restart(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
session_id: &str,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(
|
||||
stamps,
|
||||
episode,
|
||||
Some(session_id),
|
||||
None,
|
||||
ExplicitAction::Restart,
|
||||
)
|
||||
}
|
||||
|
||||
/// "Mark played": status completed, explicit generation. Creates no
|
||||
/// listening session and invents no listening time.
|
||||
pub fn mark_played(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(stamps, episode, None, None, ExplicitAction::MarkPlayed)
|
||||
}
|
||||
|
||||
/// "Mark unplayed": status not started at position 0, explicit generation.
|
||||
pub fn mark_unplayed(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
episode: &PodcastEpisodeRef,
|
||||
) -> Result<ProgressWrite> {
|
||||
self.explicit_progress(stamps, episode, None, None, ExplicitAction::MarkUnplayed)
|
||||
}
|
||||
|
||||
/// Records (or grows) a listening session of this device.
|
||||
///
|
||||
/// Sessions below the history threshold (`ListenSession::should_record`)
|
||||
/// and versions that do not grow the stored session emit nothing.
|
||||
/// `session.device_id` must be the local device id. The episode reference
|
||||
/// defaults to the catalog's.
|
||||
pub fn record_listen_session(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
session: &ListenSession,
|
||||
episode: Option<&PodcastEpisodeRef>,
|
||||
) -> Result<Option<PodcastLocalOp>> {
|
||||
if session.device_id != stamps.device_id() {
|
||||
return Err(invalid(
|
||||
"session device_id differs from the local device id",
|
||||
));
|
||||
}
|
||||
session.validate().map_err(invalid)?;
|
||||
if !session.should_record() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
if let Some(current) = apply::load_session(&tx, &session.session_id)? {
|
||||
let merged = merge_listen_session(¤t, session).map_err(invalid)?;
|
||||
if merged == current {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
let reference = match episode {
|
||||
Some(episode) => Some(op_ref(episode)),
|
||||
None => super::queries::episode_ref_in(&tx, &session.episode_id)?.map(|r| op_ref(&r)),
|
||||
};
|
||||
let op = emit(&tx, stamps, |_| SyncOpPayload::PodcastListenRecorded {
|
||||
session: session.clone(),
|
||||
episode: reference.clone(),
|
||||
})?;
|
||||
tx.commit()?;
|
||||
Ok(Some(op))
|
||||
}
|
||||
|
||||
/// Resolves a feed URL to a podcast id: the existing podcast when the
|
||||
/// URL is a known feed URL or alias (`resolve_podcast_id`), otherwise
|
||||
/// the id a first subscription through this URL would derive.
|
||||
pub fn resolve_podcast_id(&self, url: &FeedUrl) -> Result<PodcastId> {
|
||||
resolve_podcast(&self.lock(), url)
|
||||
}
|
||||
|
||||
/// The episode reference for queueing or progress actions, or `None`
|
||||
/// when the episode is unknown.
|
||||
pub fn episode_ref(&self, episode_id: &EpisodeId) -> Result<Option<PodcastEpisodeRef>> {
|
||||
super::queries::episode_ref_in(&self.lock(), episode_id)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,742 @@
|
||||
//! The single code path that applies podcast records to the store.
|
||||
//!
|
||||
//! Remote operations, local actions and snapshot sections all end here.
|
||||
//! Records come from the `SyncOpWire` materializers of
|
||||
//! `music_dht::device_sync` and are merged with the functions of
|
||||
//! `music_dht::podcast`, so applying is idempotent and order-independent.
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot};
|
||||
use music_dht::podcast::{
|
||||
EpisodeIdentityHints, EpisodeMeta, FeedUrl, KnownEpisode, ListenSession, PodcastEpisodeRef,
|
||||
PodcastError, PodcastId, ProgressRecord, ShareRecord, SubscriptionRecord, merge_feed_aliases,
|
||||
merge_listen_session,
|
||||
};
|
||||
use rusqlite::{Connection, OptionalExtension as _, TransactionBehavior, params};
|
||||
|
||||
use super::{
|
||||
PROGRESS_COLUMNS, PodcastStore, PodcastStoreError, SESSION_COLUMNS, episode_id_col,
|
||||
feed_url_col, generation_to_sql, now_ms, podcast_meta_col, progress_from_row, session_from_row,
|
||||
status_to_sql,
|
||||
};
|
||||
|
||||
/// Result of applying one operation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum ApplyOutcome {
|
||||
/// Stored state changed.
|
||||
Changed,
|
||||
/// The operation was already reflected (duplicate, or older than the
|
||||
/// stored register).
|
||||
Unchanged,
|
||||
/// Not a podcast operation; nothing to do here.
|
||||
Ignored,
|
||||
/// The operation violates the contract and was skipped. Acknowledge it
|
||||
/// anyway so the log keeps moving (`PODCASTS.md`).
|
||||
Rejected(String),
|
||||
}
|
||||
|
||||
/// Counters returned by [`PodcastStore::apply_snapshot`].
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct SnapshotApplyStats {
|
||||
/// Records that changed stored state.
|
||||
pub changed: usize,
|
||||
/// Valid records already reflected locally.
|
||||
pub unchanged: usize,
|
||||
/// Records that failed validation and were skipped.
|
||||
pub rejected: usize,
|
||||
/// Whether the podcast sections were ignored for exceeding the counts.
|
||||
pub ignored_over_limits: bool,
|
||||
}
|
||||
|
||||
/// Where an episode row came from.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum EpisodeSource {
|
||||
/// Parsed from the podcast's feed on this device.
|
||||
Feed,
|
||||
/// Learned from a `PodcastEpisodeRef` (sync operation or queue item).
|
||||
Sync,
|
||||
}
|
||||
|
||||
impl EpisodeSource {
|
||||
fn as_sql(self) -> &'static str {
|
||||
match self {
|
||||
Self::Feed => "feed",
|
||||
Self::Sync => "sync",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid(error: impl ToString) -> anyhow::Error {
|
||||
PodcastStoreError::Invalid(error.to_string()).into()
|
||||
}
|
||||
|
||||
pub(crate) fn load_subscription_register(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Option<SubscriptionRecord>> {
|
||||
let row = conn
|
||||
.query_row(
|
||||
"SELECT subscribed, podcast_json, generation, hlc_ms, op_id
|
||||
FROM podcast_subscriptions WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
let generation: i64 = row.get(2)?;
|
||||
Ok(SubscriptionRecord {
|
||||
podcast_id: podcast_id.clone(),
|
||||
subscribed: row.get(0)?,
|
||||
podcast: podcast_meta_col(&row.get::<_, String>(1)?)?,
|
||||
feed_aliases: Vec::new(),
|
||||
generation: u64::try_from(generation).unwrap_or(0),
|
||||
hlc_ms: row.get(3)?,
|
||||
op_id: row.get(4)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?;
|
||||
let Some(mut record) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
record.feed_aliases = load_aliases(conn, podcast_id)?;
|
||||
Ok(Some(record))
|
||||
}
|
||||
|
||||
pub(crate) fn load_aliases(conn: &Connection, podcast_id: &PodcastId) -> Result<Vec<FeedUrl>> {
|
||||
let mut statement = conn.prepare_cached(
|
||||
"SELECT feed_url FROM podcast_feed_aliases WHERE podcast_id = ?1 ORDER BY feed_url",
|
||||
)?;
|
||||
let urls = statement
|
||||
.query_map([podcast_id.as_str()], |row| feed_url_col(row.get(0)?))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(urls)
|
||||
}
|
||||
|
||||
pub(crate) fn load_share(conn: &Connection, podcast_id: &PodcastId) -> Result<Option<ShareRecord>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT shared, generation, hlc_ms, op_id FROM podcast_shares WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
let generation: i64 = row.get(1)?;
|
||||
Ok(ShareRecord {
|
||||
podcast_id: podcast_id.clone(),
|
||||
shared: row.get(0)?,
|
||||
generation: u64::try_from(generation).unwrap_or(0),
|
||||
hlc_ms: row.get(2)?,
|
||||
op_id: row.get(3)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
pub(crate) fn load_progress(
|
||||
conn: &Connection,
|
||||
episode_id: &music_dht::podcast::EpisodeId,
|
||||
) -> Result<Option<ProgressRecord>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
&format!("SELECT {PROGRESS_COLUMNS} FROM episode_progress p WHERE p.episode_id = ?1"),
|
||||
[episode_id.as_str()],
|
||||
|row| progress_from_row(row, 0),
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
pub(crate) fn load_session(conn: &Connection, session_id: &str) -> Result<Option<ListenSession>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
&format!(
|
||||
"SELECT {SESSION_COLUMNS} FROM podcast_listen_sessions s WHERE s.session_id = ?1"
|
||||
),
|
||||
[session_id],
|
||||
|row| session_from_row(row, 0),
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
/// Replaces the alias set of a podcast with `merged` when it differs.
|
||||
fn store_aliases(conn: &Connection, podcast_id: &PodcastId, merged: &[FeedUrl]) -> Result<bool> {
|
||||
let current = load_aliases(conn, podcast_id)?;
|
||||
if current == merged {
|
||||
return Ok(false);
|
||||
}
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_feed_aliases WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
)?;
|
||||
let mut insert = conn.prepare_cached(
|
||||
"INSERT INTO podcast_feed_aliases (podcast_id, feed_url) VALUES (?1, ?2)",
|
||||
)?;
|
||||
for url in merged {
|
||||
insert.execute(params![podcast_id.as_str(), url.as_str()])?;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Merges feed URLs into the bounded grow-only alias set of a podcast.
|
||||
///
|
||||
/// Returns whether the stored set changed.
|
||||
pub(crate) fn merge_aliases(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
urls: &[FeedUrl],
|
||||
) -> Result<bool> {
|
||||
let current = load_aliases(conn, podcast_id)?;
|
||||
let merged = merge_feed_aliases(podcast_id, ¤t, urls);
|
||||
store_aliases(conn, podcast_id, &merged)
|
||||
}
|
||||
|
||||
/// Inserts the catalog row of a podcast if it does not exist yet.
|
||||
fn ensure_podcast_row(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
feed_url: &FeedUrl,
|
||||
title: &str,
|
||||
now: i64,
|
||||
) -> Result<()> {
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO podcasts
|
||||
(podcast_id, feed_url, title, meta_from_feed, created_at_ms, updated_at_ms)
|
||||
VALUES (?1, ?2, ?3, 0, ?4, ?4)",
|
||||
params![podcast_id.as_str(), feed_url.as_str(), title, now],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Applies a subscription register record (operation or snapshot).
|
||||
///
|
||||
/// The register is last-writer-wins ([`SubscriptionRecord::supersedes`]);
|
||||
/// the alias set is a bounded grow-only set merged independently, so the
|
||||
/// stored aliases are the bounded union of every URL seen (equal to what
|
||||
/// `merge_subscription` computes). The catalog row is created from the
|
||||
/// register metadata and follows the winning register until this device has
|
||||
/// refreshed the feed itself.
|
||||
pub(crate) fn apply_subscription_record(
|
||||
conn: &Connection,
|
||||
record: &SubscriptionRecord,
|
||||
now: i64,
|
||||
) -> Result<bool> {
|
||||
generation_to_sql(record.generation)?;
|
||||
let podcast_id = &record.podcast_id;
|
||||
let current = load_subscription_register(conn, podcast_id)?;
|
||||
let wins = current
|
||||
.as_ref()
|
||||
.is_none_or(|current| record.supersedes(current));
|
||||
let mut changed = false;
|
||||
if wins {
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_subscriptions
|
||||
(podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(podcast_id) DO UPDATE SET
|
||||
subscribed = excluded.subscribed, podcast_json = excluded.podcast_json,
|
||||
generation = excluded.generation, hlc_ms = excluded.hlc_ms,
|
||||
op_id = excluded.op_id",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
record.subscribed,
|
||||
serde_json::to_string(&record.podcast)?,
|
||||
generation_to_sql(record.generation)?,
|
||||
record.hlc_ms,
|
||||
record.op_id,
|
||||
],
|
||||
)?;
|
||||
changed = true;
|
||||
}
|
||||
let mut urls = record.feed_aliases.clone();
|
||||
urls.push(record.podcast.feed_url.clone());
|
||||
changed |= merge_aliases(conn, podcast_id, &urls)?;
|
||||
|
||||
ensure_podcast_row(
|
||||
conn,
|
||||
podcast_id,
|
||||
&record.podcast.feed_url,
|
||||
&record.podcast.title,
|
||||
now,
|
||||
)?;
|
||||
if wins {
|
||||
let meta = &record.podcast;
|
||||
conn.execute(
|
||||
"UPDATE podcasts
|
||||
SET feed_url = ?2, title = ?3, author = ?4,
|
||||
description = COALESCE(?5, description), link = ?6, image_url = ?7,
|
||||
updated_at_ms = ?8
|
||||
WHERE podcast_id = ?1 AND meta_from_feed = 0",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
meta.feed_url.as_str(),
|
||||
meta.title,
|
||||
meta.author,
|
||||
meta.description,
|
||||
meta.link,
|
||||
meta.image_url,
|
||||
now
|
||||
],
|
||||
)?;
|
||||
}
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
/// Applies a share register record.
|
||||
pub(crate) fn apply_share_record(conn: &Connection, record: &ShareRecord) -> Result<bool> {
|
||||
let generation = generation_to_sql(record.generation)?;
|
||||
let current = load_share(conn, &record.podcast_id)?;
|
||||
if current
|
||||
.as_ref()
|
||||
.is_some_and(|current| !record.supersedes(current))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_shares (podcast_id, shared, generation, hlc_ms, op_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)
|
||||
ON CONFLICT(podcast_id) DO UPDATE SET
|
||||
shared = excluded.shared, generation = excluded.generation,
|
||||
hlc_ms = excluded.hlc_ms, op_id = excluded.op_id",
|
||||
params![
|
||||
record.podcast_id.as_str(),
|
||||
record.shared,
|
||||
generation,
|
||||
record.hlc_ms,
|
||||
record.op_id
|
||||
],
|
||||
)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Applies a progress register record. Never touches the catalog.
|
||||
pub(crate) fn apply_progress_record(conn: &Connection, record: &ProgressRecord) -> Result<bool> {
|
||||
let generation = generation_to_sql(record.generation)?;
|
||||
let current = load_progress(conn, &record.episode_id)?;
|
||||
if current
|
||||
.as_ref()
|
||||
.is_some_and(|current| !record.supersedes(current))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO episode_progress
|
||||
(episode_id, podcast_id, position_ms, status, generation, session_id, hlc_ms,
|
||||
device_id, op_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
|
||||
ON CONFLICT(episode_id) DO UPDATE SET
|
||||
podcast_id = excluded.podcast_id, position_ms = excluded.position_ms,
|
||||
status = excluded.status, generation = excluded.generation,
|
||||
session_id = excluded.session_id, hlc_ms = excluded.hlc_ms,
|
||||
device_id = excluded.device_id, op_id = excluded.op_id",
|
||||
params![
|
||||
record.episode_id.as_str(),
|
||||
record.podcast_id.as_str(),
|
||||
record.position_ms,
|
||||
status_to_sql(record.status),
|
||||
generation,
|
||||
record.session_id,
|
||||
record.hlc_ms,
|
||||
record.device_id,
|
||||
record.op_id,
|
||||
],
|
||||
)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Adds or grows a listening session (grow-only set keyed by `session_id`).
|
||||
pub(crate) fn apply_listen_session(conn: &Connection, session: &ListenSession) -> Result<bool> {
|
||||
let current = load_session(conn, &session.session_id)?;
|
||||
let merged = match ¤t {
|
||||
Some(current) => merge_listen_session(current, session).map_err(invalid)?,
|
||||
None => session.clone(),
|
||||
};
|
||||
if current.as_ref() == Some(&merged) {
|
||||
return Ok(false);
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_listen_sessions
|
||||
(session_id, episode_id, podcast_id, device_id, started_at_ms, ended_at_ms,
|
||||
start_position_ms, end_position_ms, listened_ms, completed)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||
ON CONFLICT(session_id) DO UPDATE SET
|
||||
started_at_ms = excluded.started_at_ms, ended_at_ms = excluded.ended_at_ms,
|
||||
start_position_ms = excluded.start_position_ms,
|
||||
end_position_ms = excluded.end_position_ms,
|
||||
listened_ms = excluded.listened_ms, completed = excluded.completed",
|
||||
params![
|
||||
merged.session_id,
|
||||
merged.episode_id.as_str(),
|
||||
merged.podcast_id.as_str(),
|
||||
merged.device_id,
|
||||
merged.started_at_ms,
|
||||
merged.ended_at_ms,
|
||||
merged.start_position_ms,
|
||||
merged.end_position_ms,
|
||||
merged.listened_ms,
|
||||
merged.completed,
|
||||
],
|
||||
)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Inserts an episode row.
|
||||
pub(crate) fn insert_episode(
|
||||
conn: &Connection,
|
||||
meta: &EpisodeMeta,
|
||||
source: EpisodeSource,
|
||||
in_feed: bool,
|
||||
legacy_guid: Option<&str>,
|
||||
first_seen_at_ms: i64,
|
||||
now: i64,
|
||||
) -> Result<()> {
|
||||
conn.execute(
|
||||
"INSERT INTO episodes
|
||||
(episode_id, podcast_id, guid, title, description, season, episode_number,
|
||||
episode_type, published_at_ms, duration_ms, audio_url, mime, byte_len,
|
||||
image_url, link, source, in_feed, legacy_guid, first_seen_at_ms, updated_at_ms)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17,
|
||||
?18, ?19, ?20)",
|
||||
params![
|
||||
meta.episode_id.as_str(),
|
||||
meta.podcast_id.as_str(),
|
||||
meta.guid,
|
||||
meta.title,
|
||||
meta.description,
|
||||
meta.season,
|
||||
meta.episode_number,
|
||||
episode_type_to_sql(meta.episode_type),
|
||||
meta.published_at_ms,
|
||||
meta.duration_ms,
|
||||
meta.audio_url,
|
||||
meta.mime,
|
||||
meta.byte_len.and_then(|len| i64::try_from(len).ok()),
|
||||
meta.image_url,
|
||||
meta.link,
|
||||
source.as_sql(),
|
||||
in_feed,
|
||||
legacy_guid,
|
||||
first_seen_at_ms,
|
||||
now,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn episode_type_to_sql(kind: music_dht::podcast::EpisodeType) -> &'static str {
|
||||
match kind {
|
||||
music_dht::podcast::EpisodeType::Full => "full",
|
||||
music_dht::podcast::EpisodeType::Trailer => "trailer",
|
||||
music_dht::podcast::EpisodeType::Bonus => "bonus",
|
||||
}
|
||||
}
|
||||
|
||||
/// Records an episode learned through sync when the catalog does not know it.
|
||||
///
|
||||
/// Feed metadata always wins: an existing row is left untouched. The
|
||||
/// podcast catalog row is created from the reference when missing (no
|
||||
/// register is written).
|
||||
pub(crate) fn ensure_episode_from_ref(
|
||||
conn: &Connection,
|
||||
episode: &PodcastEpisodeRef,
|
||||
) -> Result<()> {
|
||||
let exists = conn
|
||||
.query_row(
|
||||
"SELECT 1 FROM episodes WHERE episode_id = ?1",
|
||||
[episode.episode_id.as_str()],
|
||||
|_| Ok(()),
|
||||
)
|
||||
.optional()?
|
||||
.is_some();
|
||||
if exists {
|
||||
return Ok(());
|
||||
}
|
||||
let now = now_ms();
|
||||
ensure_podcast_row(
|
||||
conn,
|
||||
&episode.podcast_id,
|
||||
&episode.feed_url,
|
||||
&episode.podcast_title,
|
||||
now,
|
||||
)?;
|
||||
let meta = EpisodeMeta {
|
||||
episode_id: episode.episode_id.clone(),
|
||||
podcast_id: episode.podcast_id.clone(),
|
||||
guid: None,
|
||||
title: episode.episode_title.clone(),
|
||||
description: None,
|
||||
season: episode.season,
|
||||
episode_number: episode.episode_number,
|
||||
episode_type: Default::default(),
|
||||
published_at_ms: episode.published_at_ms,
|
||||
duration_ms: episode.duration_ms,
|
||||
audio_url: episode.audio_url.clone(),
|
||||
mime: episode.mime.clone(),
|
||||
byte_len: None,
|
||||
image_url: episode.image_url.clone(),
|
||||
link: None,
|
||||
};
|
||||
insert_episode(conn, &meta, EpisodeSource::Sync, false, None, now, now)?;
|
||||
refresh_seasons(conn, &episode.podcast_id)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Recomputes the derived season rows of a podcast from its episodes.
|
||||
pub(crate) fn refresh_seasons(conn: &Connection, podcast_id: &PodcastId) -> Result<()> {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_seasons WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
)?;
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_seasons
|
||||
(podcast_id, season, episode_count, first_published_at_ms, last_published_at_ms)
|
||||
SELECT podcast_id, season, COUNT(*), MIN(published_at_ms), MAX(published_at_ms)
|
||||
FROM episodes WHERE podcast_id = ?1 GROUP BY season",
|
||||
[podcast_id.as_str()],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Known episodes of a podcast with the identity hints they were last seen with.
|
||||
pub(crate) fn known_episodes(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Vec<KnownEpisode>> {
|
||||
let mut statement = conn.prepare_cached(
|
||||
"SELECT episode_id, guid, audio_url, link, published_at_ms, title
|
||||
FROM episodes WHERE podcast_id = ?1 ORDER BY episode_id",
|
||||
)?;
|
||||
let known = statement
|
||||
.query_map([podcast_id.as_str()], |row| {
|
||||
Ok(KnownEpisode {
|
||||
episode_id: episode_id_col(row.get(0)?)?,
|
||||
hints: EpisodeIdentityHints {
|
||||
guid: row.get(1)?,
|
||||
enclosure_url: Some(row.get(2)?),
|
||||
link: row.get(3)?,
|
||||
published_at_ms: row.get(4)?,
|
||||
title: Some(row.get(5)?),
|
||||
},
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(known)
|
||||
}
|
||||
|
||||
/// Applies one podcast operation inside the caller's transaction.
|
||||
pub(crate) fn apply_op(conn: &Connection, op: &SyncOpWire) -> Result<ApplyOutcome> {
|
||||
if !op.payload.is_podcast() {
|
||||
return Ok(ApplyOutcome::Ignored);
|
||||
}
|
||||
if let Err(error) = op.validate_podcast() {
|
||||
return Ok(ApplyOutcome::Rejected(error.to_string()));
|
||||
}
|
||||
let result = match &op.payload {
|
||||
SyncOpPayload::PodcastSubscriptionSet { .. } => {
|
||||
let record = op.podcast_subscription().expect("subscription payload");
|
||||
apply_subscription_record(conn, &record, now_ms())
|
||||
}
|
||||
SyncOpPayload::PodcastShareSet { .. } => {
|
||||
apply_share_record(conn, &op.podcast_share().expect("share payload"))
|
||||
}
|
||||
SyncOpPayload::PodcastFeedAliasesAdded { .. } => {
|
||||
let (podcast_id, urls) = op.podcast_feed_aliases().expect("alias payload");
|
||||
merge_aliases(conn, podcast_id, urls)
|
||||
}
|
||||
SyncOpPayload::PodcastProgressSet { episode, .. } => {
|
||||
let record = op.podcast_progress().expect("progress payload");
|
||||
if let Err(error) = generation_to_sql(record.generation) {
|
||||
return Ok(ApplyOutcome::Rejected(error.to_string()));
|
||||
}
|
||||
if let Some(episode) = episode {
|
||||
ensure_episode_from_ref(conn, episode)?;
|
||||
}
|
||||
apply_progress_record(conn, &record)
|
||||
}
|
||||
SyncOpPayload::PodcastListenRecorded { session, episode } => {
|
||||
if let Some(current) = load_session(conn, &session.session_id)?
|
||||
&& let Err(error) = merge_listen_session(¤t, session)
|
||||
{
|
||||
return Ok(ApplyOutcome::Rejected(error.to_string()));
|
||||
}
|
||||
if let Some(episode) = episode {
|
||||
ensure_episode_from_ref(conn, episode)?;
|
||||
}
|
||||
apply_listen_session(conn, session)
|
||||
}
|
||||
_ => return Ok(ApplyOutcome::Ignored),
|
||||
};
|
||||
match result {
|
||||
Ok(true) => Ok(ApplyOutcome::Changed),
|
||||
Ok(false) => Ok(ApplyOutcome::Unchanged),
|
||||
Err(error) => match error.downcast_ref::<PodcastStoreError>() {
|
||||
Some(PodcastStoreError::Invalid(reason)) => Ok(ApplyOutcome::Rejected(reason.clone())),
|
||||
_ => Err(error),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn record_outcome(stats: &mut SnapshotApplyStats, result: Result<bool>) -> Result<()> {
|
||||
match result {
|
||||
Ok(true) => stats.changed += 1,
|
||||
Ok(false) => stats.unchanged += 1,
|
||||
Err(error) => match error.downcast_ref::<PodcastStoreError>() {
|
||||
Some(PodcastStoreError::Invalid(_)) => stats.rejected += 1,
|
||||
_ => return Err(error),
|
||||
},
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check(result: Result<(), PodcastError>) -> Result<()> {
|
||||
result.map_err(invalid)
|
||||
}
|
||||
|
||||
impl PodcastStore {
|
||||
/// Applies a device-sync operation received from a trusted device.
|
||||
///
|
||||
/// Non-podcast operations return [`ApplyOutcome::Ignored`]. Podcast
|
||||
/// operations are validated (`SyncOpWire::validate_podcast`) and merged
|
||||
/// with the contract functions in one transaction; applying the same
|
||||
/// operations in any order, with duplicates, gives the same state.
|
||||
pub fn apply_remote_op(&self, op: &SyncOpWire) -> Result<ApplyOutcome> {
|
||||
if !op.payload.is_podcast() {
|
||||
return Ok(ApplyOutcome::Ignored);
|
||||
}
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let outcome = apply_op(&tx, op)?;
|
||||
tx.commit()?;
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Merges the podcast sections of a sync snapshot.
|
||||
///
|
||||
/// Sections exceeding the contract counts are ignored
|
||||
/// (`SyncSnapshot::podcast_state_within_limits`); every record is
|
||||
/// validated and merged with the same functions as operations, so a
|
||||
/// snapshot never replaces newer local state. One transaction.
|
||||
pub fn apply_snapshot(&self, snapshot: &SyncSnapshot) -> Result<SnapshotApplyStats> {
|
||||
let mut stats = SnapshotApplyStats::default();
|
||||
if !snapshot.podcast_state_within_limits() {
|
||||
stats.ignored_over_limits = true;
|
||||
return Ok(stats);
|
||||
}
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let now = now_ms();
|
||||
for record in &snapshot.podcast_subscriptions {
|
||||
let mut record = record.clone();
|
||||
record.normalize();
|
||||
let result = check(record.validate())
|
||||
.and_then(|()| apply_subscription_record(&tx, &record, now));
|
||||
record_outcome(&mut stats, result)?;
|
||||
}
|
||||
for record in &snapshot.podcast_shares {
|
||||
let result = check(record.validate()).and_then(|()| apply_share_record(&tx, record));
|
||||
record_outcome(&mut stats, result)?;
|
||||
}
|
||||
for record in &snapshot.podcast_progress {
|
||||
let result = check(record.validate()).and_then(|()| apply_progress_record(&tx, record));
|
||||
record_outcome(&mut stats, result)?;
|
||||
}
|
||||
for session in &snapshot.podcast_listen_sessions {
|
||||
let result =
|
||||
check(session.validate()).and_then(|()| apply_listen_session(&tx, session));
|
||||
record_outcome(&mut stats, result)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// Fills the podcast sections of `snapshot` with this store's registers
|
||||
/// and sessions, bounded by `SyncSnapshot::bound_podcast_state_to`.
|
||||
///
|
||||
/// `budget_bytes` is usually `MAX_PODCAST_SNAPSHOT_BYTES` or
|
||||
/// `device_sync::podcast_snapshot_budget(other_bytes)`. Records are read
|
||||
/// newest first and capped by the per-section counts before the byte
|
||||
/// budget is applied. Local fetch state and the catalog are never part
|
||||
/// of the snapshot. Send it only to v4 peers (`SyncSnapshot::for_peer`).
|
||||
pub fn fill_snapshot(&self, snapshot: &mut SyncSnapshot, budget_bytes: usize) -> Result<()> {
|
||||
use music_dht::podcast::{
|
||||
MAX_SNAPSHOT_LISTEN_SESSIONS, MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES,
|
||||
MAX_SNAPSHOT_SUBSCRIPTIONS,
|
||||
};
|
||||
let conn = self.lock();
|
||||
let ids: Vec<PodcastId> = {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id FROM podcast_subscriptions
|
||||
ORDER BY generation DESC, hlc_ms DESC, op_id DESC, podcast_id DESC
|
||||
LIMIT ?1",
|
||||
)?;
|
||||
statement
|
||||
.query_map([MAX_SNAPSHOT_SUBSCRIPTIONS as i64], |row| {
|
||||
super::podcast_id_col(row.get(0)?)
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let mut subscriptions = Vec::with_capacity(ids.len());
|
||||
for id in &ids {
|
||||
if let Some(record) = load_subscription_register(&conn, id)? {
|
||||
subscriptions.push(record);
|
||||
}
|
||||
}
|
||||
let shares = {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id, shared, generation, hlc_ms, op_id FROM podcast_shares
|
||||
ORDER BY generation DESC, hlc_ms DESC, op_id DESC, podcast_id DESC
|
||||
LIMIT ?1",
|
||||
)?;
|
||||
statement
|
||||
.query_map([MAX_SNAPSHOT_SHARES as i64], |row| {
|
||||
let generation: i64 = row.get(2)?;
|
||||
Ok(ShareRecord {
|
||||
podcast_id: super::podcast_id_col(row.get(0)?)?,
|
||||
shared: row.get(1)?,
|
||||
generation: u64::try_from(generation).unwrap_or(0),
|
||||
hlc_ms: row.get(3)?,
|
||||
op_id: row.get(4)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let progress = {
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT {PROGRESS_COLUMNS} FROM episode_progress p
|
||||
ORDER BY p.generation DESC, p.hlc_ms DESC, p.op_id DESC, p.episode_id DESC
|
||||
LIMIT ?1"
|
||||
))?;
|
||||
statement
|
||||
.query_map([MAX_SNAPSHOT_PROGRESS as i64], |row| {
|
||||
progress_from_row(row, 0)
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let sessions = {
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT {SESSION_COLUMNS} FROM podcast_listen_sessions s
|
||||
ORDER BY s.ended_at_ms DESC, s.session_id DESC
|
||||
LIMIT ?1"
|
||||
))?;
|
||||
statement
|
||||
.query_map([MAX_SNAPSHOT_LISTEN_SESSIONS as i64], |row| {
|
||||
session_from_row(row, 0)
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
drop(conn);
|
||||
snapshot.podcast_subscriptions = subscriptions;
|
||||
snapshot.podcast_shares = shares;
|
||||
snapshot.podcast_progress = progress;
|
||||
snapshot.podcast_listen_sessions = sessions;
|
||||
snapshot.bound_podcast_state_to(budget_bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A snapshot holding only this store's podcast sections, bounded with
|
||||
/// the default budget (`MAX_PODCAST_SNAPSHOT_BYTES`).
|
||||
pub fn export_snapshot(&self) -> Result<SyncSnapshot> {
|
||||
let mut snapshot = SyncSnapshot::default();
|
||||
self.fill_snapshot(
|
||||
&mut snapshot,
|
||||
music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES,
|
||||
)?;
|
||||
Ok(snapshot)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
//! SQLite reference store for podcasts (`podcasts.sqlite3`).
|
||||
//!
|
||||
//! [`PodcastStore`] persists the podcast contract of `music_dht::podcast`
|
||||
//! for SQLite clients: the local episode catalog refreshed from RSS, the
|
||||
//! synchronized personal registers (subscription, share, progress), the
|
||||
//! grow-only listening sessions and the local-only fetch state. It owns no
|
||||
//! networking, HTTP, scheduling or audio: clients fetch feeds, parse them
|
||||
//! with `music_dht::podcast::feed`, append sync operations to their own log
|
||||
//! and exchange snapshots. The normative rules live in the workspace
|
||||
//! `PODCASTS.md` (section "SQLite reference store" describes this schema).
|
||||
//!
|
||||
//! # Files and migrations
|
||||
//!
|
||||
//! The store lives in its own file, conventionally [`PODCAST_DB_FILE`] next
|
||||
//! to the library database; the caller passes the path. Connections use
|
||||
//! WAL, foreign keys and a busy timeout. `PRAGMA user_version` drives the
|
||||
//! migrations, each applied in its own `BEGIN IMMEDIATE` transaction that
|
||||
//! re-checks the version, so concurrent openers and interrupted upgrades are
|
||||
//! safe:
|
||||
//!
|
||||
//! | `user_version` | Schema |
|
||||
//! | --- | --- |
|
||||
//! | 0 | empty file |
|
||||
//! | 1 | prototype schema of the original podcast PR (`podcasts`, `podcast_episodes`) |
|
||||
//! | 2 | contract model; the prototype tables are renamed `legacy_*` and imported |
|
||||
//!
|
||||
//! A database written by a newer Furumi ([`PODCAST_SCHEMA_VERSION`] lower
|
||||
//! than its `user_version`) or a file that is not a podcast database is
|
||||
//! refused before anything is written ([`PodcastStoreError`]).
|
||||
//!
|
||||
//! # Personal state and sync
|
||||
//!
|
||||
//! Local actions ([`PodcastStore::subscribe`], [`PodcastStore::seek`], ...)
|
||||
//! draw operation stamps from a [`StampSource`], build the device-sync
|
||||
//! operation, apply it through the same code path as
|
||||
//! [`PodcastStore::apply_remote_op`] (the `SyncOpWire` materializers of
|
||||
//! `music_dht::device_sync` and the merge functions of `music_dht::podcast`)
|
||||
//! and return the [`PodcastLocalOp`]s. The caller appends them to its sync
|
||||
//! log **with exactly these stamps**; applying them again is a no-op.
|
||||
//! Applying operations and snapshot sections is idempotent and independent
|
||||
//! of delivery order.
|
||||
//!
|
||||
//! RSS refreshes ([`PodcastStore::apply_feed`]) only touch the catalog and
|
||||
//! feed aliases: they never write a register, progress or a session.
|
||||
//!
|
||||
//! ```
|
||||
//! use furumi_library::podcasts::{ApplyOutcome, OpStamp, PodcastStore, StampSource};
|
||||
//! use music_dht::podcast::{FeedUrl, PodcastMeta};
|
||||
//!
|
||||
//! /// Stand-in for the client's sync log, which reserves seq and hlc.
|
||||
//! struct Log {
|
||||
//! device: String,
|
||||
//! seq: i64,
|
||||
//! }
|
||||
//!
|
||||
//! impl StampSource for Log {
|
||||
//! fn device_id(&self) -> &str {
|
||||
//! &self.device
|
||||
//! }
|
||||
//! fn next_stamp(&mut self) -> anyhow::Result<OpStamp> {
|
||||
//! self.seq += 1;
|
||||
//! let op_id = format!("{}:{}", self.device, self.seq);
|
||||
//! Ok(OpStamp::new(op_id, self.device.clone(), 1_700_000_000_000 + self.seq))
|
||||
//! }
|
||||
//! }
|
||||
//!
|
||||
//! let store = PodcastStore::open_in_memory()?;
|
||||
//! let mut log = Log { device: "device-a".into(), seq: 0 };
|
||||
//! let meta = PodcastMeta {
|
||||
//! feed_url: FeedUrl::parse("https://example.test/feed.xml")?,
|
||||
//! title: "Show".into(),
|
||||
//! author: None,
|
||||
//! description: None,
|
||||
//! link: None,
|
||||
//! image_url: None,
|
||||
//! };
|
||||
//! let outcome = store.subscribe(&mut log, &meta)?;
|
||||
//! assert_eq!(outcome.ops.len(), 1);
|
||||
//! for op in outcome.ops {
|
||||
//! // Append to the sync log with this stamp; re-applying is a no-op.
|
||||
//! let op = op.into_wire(log.seq);
|
||||
//! assert_eq!(store.apply_remote_op(&op)?, ApplyOutcome::Unchanged);
|
||||
//! }
|
||||
//! # Ok::<(), anyhow::Error>(())
|
||||
//! ```
|
||||
//!
|
||||
//! # Thread safety
|
||||
//!
|
||||
//! All methods take `&self` and serialize on an internal mutex around one
|
||||
//! synchronous `rusqlite` connection, so an `Arc<PodcastStore>` can be shared
|
||||
//! between threads. Calls block on SQLite I/O: run them on a blocking pool
|
||||
//! (for example `tokio::task::spawn_blocking`), never on an async reactor
|
||||
//! thread. Several processes may open the same file; SQLite serializes the
|
||||
//! writers.
|
||||
|
||||
mod actions;
|
||||
mod apply;
|
||||
mod queries;
|
||||
mod refresh;
|
||||
mod schema;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::fmt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Mutex, MutexGuard, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, FeedUrl, ListenSession, PodcastError, PodcastId, PodcastMeta, ProgressRecord,
|
||||
ProgressStatus,
|
||||
};
|
||||
use rusqlite::Connection;
|
||||
|
||||
pub use actions::{CheckpointOutcome, ProgressSession, ProgressWrite, SubscribeOutcome};
|
||||
pub use apply::{ApplyOutcome, SnapshotApplyStats};
|
||||
pub use queries::{
|
||||
EpisodeListItem, EpisodeQuery, EpisodeSort, LegacyQuarantineEntry, PlayedFilter,
|
||||
PodcastListFilter, PodcastListenEntry, PodcastSummary, SeasonFilter, SeasonSummary,
|
||||
};
|
||||
pub use refresh::{FeedRefresh, FetchState, RefreshOutcome};
|
||||
pub use schema::{LEGACY_PROTOTYPE_SCHEMA_VERSION, PODCAST_SCHEMA_VERSION};
|
||||
|
||||
/// Conventional file name of the podcast store, next to `library.db`.
|
||||
pub const PODCAST_DB_FILE: &str = "podcasts.sqlite3";
|
||||
|
||||
/// `op_id` prefix of registers imported from the prototype database.
|
||||
///
|
||||
/// Imported subscriptions use generation 0, `hlc_ms` = the prototype
|
||||
/// `added_at` and `op_id` = this prefix followed by the podcast id, so every
|
||||
/// write made through the contract orders after them.
|
||||
pub const LEGACY_IMPORT_OP_PREFIX: &str = "legacy-import:";
|
||||
|
||||
/// Typed errors of [`PodcastStore`]; other failures are SQLite or I/O errors.
|
||||
///
|
||||
/// They are returned inside `anyhow::Error`; use `downcast_ref` to match.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum PodcastStoreError {
|
||||
/// The database was written by a newer Furumi. It was not modified.
|
||||
NewerSchema {
|
||||
/// `user_version` found in the file.
|
||||
found: i64,
|
||||
/// Newest version this build understands.
|
||||
supported: i64,
|
||||
},
|
||||
/// The file is not a podcast database (or has an unexpected shape). It
|
||||
/// was not modified.
|
||||
UnrecognizedDatabase(String),
|
||||
/// The podcast is unknown to this store.
|
||||
UnknownPodcast(PodcastId),
|
||||
/// The action needs an active subscription.
|
||||
NotSubscribed(PodcastId),
|
||||
/// Input violates the podcast contract.
|
||||
Invalid(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for PodcastStoreError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::NewerSchema { found, supported } => write!(
|
||||
f,
|
||||
"podcast database schema v{found} is newer than this Furumi supports (v{supported}); \
|
||||
update Furumi, the database was left unchanged"
|
||||
),
|
||||
Self::UnrecognizedDatabase(reason) => {
|
||||
write!(f, "not a Furumi podcast database: {reason}")
|
||||
}
|
||||
Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"),
|
||||
Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"),
|
||||
Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for PodcastStoreError {}
|
||||
|
||||
impl From<PodcastError> for PodcastStoreError {
|
||||
fn from(error: PodcastError) -> Self {
|
||||
Self::Invalid(error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Stamp of a device-sync operation: the fields the `SyncOpWire`
|
||||
/// materializers turn into register stamps.
|
||||
///
|
||||
/// `seq` is not part of it: the client's sync log assigns it when it appends
|
||||
/// the operation ([`PodcastLocalOp::into_wire`]).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct OpStamp {
|
||||
/// Unique operation id, usually `<origin_device_id>:<seq>`.
|
||||
pub op_id: String,
|
||||
/// Origin device id (the progress register `device_id`).
|
||||
pub origin_device_id: String,
|
||||
/// Operation timestamp (register `hlc_ms`).
|
||||
pub hlc_ms: i64,
|
||||
}
|
||||
|
||||
impl OpStamp {
|
||||
/// Builds a stamp.
|
||||
pub fn new(op_id: impl Into<String>, origin_device_id: impl Into<String>, hlc_ms: i64) -> Self {
|
||||
Self {
|
||||
op_id: op_id.into(),
|
||||
origin_device_id: origin_device_id.into(),
|
||||
hlc_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// The stamp of an existing operation.
|
||||
pub fn of(op: &SyncOpWire) -> Self {
|
||||
Self::new(op.op_id.clone(), op.origin_device_id.clone(), op.hlc_ms)
|
||||
}
|
||||
|
||||
pub(crate) fn wire(&self, payload: SyncOpPayload) -> SyncOpWire {
|
||||
SyncOpWire {
|
||||
op_id: self.op_id.clone(),
|
||||
origin_device_id: self.origin_device_id.clone(),
|
||||
seq: 0,
|
||||
hlc_ms: self.hlc_ms,
|
||||
payload,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Source of operation stamps for local actions, usually the client's sync
|
||||
/// log (which reserves the next sequence number and timestamp).
|
||||
///
|
||||
/// A stamp is drawn only when an action actually emits an operation.
|
||||
pub trait StampSource {
|
||||
/// The local device id; every stamp must use it as origin.
|
||||
fn device_id(&self) -> &str;
|
||||
/// Reserves the stamp of the next local operation.
|
||||
fn next_stamp(&mut self) -> Result<OpStamp>;
|
||||
}
|
||||
|
||||
/// A local operation produced by a [`PodcastStore`] action.
|
||||
///
|
||||
/// The store has already applied it. Append it to the sync log with exactly
|
||||
/// this stamp: a different `op_id` or `hlc_ms` would make other devices
|
||||
/// materialize a different register than this one.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PodcastLocalOp {
|
||||
/// Stamp drawn from the [`StampSource`].
|
||||
pub stamp: OpStamp,
|
||||
/// Operation payload (one of the `podcast_*` kinds).
|
||||
pub payload: SyncOpPayload,
|
||||
}
|
||||
|
||||
impl PodcastLocalOp {
|
||||
/// The operation with the sequence number assigned by the sync log.
|
||||
pub fn into_wire(self, seq: i64) -> SyncOpWire {
|
||||
let mut op = self.stamp.wire(self.payload);
|
||||
op.seq = seq;
|
||||
op
|
||||
}
|
||||
}
|
||||
|
||||
/// SQLite store of podcast catalog, personal state and fetch state.
|
||||
///
|
||||
/// See the [module documentation](self) for the schema, the sync model and
|
||||
/// thread safety.
|
||||
pub struct PodcastStore {
|
||||
conn: Mutex<Connection>,
|
||||
path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for PodcastStore {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("PodcastStore")
|
||||
.field("path", &self.path)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl PodcastStore {
|
||||
/// Opens (creating and migrating if needed) the podcast store at `path`.
|
||||
///
|
||||
/// A prototype database (`user_version` 1) is imported; see the module
|
||||
/// documentation. Fails without modifying the file when it was written
|
||||
/// by a newer Furumi or is not a podcast database.
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
if let Some(parent) = path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
std::fs::create_dir_all(parent)
|
||||
.with_context(|| format!("creating {}", parent.display()))?;
|
||||
}
|
||||
let conn = Connection::open(path)
|
||||
.with_context(|| format!("opening podcast database {}", path.display()))?;
|
||||
Self::init(conn, Some(path.to_path_buf()))
|
||||
}
|
||||
|
||||
/// Opens a private in-memory store (tests and previews).
|
||||
pub fn open_in_memory() -> Result<Self> {
|
||||
Self::init(Connection::open_in_memory()?, None)
|
||||
}
|
||||
|
||||
fn init(mut conn: Connection, path: Option<PathBuf>) -> Result<Self> {
|
||||
conn.busy_timeout(Duration::from_secs(5))?;
|
||||
// Refuse newer or foreign files before anything (WAL included) is written.
|
||||
schema::check_openable(&conn)?;
|
||||
if path.is_some() {
|
||||
let _: String = conn.query_row("PRAGMA journal_mode=WAL", [], |row| row.get(0))?;
|
||||
}
|
||||
conn.pragma_update(None, "foreign_keys", "ON")?;
|
||||
register_norm_function(&conn)?;
|
||||
schema::migrate(&mut conn)?;
|
||||
Ok(Self {
|
||||
conn: Mutex::new(conn),
|
||||
path,
|
||||
})
|
||||
}
|
||||
|
||||
/// Path of the database file (`None` for in-memory stores).
|
||||
pub fn path(&self) -> Option<&Path> {
|
||||
self.path.as_deref()
|
||||
}
|
||||
|
||||
/// Current `PRAGMA user_version` (always [`PODCAST_SCHEMA_VERSION`]
|
||||
/// after a successful open).
|
||||
pub fn schema_version(&self) -> Result<i64> {
|
||||
Ok(self
|
||||
.lock()
|
||||
.query_row("PRAGMA user_version", [], |row| row.get(0))?)
|
||||
}
|
||||
|
||||
pub(crate) fn lock(&self) -> MutexGuard<'_, Connection> {
|
||||
self.conn.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
|
||||
fn register_norm_function(conn: &Connection) -> Result<()> {
|
||||
use rusqlite::functions::FunctionFlags;
|
||||
conn.create_scalar_function(
|
||||
"norm",
|
||||
1,
|
||||
FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
|
||||
|ctx| {
|
||||
let value: Option<String> = ctx.get(0)?;
|
||||
Ok(music_dht::normalize_name(
|
||||
value.as_deref().unwrap_or_default(),
|
||||
))
|
||||
},
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn now_ms() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|duration| i64::try_from(duration.as_millis()).unwrap_or(i64::MAX))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Converts a contract generation to the SQLite column value.
|
||||
///
|
||||
/// Generations above `i64::MAX` cannot be stored; records carrying one are
|
||||
/// rejected (no conforming client produces them).
|
||||
pub(crate) fn generation_to_sql(generation: u64) -> Result<i64, PodcastStoreError> {
|
||||
i64::try_from(generation)
|
||||
.map_err(|_| PodcastStoreError::Invalid("generation exceeds i64::MAX".to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn status_to_sql(status: ProgressStatus) -> &'static str {
|
||||
match status {
|
||||
ProgressStatus::NotStarted => "not_started",
|
||||
ProgressStatus::InProgress => "in_progress",
|
||||
ProgressStatus::Completed => "completed",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn status_from_sql(value: &str) -> ProgressStatus {
|
||||
match value {
|
||||
"in_progress" => ProgressStatus::InProgress,
|
||||
"completed" => ProgressStatus::Completed,
|
||||
_ => ProgressStatus::NotStarted,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a stored id; stored ids are validated on write.
|
||||
pub(crate) fn podcast_id_col(value: String) -> rusqlite::Result<PodcastId> {
|
||||
PodcastId::try_from(value).map_err(|error| conversion_error(error.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn episode_id_col(value: String) -> rusqlite::Result<EpisodeId> {
|
||||
EpisodeId::try_from(value).map_err(|error| conversion_error(error.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn feed_url_col(value: String) -> rusqlite::Result<FeedUrl> {
|
||||
FeedUrl::try_from(value).map_err(|error| conversion_error(error.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn podcast_meta_col(value: &str) -> rusqlite::Result<PodcastMeta> {
|
||||
serde_json::from_str(value).map_err(|error| conversion_error(error.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn conversion_error(message: String) -> rusqlite::Error {
|
||||
rusqlite::Error::FromSqlConversionFailure(
|
||||
0,
|
||||
rusqlite::types::Type::Text,
|
||||
Box::new(PodcastStoreError::Invalid(message)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn progress_from_row(
|
||||
row: &rusqlite::Row<'_>,
|
||||
at: usize,
|
||||
) -> rusqlite::Result<ProgressRecord> {
|
||||
let generation: i64 = row.get(at + 4)?;
|
||||
Ok(ProgressRecord {
|
||||
episode_id: episode_id_col(row.get(at)?)?,
|
||||
podcast_id: podcast_id_col(row.get(at + 1)?)?,
|
||||
position_ms: row.get(at + 2)?,
|
||||
status: status_from_sql(&row.get::<_, String>(at + 3)?),
|
||||
generation: u64::try_from(generation).unwrap_or(0),
|
||||
session_id: row.get(at + 5)?,
|
||||
hlc_ms: row.get(at + 6)?,
|
||||
device_id: row.get(at + 7)?,
|
||||
op_id: row.get(at + 8)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Column list matching [`progress_from_row`], for table alias `p`.
|
||||
pub(crate) const PROGRESS_COLUMNS: &str = "p.episode_id, p.podcast_id, p.position_ms, p.status, \
|
||||
p.generation, p.session_id, p.hlc_ms, p.device_id, p.op_id";
|
||||
|
||||
pub(crate) fn session_from_row(
|
||||
row: &rusqlite::Row<'_>,
|
||||
at: usize,
|
||||
) -> rusqlite::Result<ListenSession> {
|
||||
Ok(ListenSession {
|
||||
session_id: row.get(at)?,
|
||||
episode_id: episode_id_col(row.get(at + 1)?)?,
|
||||
podcast_id: podcast_id_col(row.get(at + 2)?)?,
|
||||
device_id: row.get(at + 3)?,
|
||||
started_at_ms: row.get(at + 4)?,
|
||||
ended_at_ms: row.get(at + 5)?,
|
||||
start_position_ms: row.get(at + 6)?,
|
||||
end_position_ms: row.get(at + 7)?,
|
||||
listened_ms: row.get(at + 8)?,
|
||||
completed: row.get(at + 9)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Column list matching [`session_from_row`], for table alias `s`.
|
||||
pub(crate) const SESSION_COLUMNS: &str = "s.session_id, s.episode_id, s.podcast_id, s.device_id, \
|
||||
s.started_at_ms, s.ended_at_ms, s.start_position_ms, s.end_position_ms, s.listened_ms, \
|
||||
s.completed";
|
||||
@@ -0,0 +1,605 @@
|
||||
//! Read-only queries for clients' podcast UI.
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, EpisodeMeta, EpisodeType, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId,
|
||||
PodcastMeta, ProgressRecord, ProgressStatus, ShareRecord, SubscriptionRecord, effective_share,
|
||||
};
|
||||
use rusqlite::types::Value;
|
||||
use rusqlite::{Connection, OptionalExtension as _, params, params_from_iter};
|
||||
|
||||
use super::apply;
|
||||
use super::{
|
||||
PROGRESS_COLUMNS, PodcastStore, SESSION_COLUMNS, episode_id_col, feed_url_col, podcast_id_col,
|
||||
progress_from_row, session_from_row,
|
||||
};
|
||||
use crate::TrackItem;
|
||||
|
||||
/// Which podcasts [`PodcastStore::list_podcasts`] returns.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum PodcastListFilter {
|
||||
/// Podcasts with an active subscription.
|
||||
#[default]
|
||||
Subscribed,
|
||||
/// Every podcast in the catalog, including unsubscribed ones and
|
||||
/// podcasts only known from synced progress or queue items.
|
||||
All,
|
||||
}
|
||||
|
||||
/// A podcast with its personal state and episode counters.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PodcastSummary {
|
||||
/// Podcast id.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Local channel metadata; `feed_url` is the URL this device fetches.
|
||||
pub meta: PodcastMeta,
|
||||
/// Whether the subscription register says subscribed.
|
||||
pub subscribed: bool,
|
||||
/// Whether the podcast must currently be published (`effective_share`).
|
||||
pub shared: bool,
|
||||
/// Episodes in the catalog.
|
||||
pub episode_count: u64,
|
||||
/// Episodes not completed (no progress, not started or in progress).
|
||||
pub unplayed_count: u64,
|
||||
/// Episodes in progress.
|
||||
pub in_progress_count: u64,
|
||||
/// Newest publication time.
|
||||
pub latest_published_at_ms: Option<i64>,
|
||||
/// Numbered seasons (episodes without a season are not counted).
|
||||
pub season_count: u64,
|
||||
}
|
||||
|
||||
/// One season group of a podcast. `season == None` groups the episodes
|
||||
/// without `itunes:season`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SeasonSummary {
|
||||
/// Season number; `None` for episodes without a season.
|
||||
pub season: Option<u32>,
|
||||
/// Episodes in the group.
|
||||
pub episode_count: u64,
|
||||
/// Episodes not completed.
|
||||
pub unplayed_count: u64,
|
||||
/// Oldest publication time.
|
||||
pub first_published_at_ms: Option<i64>,
|
||||
/// Newest publication time.
|
||||
pub last_published_at_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// Season filter of an [`EpisodeQuery`].
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum SeasonFilter {
|
||||
/// All episodes.
|
||||
#[default]
|
||||
Any,
|
||||
/// Episodes of one season.
|
||||
Season(u32),
|
||||
/// Episodes without a season.
|
||||
NoSeason,
|
||||
}
|
||||
|
||||
/// Listening-state filter of an [`EpisodeQuery`].
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum PlayedFilter {
|
||||
/// All episodes.
|
||||
#[default]
|
||||
Any,
|
||||
/// Not completed (not started or in progress).
|
||||
Unplayed,
|
||||
/// Never started or marked unplayed.
|
||||
NotStarted,
|
||||
/// In progress.
|
||||
InProgress,
|
||||
/// Completed.
|
||||
Played,
|
||||
}
|
||||
|
||||
/// Episode ordering of an [`EpisodeQuery`].
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum EpisodeSort {
|
||||
/// Newest publication first; undated episodes last.
|
||||
#[default]
|
||||
NewestFirst,
|
||||
/// Oldest publication first; undated episodes last.
|
||||
OldestFirst,
|
||||
/// By season, then episode number, ascending; unnumbered last.
|
||||
EpisodeNumber,
|
||||
/// By season, then episode number, descending; unnumbered last.
|
||||
EpisodeNumberDesc,
|
||||
}
|
||||
|
||||
/// Parameters of [`PodcastStore::episodes`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct EpisodeQuery {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Season filter.
|
||||
pub season: SeasonFilter,
|
||||
/// Listening-state filter.
|
||||
pub played: PlayedFilter,
|
||||
/// Text searched in titles and descriptions (Unicode-normalized,
|
||||
/// case-insensitive substring match). Blank means no filter.
|
||||
pub search: Option<String>,
|
||||
/// Ordering.
|
||||
pub sort: EpisodeSort,
|
||||
/// Maximum rows (`None`: all).
|
||||
pub limit: Option<usize>,
|
||||
/// Rows to skip.
|
||||
pub offset: usize,
|
||||
}
|
||||
|
||||
impl EpisodeQuery {
|
||||
/// All episodes of a podcast, newest first.
|
||||
pub fn new(podcast_id: PodcastId) -> Self {
|
||||
Self {
|
||||
podcast_id,
|
||||
season: SeasonFilter::Any,
|
||||
played: PlayedFilter::Any,
|
||||
search: None,
|
||||
sort: EpisodeSort::NewestFirst,
|
||||
limit: None,
|
||||
offset: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An episode with its personal state.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct EpisodeListItem {
|
||||
/// Episode metadata (feed or synced reference).
|
||||
pub episode: EpisodeMeta,
|
||||
/// Progress register, if any.
|
||||
pub progress: Option<ProgressRecord>,
|
||||
/// Distinct completed listening sessions (`completed_listen_count`).
|
||||
pub completed_count: u64,
|
||||
/// Whether the latest refresh of the feed listed the episode.
|
||||
pub in_feed: bool,
|
||||
}
|
||||
|
||||
impl EpisodeListItem {
|
||||
/// Listening status (`NotStarted` without a register).
|
||||
pub fn status(&self) -> ProgressStatus {
|
||||
self.progress
|
||||
.as_ref()
|
||||
.map_or(ProgressStatus::NotStarted, |progress| progress.status)
|
||||
}
|
||||
|
||||
/// Position ordinary Play resumes from.
|
||||
pub fn resume_position_ms(&self) -> i64 {
|
||||
self.progress
|
||||
.as_ref()
|
||||
.map_or(0, ProgressRecord::resume_position_ms)
|
||||
}
|
||||
}
|
||||
|
||||
/// A listening session in history, with display titles when known.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PodcastListenEntry {
|
||||
/// The session.
|
||||
pub session: ListenSession,
|
||||
/// Episode title from the catalog.
|
||||
pub episode_title: Option<String>,
|
||||
/// Podcast title from the catalog.
|
||||
pub podcast_title: Option<String>,
|
||||
}
|
||||
|
||||
/// A prototype row that could not be imported (kept in its `legacy_*` table).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct LegacyQuarantineEntry {
|
||||
/// `legacy_podcasts` or `legacy_podcast_episodes`.
|
||||
pub legacy_table: String,
|
||||
/// Row id in that table.
|
||||
pub legacy_id: i64,
|
||||
/// Short reason.
|
||||
pub reason: String,
|
||||
/// Details (the offending value or error).
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
const EPISODE_COLUMNS: &str = "e.episode_id, e.podcast_id, e.guid, e.title, e.description, \
|
||||
e.season, e.episode_number, e.episode_type, e.published_at_ms, e.duration_ms, \
|
||||
e.audio_url, e.mime, e.byte_len, e.image_url, e.link";
|
||||
const EPISODE_COLUMN_COUNT: usize = 15;
|
||||
|
||||
fn episode_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<EpisodeMeta> {
|
||||
Ok(EpisodeMeta {
|
||||
episode_id: episode_id_col(row.get(0)?)?,
|
||||
podcast_id: podcast_id_col(row.get(1)?)?,
|
||||
guid: row.get(2)?,
|
||||
title: row.get(3)?,
|
||||
description: row.get(4)?,
|
||||
season: row.get(5)?,
|
||||
episode_number: row.get(6)?,
|
||||
episode_type: EpisodeType::parse(&row.get::<_, String>(7)?),
|
||||
published_at_ms: row.get(8)?,
|
||||
duration_ms: row.get(9)?,
|
||||
audio_url: row.get(10)?,
|
||||
mime: row.get(11)?,
|
||||
byte_len: row
|
||||
.get::<_, Option<i64>>(12)?
|
||||
.and_then(|len| u64::try_from(len).ok()),
|
||||
image_url: row.get(13)?,
|
||||
link: row.get(14)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn optional_progress(
|
||||
row: &rusqlite::Row<'_>,
|
||||
at: usize,
|
||||
) -> rusqlite::Result<Option<ProgressRecord>> {
|
||||
if row.get::<_, Option<String>>(at)?.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
progress_from_row(row, at).map(Some)
|
||||
}
|
||||
|
||||
pub(crate) fn episode_meta_in(
|
||||
conn: &Connection,
|
||||
episode_id: &EpisodeId,
|
||||
) -> Result<Option<EpisodeMeta>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
&format!("SELECT {EPISODE_COLUMNS} FROM episodes e WHERE e.episode_id = ?1"),
|
||||
[episode_id.as_str()],
|
||||
episode_from_row,
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
pub(crate) fn podcast_meta_in(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<Option<PodcastMeta>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT feed_url, title, author, description, link, image_url
|
||||
FROM podcasts WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
Ok(PodcastMeta {
|
||||
feed_url: feed_url_col(row.get(0)?)?,
|
||||
title: row.get(1)?,
|
||||
author: row.get(2)?,
|
||||
description: row.get(3)?,
|
||||
link: row.get(4)?,
|
||||
image_url: row.get(5)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
/// Queue reference of a catalog episode, with `progress_generation` set to
|
||||
/// the generation of its progress register (handoff hint).
|
||||
pub(crate) fn episode_ref_in(
|
||||
conn: &Connection,
|
||||
episode_id: &EpisodeId,
|
||||
) -> Result<Option<PodcastEpisodeRef>> {
|
||||
let Some(episode) = episode_meta_in(conn, episode_id)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(podcast) = podcast_meta_in(conn, &episode.podcast_id)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut reference = episode.to_ref(&podcast);
|
||||
reference.progress_generation =
|
||||
apply::load_progress(conn, episode_id)?.map(|progress| progress.generation);
|
||||
Ok(Some(reference))
|
||||
}
|
||||
|
||||
const NOT_COMPLETED: &str = "IFNULL(g.status, 'not_started') <> 'completed'";
|
||||
|
||||
impl PodcastStore {
|
||||
/// Podcasts with subscription state and counters, sorted by title.
|
||||
pub fn list_podcasts(&self, filter: PodcastListFilter) -> Result<Vec<PodcastSummary>> {
|
||||
let conn = self.lock();
|
||||
let join = match filter {
|
||||
PodcastListFilter::Subscribed => {
|
||||
"JOIN podcast_subscriptions r ON r.podcast_id = p.podcast_id AND r.subscribed = 1"
|
||||
}
|
||||
PodcastListFilter::All => "",
|
||||
};
|
||||
let ids: Vec<PodcastId> = {
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT p.podcast_id FROM podcasts p {join}
|
||||
ORDER BY p.title COLLATE NOCASE, p.podcast_id"
|
||||
))?;
|
||||
statement
|
||||
.query_map([], |row| podcast_id_col(row.get(0)?))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let mut out = Vec::with_capacity(ids.len());
|
||||
for id in ids {
|
||||
if let Some(summary) = summary_in(&conn, &id)? {
|
||||
out.push(summary);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// One podcast with subscription state and counters.
|
||||
pub fn podcast(&self, podcast_id: &PodcastId) -> Result<Option<PodcastSummary>> {
|
||||
summary_in(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// The subscription register (with the stored alias set), if any.
|
||||
pub fn subscription(&self, podcast_id: &PodcastId) -> Result<Option<SubscriptionRecord>> {
|
||||
apply::load_subscription_register(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// The share register, if any.
|
||||
pub fn share(&self, podcast_id: &PodcastId) -> Result<Option<ShareRecord>> {
|
||||
apply::load_share(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// The bounded, synced feed alias set of a podcast.
|
||||
pub fn feed_aliases(&self, podcast_id: &PodcastId) -> Result<Vec<FeedUrl>> {
|
||||
apply::load_aliases(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// Season groups of a podcast: numbered seasons ascending, then the
|
||||
/// group of episodes without a season.
|
||||
pub fn seasons(&self, podcast_id: &PodcastId) -> Result<Vec<SeasonSummary>> {
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT s.season, s.episode_count, s.first_published_at_ms, s.last_published_at_ms,
|
||||
(SELECT COUNT(*) FROM episodes e
|
||||
LEFT JOIN episode_progress g ON g.episode_id = e.episode_id
|
||||
WHERE e.podcast_id = s.podcast_id AND e.season IS s.season
|
||||
AND {NOT_COMPLETED})
|
||||
FROM podcast_seasons s WHERE s.podcast_id = ?1
|
||||
ORDER BY s.season IS NULL, s.season"
|
||||
))?;
|
||||
let seasons = statement
|
||||
.query_map([podcast_id.as_str()], |row| {
|
||||
Ok(SeasonSummary {
|
||||
season: row.get(0)?,
|
||||
episode_count: count(row.get(1)?),
|
||||
first_published_at_ms: row.get(2)?,
|
||||
last_published_at_ms: row.get(3)?,
|
||||
unplayed_count: count(row.get(4)?),
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(seasons)
|
||||
}
|
||||
|
||||
/// Episodes of a podcast or season with their progress.
|
||||
pub fn episodes(&self, query: &EpisodeQuery) -> Result<Vec<EpisodeListItem>> {
|
||||
let mut conditions = vec!["e.podcast_id = ?".to_string()];
|
||||
let mut values = vec![Value::Text(query.podcast_id.as_str().to_string())];
|
||||
match query.season {
|
||||
SeasonFilter::Any => {}
|
||||
SeasonFilter::Season(season) => {
|
||||
conditions.push("e.season = ?".to_string());
|
||||
values.push(Value::Integer(i64::from(season)));
|
||||
}
|
||||
SeasonFilter::NoSeason => conditions.push("e.season IS NULL".to_string()),
|
||||
}
|
||||
match query.played {
|
||||
PlayedFilter::Any => {}
|
||||
PlayedFilter::Unplayed => conditions.push(NOT_COMPLETED.to_string()),
|
||||
PlayedFilter::NotStarted => {
|
||||
conditions.push("IFNULL(g.status, 'not_started') = 'not_started'".to_string());
|
||||
}
|
||||
PlayedFilter::InProgress => conditions.push("g.status = 'in_progress'".to_string()),
|
||||
PlayedFilter::Played => conditions.push("g.status = 'completed'".to_string()),
|
||||
}
|
||||
if let Some(search) = query.search.as_deref() {
|
||||
let needle = music_dht::normalize_name(search);
|
||||
if !needle.is_empty() {
|
||||
conditions.push(
|
||||
"(instr(norm(e.title), ?) > 0 OR instr(norm(e.description), ?) > 0)"
|
||||
.to_string(),
|
||||
);
|
||||
values.push(Value::Text(needle.clone()));
|
||||
values.push(Value::Text(needle));
|
||||
}
|
||||
}
|
||||
let order = match query.sort {
|
||||
EpisodeSort::NewestFirst => {
|
||||
"e.published_at_ms IS NULL, e.published_at_ms DESC, e.episode_id"
|
||||
}
|
||||
EpisodeSort::OldestFirst => {
|
||||
"e.published_at_ms IS NULL, e.published_at_ms, e.episode_id"
|
||||
}
|
||||
EpisodeSort::EpisodeNumber => {
|
||||
"e.season IS NULL, e.season, e.episode_number IS NULL, e.episode_number, \
|
||||
e.published_at_ms IS NULL, e.published_at_ms, e.episode_id"
|
||||
}
|
||||
EpisodeSort::EpisodeNumberDesc => {
|
||||
"e.season IS NULL, e.season DESC, e.episode_number IS NULL, \
|
||||
e.episode_number DESC, e.published_at_ms IS NULL, e.published_at_ms DESC, \
|
||||
e.episode_id"
|
||||
}
|
||||
};
|
||||
values.push(Value::Integer(
|
||||
query
|
||||
.limit
|
||||
.map_or(-1, |limit| i64::try_from(limit).unwrap_or(i64::MAX)),
|
||||
));
|
||||
values.push(Value::Integer(
|
||||
i64::try_from(query.offset).unwrap_or(i64::MAX),
|
||||
));
|
||||
let progress_columns = PROGRESS_COLUMNS.replace("p.", "g.");
|
||||
let sql = format!(
|
||||
"SELECT {EPISODE_COLUMNS}, e.in_feed, {progress_columns},
|
||||
(SELECT COUNT(*) FROM podcast_listen_sessions s
|
||||
WHERE s.episode_id = e.episode_id AND s.completed = 1)
|
||||
FROM episodes e
|
||||
LEFT JOIN episode_progress g ON g.episode_id = e.episode_id
|
||||
WHERE {}
|
||||
ORDER BY {order}
|
||||
LIMIT ? OFFSET ?",
|
||||
conditions.join(" AND ")
|
||||
);
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(&sql)?;
|
||||
let items = statement
|
||||
.query_map(params_from_iter(values), list_item_from_row)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// One episode with its progress.
|
||||
pub fn episode(&self, episode_id: &EpisodeId) -> Result<Option<EpisodeListItem>> {
|
||||
let progress_columns = PROGRESS_COLUMNS.replace("p.", "g.");
|
||||
Ok(self
|
||||
.lock()
|
||||
.query_row(
|
||||
&format!(
|
||||
"SELECT {EPISODE_COLUMNS}, e.in_feed, {progress_columns},
|
||||
(SELECT COUNT(*) FROM podcast_listen_sessions s
|
||||
WHERE s.episode_id = e.episode_id AND s.completed = 1)
|
||||
FROM episodes e
|
||||
LEFT JOIN episode_progress g ON g.episode_id = e.episode_id
|
||||
WHERE e.episode_id = ?1"
|
||||
),
|
||||
[episode_id.as_str()],
|
||||
list_item_from_row,
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
/// Progress register of an episode (also for episodes the catalog does
|
||||
/// not know yet).
|
||||
pub fn progress(&self, episode_id: &EpisodeId) -> Result<Option<ProgressRecord>> {
|
||||
apply::load_progress(&self.lock(), episode_id)
|
||||
}
|
||||
|
||||
/// Position ordinary Play resumes from (`ProgressRecord::resume_position_ms`, 0 without progress).
|
||||
pub fn resume_position_ms(&self, episode_id: &EpisodeId) -> Result<i64> {
|
||||
Ok(self
|
||||
.progress(episode_id)?
|
||||
.map_or(0, |progress| progress.resume_position_ms()))
|
||||
}
|
||||
|
||||
/// A queue item for an episode (see [`TrackItem::podcast_episode`]).
|
||||
pub fn episode_track(&self, episode_id: &EpisodeId) -> Result<Option<TrackItem>> {
|
||||
Ok(self
|
||||
.episode_ref(episode_id)?
|
||||
.map(TrackItem::podcast_episode))
|
||||
}
|
||||
|
||||
/// Listening history (sessions that `should_record`), newest first.
|
||||
pub fn listen_history(
|
||||
&self,
|
||||
podcast_id: Option<&PodcastId>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<PodcastListenEntry>> {
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(&format!(
|
||||
"SELECT {SESSION_COLUMNS}, e.title, p.title
|
||||
FROM podcast_listen_sessions s
|
||||
LEFT JOIN episodes e ON e.episode_id = s.episode_id
|
||||
LEFT JOIN podcasts p ON p.podcast_id = s.podcast_id
|
||||
WHERE (s.completed = 1 OR s.listened_ms >= ?1)
|
||||
AND (?2 IS NULL OR s.podcast_id = ?2)
|
||||
ORDER BY s.ended_at_ms DESC, s.session_id DESC
|
||||
LIMIT ?3"
|
||||
))?;
|
||||
let entries = statement
|
||||
.query_map(
|
||||
params![
|
||||
music_dht::podcast::MIN_RECORDED_EPISODE_LISTEN_MS,
|
||||
podcast_id.map(PodcastId::as_str),
|
||||
i64::try_from(limit).unwrap_or(i64::MAX)
|
||||
],
|
||||
|row| {
|
||||
Ok(PodcastListenEntry {
|
||||
session: session_from_row(row, 0)?,
|
||||
episode_title: row.get(10)?,
|
||||
podcast_title: row.get(11)?,
|
||||
})
|
||||
},
|
||||
)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Number of distinct completed sessions of an episode.
|
||||
pub fn completed_listen_count(&self, episode_id: &EpisodeId) -> Result<u64> {
|
||||
let total: i64 = self.lock().query_row(
|
||||
"SELECT COUNT(*) FROM podcast_listen_sessions WHERE episode_id = ?1 AND completed = 1",
|
||||
[episode_id.as_str()],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
Ok(count(total))
|
||||
}
|
||||
|
||||
/// Prototype rows that could not be imported.
|
||||
pub fn legacy_quarantine(&self) -> Result<Vec<LegacyQuarantineEntry>> {
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT legacy_table, legacy_id, reason, detail FROM legacy_quarantine
|
||||
ORDER BY legacy_table, legacy_id",
|
||||
)?;
|
||||
let entries = statement
|
||||
.query_map([], |row| {
|
||||
Ok(LegacyQuarantineEntry {
|
||||
legacy_table: row.get(0)?,
|
||||
legacy_id: row.get(1)?,
|
||||
reason: row.get(2)?,
|
||||
detail: row.get(3)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(entries)
|
||||
}
|
||||
}
|
||||
|
||||
fn count(value: i64) -> u64 {
|
||||
u64::try_from(value).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn list_item_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<EpisodeListItem> {
|
||||
Ok(EpisodeListItem {
|
||||
episode: episode_from_row(row)?,
|
||||
in_feed: row.get(EPISODE_COLUMN_COUNT)?,
|
||||
progress: optional_progress(row, EPISODE_COLUMN_COUNT + 1)?,
|
||||
completed_count: count(row.get(EPISODE_COLUMN_COUNT + 10)?),
|
||||
})
|
||||
}
|
||||
|
||||
fn summary_in(conn: &Connection, podcast_id: &PodcastId) -> Result<Option<PodcastSummary>> {
|
||||
let Some(meta) = podcast_meta_in(conn, podcast_id)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (episode_count, unplayed_count, in_progress_count, latest, season_count) = conn.query_row(
|
||||
&format!(
|
||||
"SELECT COUNT(*),
|
||||
COUNT(*) FILTER (WHERE {NOT_COMPLETED}),
|
||||
COUNT(*) FILTER (WHERE g.status = 'in_progress'),
|
||||
MAX(e.published_at_ms),
|
||||
(SELECT COUNT(*) FROM podcast_seasons s
|
||||
WHERE s.podcast_id = ?1 AND s.season IS NOT NULL)
|
||||
FROM episodes e
|
||||
LEFT JOIN episode_progress g ON g.episode_id = e.episode_id
|
||||
WHERE e.podcast_id = ?1"
|
||||
),
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get::<_, i64>(0)?,
|
||||
row.get::<_, i64>(1)?,
|
||||
row.get::<_, i64>(2)?,
|
||||
row.get::<_, Option<i64>>(3)?,
|
||||
row.get::<_, i64>(4)?,
|
||||
))
|
||||
},
|
||||
)?;
|
||||
let subscription = apply::load_subscription_register(conn, podcast_id)?;
|
||||
let share = apply::load_share(conn, podcast_id)?;
|
||||
Ok(Some(PodcastSummary {
|
||||
podcast_id: podcast_id.clone(),
|
||||
meta,
|
||||
subscribed: subscription
|
||||
.as_ref()
|
||||
.is_some_and(|record| record.subscribed),
|
||||
shared: effective_share(subscription.as_ref(), share.as_ref()),
|
||||
episode_count: count(episode_count),
|
||||
unplayed_count: count(unplayed_count),
|
||||
in_progress_count: count(in_progress_count),
|
||||
latest_published_at_ms: latest,
|
||||
season_count: count(season_count),
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,618 @@
|
||||
//! Feed refresh ingestion and the local-only fetch state.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use anyhow::Result;
|
||||
use music_dht::device_sync::SyncOpPayload;
|
||||
use music_dht::podcast::feed::ParsedFeed;
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, EpisodeIdentityHints, EpisodeMeta, FeedUrl, KnownEpisode, PodcastId,
|
||||
merge_feed_aliases, resolve_episode_ids,
|
||||
};
|
||||
use rusqlite::{Connection, OptionalExtension as _, TransactionBehavior, params};
|
||||
|
||||
use super::apply::{self, EpisodeSource};
|
||||
use super::{PodcastLocalOp, PodcastStore, PodcastStoreError, StampSource, now_ms};
|
||||
|
||||
/// A fetched and parsed feed document for [`PodcastStore::apply_feed`].
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct FeedRefresh<'a> {
|
||||
/// Podcast the feed belongs to.
|
||||
pub podcast_id: &'a PodcastId,
|
||||
/// The parsed document (`music_dht::podcast::feed::parse_feed_with_base`
|
||||
/// with `fetched_from` as base).
|
||||
pub feed: &'a ParsedFeed,
|
||||
/// URL the document was finally served from, after redirects.
|
||||
pub fetched_from: &'a FeedUrl,
|
||||
/// Whether every redirect leading to `fetched_from` was permanent
|
||||
/// (301/308). Temporary redirects do not move the feed.
|
||||
pub moved_permanently: bool,
|
||||
}
|
||||
|
||||
/// Result of [`PodcastStore::apply_feed`].
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct RefreshOutcome {
|
||||
/// Episodes seen for the first time.
|
||||
pub new_episodes: usize,
|
||||
/// Known episodes whose metadata changed (title, audio URL, ...).
|
||||
pub updated_episodes: usize,
|
||||
/// Known episodes without metadata changes.
|
||||
pub unchanged_episodes: usize,
|
||||
/// Items without identity or with invalid metadata.
|
||||
pub skipped_items: usize,
|
||||
/// Feed URLs learned from a permanent redirect or `itunes:new-feed-url`.
|
||||
pub learned_aliases: Vec<FeedUrl>,
|
||||
/// URL the podcast is fetched from after this refresh.
|
||||
pub feed_url: Option<FeedUrl>,
|
||||
/// `podcast_feed_aliases_added`, only when the learned URLs changed the
|
||||
/// stored alias set. Append it to the sync log.
|
||||
pub ops: Vec<PodcastLocalOp>,
|
||||
}
|
||||
|
||||
/// Local-only fetch state of a podcast. Never part of sync output.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct FetchState {
|
||||
/// HTTP `ETag` validator of the last successful fetch.
|
||||
pub etag: Option<String>,
|
||||
/// HTTP `Last-Modified` validator of the last successful fetch.
|
||||
pub last_modified: Option<String>,
|
||||
/// Last fetch attempt, unix milliseconds.
|
||||
pub checked_at_ms: Option<i64>,
|
||||
/// Last successful fetch (including "not modified").
|
||||
pub last_success_at_ms: Option<i64>,
|
||||
/// When the scheduler should check again (`None`: due now).
|
||||
pub next_check_at_ms: Option<i64>,
|
||||
/// Error of the last attempt; `None` after a success.
|
||||
pub last_error: Option<String>,
|
||||
/// Failed attempts since the last success.
|
||||
pub consecutive_failures: u32,
|
||||
/// Image URL the cached cover was downloaded from.
|
||||
pub cover_url: Option<String>,
|
||||
/// Local path of the cached cover.
|
||||
pub cover_path: Option<String>,
|
||||
}
|
||||
|
||||
fn podcast_fetch_url(conn: &Connection, podcast_id: &PodcastId) -> Result<Option<FeedUrl>> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT feed_url FROM podcasts WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| super::feed_url_col(row.get(0)?),
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
fn fetch_state_in(conn: &Connection, podcast_id: &PodcastId) -> Result<FetchState> {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT etag, last_modified, checked_at_ms, last_success_at_ms,
|
||||
next_check_at_ms, last_error, consecutive_failures, cover_url,
|
||||
cover_path
|
||||
FROM podcast_fetch_state WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
|row| {
|
||||
Ok(FetchState {
|
||||
etag: row.get(0)?,
|
||||
last_modified: row.get(1)?,
|
||||
checked_at_ms: row.get(2)?,
|
||||
last_success_at_ms: row.get(3)?,
|
||||
next_check_at_ms: row.get(4)?,
|
||||
last_error: row.get(5)?,
|
||||
consecutive_failures: u32::try_from(row.get::<_, i64>(6)?).unwrap_or(u32::MAX),
|
||||
cover_url: row.get(7)?,
|
||||
cover_path: row.get(8)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?
|
||||
.unwrap_or_default())
|
||||
}
|
||||
|
||||
fn has_personal_state(conn: &Connection, episode_id: &EpisodeId) -> Result<bool> {
|
||||
Ok(conn.query_row(
|
||||
"SELECT EXISTS (SELECT 1 FROM episode_progress WHERE episode_id = ?1)
|
||||
OR EXISTS (SELECT 1 FROM podcast_listen_sessions WHERE episode_id = ?1)",
|
||||
[episode_id.as_str()],
|
||||
|row| row.get(0),
|
||||
)?)
|
||||
}
|
||||
|
||||
/// Re-keys episodes imported from the prototype whose stored `guid` was the
|
||||
/// enclosure URL (`legacy_guid`) when the feed shows it is a real GUID.
|
||||
///
|
||||
/// Only episodes without progress or sessions move, so no personal state is
|
||||
/// touched; otherwise the GUID item becomes a separate episode. When another
|
||||
/// episode already holds the GUID id (for example one a remote operation
|
||||
/// created), the imported episode is a duplicate of it and is deleted.
|
||||
fn rekey_legacy_guids(conn: &Connection, podcast_id: &PodcastId, feed: &ParsedFeed) -> Result<()> {
|
||||
for item in &feed.episodes {
|
||||
let Some(guid) = item.guid.as_deref() else {
|
||||
continue;
|
||||
};
|
||||
let Some(guid_id) = EpisodeId::from_guid(podcast_id, guid) else {
|
||||
continue;
|
||||
};
|
||||
let taken = conn
|
||||
.query_row(
|
||||
"SELECT 1 FROM episodes WHERE episode_id = ?1",
|
||||
[guid_id.as_str()],
|
||||
|_| Ok(()),
|
||||
)
|
||||
.optional()?
|
||||
.is_some();
|
||||
let legacy: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT episode_id FROM episodes
|
||||
WHERE podcast_id = ?1 AND legacy_guid = ?2 ORDER BY episode_id LIMIT 1",
|
||||
params![podcast_id.as_str(), guid],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
let Some(legacy) = legacy else {
|
||||
continue;
|
||||
};
|
||||
let legacy = super::episode_id_col(legacy)?;
|
||||
if legacy == guid_id || has_personal_state(conn, &legacy)? {
|
||||
continue;
|
||||
}
|
||||
if taken {
|
||||
conn.execute(
|
||||
"DELETE FROM episodes WHERE episode_id = ?1",
|
||||
[legacy.as_str()],
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
conn.execute(
|
||||
"UPDATE episodes SET episode_id = ?2, guid = ?3, legacy_guid = NULL
|
||||
WHERE episode_id = ?1",
|
||||
params![legacy.as_str(), guid_id.as_str(), guid],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Episodes imported from the prototype without a GUID (`legacy_guid` set)
|
||||
/// that carry no progress and no session.
|
||||
///
|
||||
/// Their ids were resolved over every stored prototype row at once, stale
|
||||
/// rows included, so they may differ from what a fresh device derives from
|
||||
/// the current feed. They are provisional until a refresh confirms them.
|
||||
fn provisional_episode_ids(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
) -> Result<BTreeSet<EpisodeId>> {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT e.episode_id FROM episodes e
|
||||
WHERE e.podcast_id = ?1 AND e.legacy_guid IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM episode_progress p WHERE p.episode_id = e.episode_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM podcast_listen_sessions s
|
||||
WHERE s.episode_id = e.episode_id)",
|
||||
)?;
|
||||
let ids = statement
|
||||
.query_map([podcast_id.as_str()], |row| {
|
||||
super::episode_id_col(row.get(0)?)
|
||||
})?
|
||||
.collect::<rusqlite::Result<BTreeSet<_>>>()?;
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
/// Resolves the ids of the feed items as a device without prototype history
|
||||
/// would, and moves provisional imported episodes onto them.
|
||||
///
|
||||
/// * The ids come from the contract identity rule against every stored
|
||||
/// episode except the provisional ones (episodes from earlier refreshes,
|
||||
/// from sync, and imported ones that already carry personal state).
|
||||
/// * Each provisional episode is matched to the item it was imported from
|
||||
/// with the same rule (enclosure, then link, then date and title; GUIDs
|
||||
/// ignored, since the prototype may have stored the enclosure URL as GUID)
|
||||
/// and re-keyed to that item's id. When another stored episode that stays
|
||||
/// already holds that id, it is the item's episode and the provisional one
|
||||
/// is a duplicate: it is deleted. Unmatched provisional episodes stay as
|
||||
/// they are (not in the feed); the next refresh that lists them re-keys
|
||||
/// them.
|
||||
///
|
||||
/// Provisional episodes have no progress and no session, so no personal
|
||||
/// state moves. The refresh then clears `legacy_guid` on every episode it
|
||||
/// lists, which makes their ids final.
|
||||
fn resolve_and_rekey_provisional(
|
||||
conn: &Connection,
|
||||
podcast_id: &PodcastId,
|
||||
feed: &ParsedFeed,
|
||||
) -> Result<Vec<Option<EpisodeId>>> {
|
||||
let known = apply::known_episodes(conn, podcast_id)?;
|
||||
let provisional = provisional_episode_ids(conn, podcast_id)?;
|
||||
if provisional.is_empty() {
|
||||
return Ok(feed.resolve_episode_ids(podcast_id, &known));
|
||||
}
|
||||
let (provisional_known, stable_known): (Vec<KnownEpisode>, Vec<KnownEpisode>) = known
|
||||
.into_iter()
|
||||
.partition(|episode| provisional.contains(&episode.episode_id));
|
||||
let ids = feed.resolve_episode_ids(podcast_id, &stable_known);
|
||||
|
||||
let guidless: Vec<EpisodeIdentityHints> = feed
|
||||
.identity_hints()
|
||||
.into_iter()
|
||||
.map(|hints| EpisodeIdentityHints {
|
||||
guid: None,
|
||||
..hints
|
||||
})
|
||||
.collect();
|
||||
let matches = resolve_episode_ids(
|
||||
podcast_id,
|
||||
feed.link.as_deref(),
|
||||
&guidless,
|
||||
&provisional_known,
|
||||
);
|
||||
let mut moves: Vec<(EpisodeId, EpisodeId)> = matches
|
||||
.into_iter()
|
||||
.zip(&ids)
|
||||
.filter_map(|(matched, target)| {
|
||||
let matched = matched.filter(|id| provisional.contains(id))?;
|
||||
let target = target.clone()?;
|
||||
(matched != target).then_some((matched, target))
|
||||
})
|
||||
.collect();
|
||||
// A target held by an episode that stays (for example a row a remote
|
||||
// operation created before this first refresh) is the item's episode
|
||||
// already: the provisional episode is a duplicate of it. It has no
|
||||
// progress and no session, and its metadata would be overwritten by the
|
||||
// item anyway, so it is removed instead of kept as a second copy. A
|
||||
// removed duplicate frees its own id for other moves.
|
||||
let mut occupied: BTreeSet<EpisodeId> = stable_known
|
||||
.iter()
|
||||
.chain(&provisional_known)
|
||||
.map(|episode| episode.episode_id.clone())
|
||||
.collect();
|
||||
let mut duplicates: Vec<EpisodeId> = Vec::new();
|
||||
loop {
|
||||
let leaving: BTreeSet<&EpisodeId> = moves.iter().map(|(from, _)| from).collect();
|
||||
let blocked: BTreeSet<EpisodeId> = moves
|
||||
.iter()
|
||||
.filter(|(_, to)| occupied.contains(to) && !leaving.contains(to))
|
||||
.map(|(from, _)| from.clone())
|
||||
.collect();
|
||||
if blocked.is_empty() {
|
||||
break;
|
||||
}
|
||||
moves.retain(|(from, _)| !blocked.contains(from));
|
||||
for from in blocked {
|
||||
occupied.remove(&from);
|
||||
duplicates.push(from);
|
||||
}
|
||||
}
|
||||
for duplicate in &duplicates {
|
||||
// Guarded again: only a provisional episode without personal state.
|
||||
conn.execute(
|
||||
"DELETE FROM episodes
|
||||
WHERE episode_id = ?1 AND legacy_guid IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM episode_progress p
|
||||
WHERE p.episode_id = episodes.episode_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM podcast_listen_sessions s
|
||||
WHERE s.episode_id = episodes.episode_id)",
|
||||
[duplicate.as_str()],
|
||||
)?;
|
||||
}
|
||||
// Two phases, so a move may take an id another move vacates.
|
||||
for (index, (from, _)) in moves.iter().enumerate() {
|
||||
conn.execute(
|
||||
"UPDATE episodes SET episode_id = ?2 WHERE episode_id = ?1",
|
||||
params![from.as_str(), format!("rekey-pending:{index}")],
|
||||
)?;
|
||||
}
|
||||
for (index, (_, to)) in moves.iter().enumerate() {
|
||||
conn.execute(
|
||||
"UPDATE episodes SET episode_id = ?2 WHERE episode_id = ?1",
|
||||
params![format!("rekey-pending:{index}"), to.as_str()],
|
||||
)?;
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
fn upsert_feed_episode(conn: &Connection, meta: &EpisodeMeta, now: i64) -> Result<Option<bool>> {
|
||||
let Some(stored) = super::queries::episode_meta_in(conn, &meta.episode_id)? else {
|
||||
apply::insert_episode(conn, meta, EpisodeSource::Feed, true, None, now, now)?;
|
||||
return Ok(None);
|
||||
};
|
||||
let changed = stored != *meta;
|
||||
conn.execute(
|
||||
"UPDATE episodes
|
||||
SET podcast_id = ?2, guid = ?3, title = ?4, description = ?5, season = ?6,
|
||||
episode_number = ?7, episode_type = ?8, published_at_ms = ?9,
|
||||
duration_ms = ?10, audio_url = ?11, mime = ?12, byte_len = ?13,
|
||||
image_url = ?14, link = ?15, source = 'feed', in_feed = 1,
|
||||
legacy_guid = NULL,
|
||||
updated_at_ms = CASE WHEN ?16 THEN ?17 ELSE updated_at_ms END
|
||||
WHERE episode_id = ?1",
|
||||
params![
|
||||
meta.episode_id.as_str(),
|
||||
meta.podcast_id.as_str(),
|
||||
meta.guid,
|
||||
meta.title,
|
||||
meta.description,
|
||||
meta.season,
|
||||
meta.episode_number,
|
||||
apply::episode_type_to_sql(meta.episode_type),
|
||||
meta.published_at_ms,
|
||||
meta.duration_ms,
|
||||
meta.audio_url,
|
||||
meta.mime,
|
||||
meta.byte_len.and_then(|len| i64::try_from(len).ok()),
|
||||
meta.image_url,
|
||||
meta.link,
|
||||
changed,
|
||||
now,
|
||||
],
|
||||
)?;
|
||||
Ok(Some(changed))
|
||||
}
|
||||
|
||||
impl PodcastStore {
|
||||
/// Ingests a fetched feed document for a podcast.
|
||||
///
|
||||
/// * Channel metadata of the catalog row is replaced by the feed's.
|
||||
/// * Episode ids are resolved with the contract identity rule
|
||||
/// (`ParsedFeed::resolve_episode_ids`) against every stored episode of
|
||||
/// the podcast, including episodes learned through sync, so a changed
|
||||
/// title, audio URL or a GUID-less item keeps its id. Known episodes
|
||||
/// get the new metadata (including the audio URL); new ones are added;
|
||||
/// episodes missing from the document are kept (`in_feed = false`).
|
||||
/// * A permanent redirect target and `itunes:new-feed-url` become feed
|
||||
/// aliases and the new fetch URL. When they change the stored alias
|
||||
/// set, one `podcast_feed_aliases_added` operation is emitted.
|
||||
/// * Subscription, share and progress registers and listening sessions
|
||||
/// are never touched, so a refresh cannot resurrect an unsubscribed
|
||||
/// podcast or move anyone's position.
|
||||
///
|
||||
/// Record the HTTP outcome separately with
|
||||
/// [`PodcastStore::record_fetch_success`] and friends.
|
||||
pub fn apply_feed(
|
||||
&self,
|
||||
stamps: &mut dyn StampSource,
|
||||
refresh: &FeedRefresh<'_>,
|
||||
) -> Result<RefreshOutcome> {
|
||||
let podcast_id = refresh.podcast_id;
|
||||
let feed = refresh.feed;
|
||||
let now = now_ms();
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let current_url = match podcast_fetch_url(&tx, podcast_id)? {
|
||||
Some(url) => url,
|
||||
None => {
|
||||
let Some(register) = apply::load_subscription_register(&tx, podcast_id)? else {
|
||||
return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into());
|
||||
};
|
||||
apply::apply_subscription_record(&tx, ®ister, now)?;
|
||||
register.podcast.feed_url
|
||||
}
|
||||
};
|
||||
|
||||
rekey_legacy_guids(&tx, podcast_id, feed)?;
|
||||
let ids = resolve_and_rekey_provisional(&tx, podcast_id, feed)?;
|
||||
tx.execute(
|
||||
"UPDATE episodes SET in_feed = 0 WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
)?;
|
||||
let mut outcome = RefreshOutcome::default();
|
||||
for (item, id) in feed.episodes.iter().zip(ids) {
|
||||
let Some(id) = id else {
|
||||
outcome.skipped_items += 1;
|
||||
continue;
|
||||
};
|
||||
let meta = item.to_meta(podcast_id, id);
|
||||
if meta.validate().is_err() {
|
||||
outcome.skipped_items += 1;
|
||||
continue;
|
||||
}
|
||||
match upsert_feed_episode(&tx, &meta, now)? {
|
||||
None => outcome.new_episodes += 1,
|
||||
Some(true) => outcome.updated_episodes += 1,
|
||||
Some(false) => outcome.unchanged_episodes += 1,
|
||||
}
|
||||
}
|
||||
|
||||
let mut fetch_url = current_url.clone();
|
||||
let mut learned = BTreeSet::new();
|
||||
if refresh.moved_permanently && *refresh.fetched_from != fetch_url {
|
||||
learned.insert(fetch_url.clone());
|
||||
learned.insert(refresh.fetched_from.clone());
|
||||
fetch_url = refresh.fetched_from.clone();
|
||||
}
|
||||
if let Some(new_url) = feed
|
||||
.new_feed_url
|
||||
.as_deref()
|
||||
.and_then(|url| FeedUrl::parse(url).ok())
|
||||
&& new_url != fetch_url
|
||||
{
|
||||
learned.insert(fetch_url.clone());
|
||||
learned.insert(new_url.clone());
|
||||
fetch_url = new_url;
|
||||
}
|
||||
let meta = feed.to_podcast_meta(fetch_url.clone());
|
||||
tx.execute(
|
||||
"UPDATE podcasts
|
||||
SET feed_url = ?2, title = ?3, author = ?4, description = ?5, link = ?6,
|
||||
image_url = ?7, meta_from_feed = 1, updated_at_ms = ?8
|
||||
WHERE podcast_id = ?1",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
meta.feed_url.as_str(),
|
||||
meta.title,
|
||||
meta.author,
|
||||
meta.description,
|
||||
meta.link,
|
||||
meta.image_url,
|
||||
now
|
||||
],
|
||||
)?;
|
||||
if !learned.is_empty() {
|
||||
let learned: Vec<FeedUrl> = learned.into_iter().collect();
|
||||
let current = apply::load_aliases(&tx, podcast_id)?;
|
||||
if merge_feed_aliases(podcast_id, ¤t, &learned) != current {
|
||||
let op = super::actions::emit(&tx, stamps, |_| {
|
||||
SyncOpPayload::PodcastFeedAliasesAdded {
|
||||
podcast_id: podcast_id.clone(),
|
||||
feed_aliases: learned.clone(),
|
||||
}
|
||||
})?;
|
||||
outcome.ops.push(op);
|
||||
}
|
||||
outcome.learned_aliases = learned;
|
||||
}
|
||||
apply::refresh_seasons(&tx, podcast_id)?;
|
||||
tx.commit()?;
|
||||
outcome.feed_url = Some(fetch_url);
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Local fetch state of a podcast (defaults when never fetched).
|
||||
pub fn fetch_state(&self, podcast_id: &PodcastId) -> Result<FetchState> {
|
||||
fetch_state_in(&self.lock(), podcast_id)
|
||||
}
|
||||
|
||||
/// Read-modify-write of the fetch state under one lock and one immediate
|
||||
/// transaction, so concurrent `record_*` / `set_cover_cache` calls (from
|
||||
/// this or another process) never lose each other's columns or failure
|
||||
/// increments.
|
||||
fn update_fetch_state(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
update: impl FnOnce(&mut FetchState),
|
||||
) -> Result<FetchState> {
|
||||
let mut conn = self.lock();
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
if podcast_fetch_url(&tx, podcast_id)?.is_none() {
|
||||
return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into());
|
||||
}
|
||||
let mut state = fetch_state_in(&tx, podcast_id)?;
|
||||
update(&mut state);
|
||||
tx.execute(
|
||||
"INSERT INTO podcast_fetch_state
|
||||
(podcast_id, etag, last_modified, checked_at_ms, last_success_at_ms,
|
||||
next_check_at_ms, last_error, consecutive_failures, cover_url, cover_path)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||
ON CONFLICT(podcast_id) DO UPDATE SET
|
||||
etag = excluded.etag, last_modified = excluded.last_modified,
|
||||
checked_at_ms = excluded.checked_at_ms,
|
||||
last_success_at_ms = excluded.last_success_at_ms,
|
||||
next_check_at_ms = excluded.next_check_at_ms,
|
||||
last_error = excluded.last_error,
|
||||
consecutive_failures = excluded.consecutive_failures,
|
||||
cover_url = excluded.cover_url, cover_path = excluded.cover_path",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
state.etag,
|
||||
state.last_modified,
|
||||
state.checked_at_ms,
|
||||
state.last_success_at_ms,
|
||||
state.next_check_at_ms,
|
||||
state.last_error,
|
||||
i64::from(state.consecutive_failures),
|
||||
state.cover_url,
|
||||
state.cover_path,
|
||||
],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
/// Records a successful fetch (HTTP 200) with its new validators.
|
||||
pub fn record_fetch_success(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
now_ms: i64,
|
||||
etag: Option<&str>,
|
||||
last_modified: Option<&str>,
|
||||
next_check_at_ms: Option<i64>,
|
||||
) -> Result<()> {
|
||||
self.update_fetch_state(podcast_id, |state| {
|
||||
state.etag = etag.map(str::to_string);
|
||||
state.last_modified = last_modified.map(str::to_string);
|
||||
state.checked_at_ms = Some(now_ms);
|
||||
state.last_success_at_ms = Some(now_ms);
|
||||
state.next_check_at_ms = next_check_at_ms;
|
||||
state.last_error = None;
|
||||
state.consecutive_failures = 0;
|
||||
})
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Records a conditional fetch answered with "304 Not Modified".
|
||||
pub fn record_fetch_not_modified(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
now_ms: i64,
|
||||
next_check_at_ms: Option<i64>,
|
||||
) -> Result<()> {
|
||||
self.update_fetch_state(podcast_id, |state| {
|
||||
state.checked_at_ms = Some(now_ms);
|
||||
state.last_success_at_ms = Some(now_ms);
|
||||
state.next_check_at_ms = next_check_at_ms;
|
||||
state.last_error = None;
|
||||
state.consecutive_failures = 0;
|
||||
})
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Records a failed fetch; returns the consecutive failure count. The
|
||||
/// stored catalog is kept.
|
||||
pub fn record_fetch_failure(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
now_ms: i64,
|
||||
error: &str,
|
||||
next_check_at_ms: Option<i64>,
|
||||
) -> Result<u32> {
|
||||
self.update_fetch_state(podcast_id, |state| {
|
||||
state.checked_at_ms = Some(now_ms);
|
||||
state.next_check_at_ms = next_check_at_ms;
|
||||
state.last_error = Some(error.to_string());
|
||||
state.consecutive_failures = state.consecutive_failures.saturating_add(1);
|
||||
})
|
||||
.map(|state| state.consecutive_failures)
|
||||
}
|
||||
|
||||
/// Records the locally cached cover image of a podcast.
|
||||
pub fn set_cover_cache(
|
||||
&self,
|
||||
podcast_id: &PodcastId,
|
||||
cover_url: Option<&str>,
|
||||
cover_path: Option<&str>,
|
||||
) -> Result<()> {
|
||||
self.update_fetch_state(podcast_id, |state| {
|
||||
state.cover_url = cover_url.map(str::to_string);
|
||||
state.cover_path = cover_path.map(str::to_string);
|
||||
})
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Subscribed podcasts due for a refresh at `now_ms` (never checked or
|
||||
/// `next_check_at_ms <= now_ms`), most overdue first, with their fetch URL.
|
||||
pub fn podcasts_due_for_refresh(
|
||||
&self,
|
||||
now_ms: i64,
|
||||
limit: usize,
|
||||
) -> Result<Vec<(PodcastId, FeedUrl)>> {
|
||||
let conn = self.lock();
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT p.podcast_id, p.feed_url
|
||||
FROM podcasts p
|
||||
JOIN podcast_subscriptions s ON s.podcast_id = p.podcast_id AND s.subscribed = 1
|
||||
LEFT JOIN podcast_fetch_state f ON f.podcast_id = p.podcast_id
|
||||
WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= ?1
|
||||
ORDER BY IFNULL(f.next_check_at_ms, 0), p.podcast_id
|
||||
LIMIT ?2",
|
||||
)?;
|
||||
let due = statement
|
||||
.query_map(
|
||||
params![now_ms, i64::try_from(limit).unwrap_or(i64::MAX)],
|
||||
|row| {
|
||||
Ok((
|
||||
super::podcast_id_col(row.get(0)?)?,
|
||||
super::feed_url_col(row.get(1)?)?,
|
||||
))
|
||||
},
|
||||
)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(due)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,796 @@
|
||||
//! Schema, migrations and the import of the prototype database.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use music_dht::podcast::{
|
||||
EpisodeId, EpisodeIdentityHints, EpisodeMeta, EpisodeType, FeedUrl, KnownEpisode,
|
||||
MAX_DESCRIPTION_CHARS, MAX_TITLE_CHARS, MAX_URL_BYTES, PodcastId, PodcastMeta,
|
||||
SubscriptionRecord, bounded_text, is_http_url, normalize_episode_title, resolve_episode_ids,
|
||||
resolve_podcast_id,
|
||||
};
|
||||
use rusqlite::{Connection, OptionalExtension as _, Transaction, TransactionBehavior, params};
|
||||
|
||||
use super::apply::{self, EpisodeSource};
|
||||
use super::{LEGACY_IMPORT_OP_PREFIX, PodcastStoreError, now_ms};
|
||||
|
||||
/// Newest schema version (`PRAGMA user_version`) this build writes.
|
||||
pub const PODCAST_SCHEMA_VERSION: i64 = 2;
|
||||
|
||||
/// `user_version` of the prototype database of the original podcast PR.
|
||||
pub const LEGACY_PROTOTYPE_SCHEMA_VERSION: i64 = 1;
|
||||
|
||||
/// Migration 1: the prototype schema of the original podcast PR, verbatim
|
||||
/// (`furumi-tui` `src/podcasts/store.rs`, `MIGRATIONS[0]`). It versioned
|
||||
/// itself with `PRAGMA user_version = 1` in the same transaction.
|
||||
const PROTOTYPE_SCHEMA_V1: &str = "
|
||||
CREATE TABLE IF NOT EXISTS podcasts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
feed_url TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
description TEXT,
|
||||
link TEXT,
|
||||
image_url TEXT,
|
||||
cover_url TEXT,
|
||||
cover_path TEXT,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
added_at INTEGER NOT NULL,
|
||||
last_checked_at INTEGER,
|
||||
last_error TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS podcast_episodes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
podcast_id INTEGER NOT NULL REFERENCES podcasts(id) ON DELETE CASCADE,
|
||||
guid TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
published_at INTEGER,
|
||||
duration_seconds REAL,
|
||||
audio_url TEXT NOT NULL,
|
||||
mime_type TEXT,
|
||||
byte_len INTEGER,
|
||||
link TEXT,
|
||||
image_url TEXT,
|
||||
first_seen_at INTEGER NOT NULL,
|
||||
UNIQUE (podcast_id, guid)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS podcast_episodes_by_date
|
||||
ON podcast_episodes (podcast_id, published_at DESC, id DESC);
|
||||
";
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) const PROTOTYPE_SCHEMA_V1_FOR_TESTS: &str = PROTOTYPE_SCHEMA_V1;
|
||||
|
||||
/// Columns every prototype `podcasts` table has.
|
||||
const PROTOTYPE_PODCAST_COLUMNS: &[&str] = &[
|
||||
"id",
|
||||
"feed_url",
|
||||
"title",
|
||||
"author",
|
||||
"description",
|
||||
"link",
|
||||
"image_url",
|
||||
"cover_url",
|
||||
"cover_path",
|
||||
"etag",
|
||||
"last_modified",
|
||||
"added_at",
|
||||
"last_checked_at",
|
||||
"last_error",
|
||||
];
|
||||
|
||||
/// Columns every prototype `podcast_episodes` table has.
|
||||
const PROTOTYPE_EPISODE_COLUMNS: &[&str] = &[
|
||||
"id",
|
||||
"podcast_id",
|
||||
"guid",
|
||||
"title",
|
||||
"description",
|
||||
"published_at",
|
||||
"duration_seconds",
|
||||
"audio_url",
|
||||
"mime_type",
|
||||
"byte_len",
|
||||
"link",
|
||||
"image_url",
|
||||
"first_seen_at",
|
||||
];
|
||||
|
||||
/// Migration 2: the contract model. See `PODCASTS.md`, "SQLite reference store".
|
||||
const SCHEMA_V2: &str = "
|
||||
CREATE TABLE IF NOT EXISTS podcasts (
|
||||
podcast_id TEXT PRIMARY KEY,
|
||||
feed_url TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
author TEXT,
|
||||
description TEXT,
|
||||
link TEXT,
|
||||
image_url TEXT,
|
||||
meta_from_feed INTEGER NOT NULL DEFAULT 0,
|
||||
created_at_ms INTEGER NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS podcast_feed_aliases (
|
||||
podcast_id TEXT NOT NULL,
|
||||
feed_url TEXT NOT NULL,
|
||||
PRIMARY KEY (podcast_id, feed_url)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS podcast_feed_aliases_by_url
|
||||
ON podcast_feed_aliases (feed_url);
|
||||
CREATE TABLE IF NOT EXISTS episodes (
|
||||
episode_id TEXT PRIMARY KEY,
|
||||
podcast_id TEXT NOT NULL REFERENCES podcasts(podcast_id),
|
||||
guid TEXT,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
season INTEGER,
|
||||
episode_number INTEGER,
|
||||
episode_type TEXT NOT NULL DEFAULT 'full',
|
||||
published_at_ms INTEGER,
|
||||
duration_ms INTEGER,
|
||||
audio_url TEXT NOT NULL,
|
||||
mime TEXT,
|
||||
byte_len INTEGER,
|
||||
image_url TEXT,
|
||||
link TEXT,
|
||||
source TEXT NOT NULL CHECK (source IN ('feed', 'sync')),
|
||||
in_feed INTEGER NOT NULL DEFAULT 0,
|
||||
legacy_guid TEXT,
|
||||
first_seen_at_ms INTEGER NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS episodes_by_podcast_date
|
||||
ON episodes (podcast_id, published_at_ms DESC);
|
||||
CREATE INDEX IF NOT EXISTS episodes_by_podcast_season
|
||||
ON episodes (podcast_id, season, episode_number);
|
||||
CREATE TABLE IF NOT EXISTS podcast_seasons (
|
||||
podcast_id TEXT NOT NULL REFERENCES podcasts(podcast_id),
|
||||
season INTEGER,
|
||||
episode_count INTEGER NOT NULL,
|
||||
first_published_at_ms INTEGER,
|
||||
last_published_at_ms INTEGER
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS podcast_seasons_key
|
||||
ON podcast_seasons (podcast_id, IFNULL(season, -1));
|
||||
CREATE TABLE IF NOT EXISTS podcast_subscriptions (
|
||||
podcast_id TEXT PRIMARY KEY,
|
||||
subscribed INTEGER NOT NULL,
|
||||
podcast_json TEXT NOT NULL,
|
||||
generation INTEGER NOT NULL CHECK (generation >= 0),
|
||||
hlc_ms INTEGER NOT NULL,
|
||||
op_id TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS podcast_shares (
|
||||
podcast_id TEXT PRIMARY KEY,
|
||||
shared INTEGER NOT NULL,
|
||||
generation INTEGER NOT NULL CHECK (generation >= 0),
|
||||
hlc_ms INTEGER NOT NULL,
|
||||
op_id TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS episode_progress (
|
||||
episode_id TEXT PRIMARY KEY,
|
||||
podcast_id TEXT NOT NULL,
|
||||
position_ms INTEGER NOT NULL,
|
||||
status TEXT NOT NULL
|
||||
CHECK (status IN ('not_started', 'in_progress', 'completed')),
|
||||
generation INTEGER NOT NULL CHECK (generation >= 0),
|
||||
session_id TEXT NOT NULL,
|
||||
hlc_ms INTEGER NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
op_id TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS episode_progress_by_podcast
|
||||
ON episode_progress (podcast_id);
|
||||
CREATE TABLE IF NOT EXISTS podcast_listen_sessions (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
episode_id TEXT NOT NULL,
|
||||
podcast_id TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
started_at_ms INTEGER NOT NULL,
|
||||
ended_at_ms INTEGER NOT NULL,
|
||||
start_position_ms INTEGER NOT NULL,
|
||||
end_position_ms INTEGER NOT NULL,
|
||||
listened_ms INTEGER NOT NULL,
|
||||
completed INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS podcast_listen_sessions_by_episode
|
||||
ON podcast_listen_sessions (episode_id);
|
||||
CREATE INDEX IF NOT EXISTS podcast_listen_sessions_by_end
|
||||
ON podcast_listen_sessions (ended_at_ms DESC);
|
||||
CREATE TABLE IF NOT EXISTS podcast_fetch_state (
|
||||
podcast_id TEXT PRIMARY KEY REFERENCES podcasts(podcast_id),
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
checked_at_ms INTEGER,
|
||||
last_success_at_ms INTEGER,
|
||||
next_check_at_ms INTEGER,
|
||||
last_error TEXT,
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
cover_url TEXT,
|
||||
cover_path TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS legacy_quarantine (
|
||||
legacy_table TEXT NOT NULL,
|
||||
legacy_id INTEGER NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
detail TEXT,
|
||||
PRIMARY KEY (legacy_table, legacy_id)
|
||||
);
|
||||
";
|
||||
|
||||
/// Tables a version-2 store owns (the `legacy_*` tables included).
|
||||
const V2_TABLES: &[&str] = &[
|
||||
"podcasts",
|
||||
"podcast_feed_aliases",
|
||||
"episodes",
|
||||
"podcast_seasons",
|
||||
"podcast_subscriptions",
|
||||
"podcast_shares",
|
||||
"episode_progress",
|
||||
"podcast_listen_sessions",
|
||||
"podcast_fetch_state",
|
||||
"legacy_quarantine",
|
||||
"legacy_podcasts",
|
||||
"legacy_podcast_episodes",
|
||||
];
|
||||
|
||||
fn user_version(conn: &Connection) -> Result<i64> {
|
||||
Ok(conn.query_row("PRAGMA user_version", [], |row| row.get(0))?)
|
||||
}
|
||||
|
||||
fn user_tables(conn: &Connection) -> Result<BTreeSet<String>> {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
|
||||
)?;
|
||||
let names = statement
|
||||
.query_map([], |row| row.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<BTreeSet<_>>>()?;
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
pub(crate) fn table_columns(conn: &Connection, table: &str) -> Result<BTreeSet<String>> {
|
||||
let mut statement = conn.prepare("SELECT name FROM pragma_table_info(?1)")?;
|
||||
let names = statement
|
||||
.query_map([table], |row| row.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<BTreeSet<_>>>()?;
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
fn unrecognized(reason: impl Into<String>) -> anyhow::Error {
|
||||
PodcastStoreError::UnrecognizedDatabase(reason.into()).into()
|
||||
}
|
||||
|
||||
fn has_columns(conn: &Connection, table: &str, required: &[&str]) -> Result<bool> {
|
||||
let columns = table_columns(conn, table)?;
|
||||
Ok(required.iter().all(|column| columns.contains(*column)))
|
||||
}
|
||||
|
||||
/// Checks the prototype tables that exist have the prototype shape.
|
||||
fn check_prototype_shape(conn: &Connection, tables: &BTreeSet<String>) -> Result<()> {
|
||||
if tables.contains("podcasts") && !has_columns(conn, "podcasts", PROTOTYPE_PODCAST_COLUMNS)? {
|
||||
return Err(unrecognized(
|
||||
"table `podcasts` does not have the prototype columns",
|
||||
));
|
||||
}
|
||||
if tables.contains("podcast_episodes")
|
||||
&& !has_columns(conn, "podcast_episodes", PROTOTYPE_EPISODE_COLUMNS)?
|
||||
{
|
||||
return Err(unrecognized(
|
||||
"table `podcast_episodes` does not have the prototype columns",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read-only check run before anything is written to the file.
|
||||
///
|
||||
/// * `user_version` above [`PODCAST_SCHEMA_VERSION`]: written by a newer
|
||||
/// Furumi, refused.
|
||||
/// * 0: must be empty, or hold only the prototype tables (the prototype
|
||||
/// sets its version in the same transaction, so this is defensive).
|
||||
/// * 1: the prototype database; its tables must have the prototype shape.
|
||||
/// * 2: the current schema.
|
||||
///
|
||||
/// Any other table means the path points at a different database (for
|
||||
/// example `library.db`), which is refused instead of being modified.
|
||||
pub(super) fn check_openable(conn: &Connection) -> Result<()> {
|
||||
let version = user_version(conn)?;
|
||||
if version > PODCAST_SCHEMA_VERSION {
|
||||
return Err(PodcastStoreError::NewerSchema {
|
||||
found: version,
|
||||
supported: PODCAST_SCHEMA_VERSION,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
if version < 0 {
|
||||
return Err(unrecognized(format!("negative user_version {version}")));
|
||||
}
|
||||
let tables = user_tables(conn)?;
|
||||
let allowed: &[&str] = if version < PODCAST_SCHEMA_VERSION {
|
||||
&["podcasts", "podcast_episodes"]
|
||||
} else {
|
||||
V2_TABLES
|
||||
};
|
||||
if let Some(table) = tables
|
||||
.iter()
|
||||
.find(|table| !allowed.contains(&table.as_str()))
|
||||
{
|
||||
return Err(unrecognized(format!(
|
||||
"unexpected table `{table}` at schema version {version}"
|
||||
)));
|
||||
}
|
||||
if version < PODCAST_SCHEMA_VERSION {
|
||||
check_prototype_shape(conn, &tables)?;
|
||||
} else if tables.contains("podcasts")
|
||||
&& !has_columns(conn, "podcasts", &["podcast_id", "feed_url"])?
|
||||
{
|
||||
return Err(unrecognized("table `podcasts` is not the version 2 table"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Applies every pending migration, each in its own immediate transaction
|
||||
/// that re-reads the version (another process may have migrated meanwhile).
|
||||
pub(super) fn migrate(conn: &mut Connection) -> Result<()> {
|
||||
for target in 1..=PODCAST_SCHEMA_VERSION {
|
||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let version = user_version(&tx)?;
|
||||
if version > PODCAST_SCHEMA_VERSION {
|
||||
return Err(PodcastStoreError::NewerSchema {
|
||||
found: version,
|
||||
supported: PODCAST_SCHEMA_VERSION,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
if version >= target {
|
||||
continue;
|
||||
}
|
||||
match target {
|
||||
1 => tx
|
||||
.execute_batch(PROTOTYPE_SCHEMA_V1)
|
||||
.context("creating the prototype podcast schema (v1)")?,
|
||||
2 => migrate_to_v2(&tx).context("migrating the podcast database to v2")?,
|
||||
_ => unreachable!("no migration for version {target}"),
|
||||
}
|
||||
tx.pragma_update(None, "user_version", target)?;
|
||||
tx.commit()
|
||||
.with_context(|| format!("committing podcast schema v{target}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_to_v2(tx: &Transaction<'_>) -> Result<()> {
|
||||
let tables = user_tables(tx)?;
|
||||
// Rename, never drop: the prototype data stays available as `legacy_*`.
|
||||
// Renaming `podcasts` also rewrites the foreign key of the episode table.
|
||||
if tables.contains("podcasts") && !tables.contains("legacy_podcasts") {
|
||||
tx.execute_batch("ALTER TABLE podcasts RENAME TO legacy_podcasts")?;
|
||||
}
|
||||
if tables.contains("podcast_episodes") && !tables.contains("legacy_podcast_episodes") {
|
||||
tx.execute_batch("ALTER TABLE podcast_episodes RENAME TO legacy_podcast_episodes")?;
|
||||
}
|
||||
tx.execute_batch(SCHEMA_V2)?;
|
||||
let tables = user_tables(tx)?;
|
||||
if tables.contains("legacy_podcasts") {
|
||||
import_prototype(tx, tables.contains("legacy_podcast_episodes"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct LegacyPodcast {
|
||||
id: i64,
|
||||
feed_url: String,
|
||||
title: String,
|
||||
author: Option<String>,
|
||||
description: Option<String>,
|
||||
link: Option<String>,
|
||||
image_url: Option<String>,
|
||||
cover_url: Option<String>,
|
||||
cover_path: Option<String>,
|
||||
etag: Option<String>,
|
||||
last_modified: Option<String>,
|
||||
added_at: i64,
|
||||
last_checked_at: Option<i64>,
|
||||
last_error: Option<String>,
|
||||
}
|
||||
|
||||
struct LegacyEpisode {
|
||||
id: i64,
|
||||
guid: String,
|
||||
title: String,
|
||||
description: Option<String>,
|
||||
published_at: Option<i64>,
|
||||
duration_seconds: Option<f64>,
|
||||
audio_url: String,
|
||||
mime_type: Option<String>,
|
||||
byte_len: Option<i64>,
|
||||
link: Option<String>,
|
||||
image_url: Option<String>,
|
||||
first_seen_at: i64,
|
||||
}
|
||||
|
||||
fn quarantine(
|
||||
tx: &Transaction<'_>,
|
||||
table: &str,
|
||||
id: i64,
|
||||
reason: &str,
|
||||
detail: &str,
|
||||
) -> Result<()> {
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO legacy_quarantine (legacy_table, legacy_id, reason, detail)
|
||||
VALUES (?1, ?2, ?3, ?4)",
|
||||
params![table, id, reason, detail],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn secs_to_ms(secs: i64) -> i64 {
|
||||
secs.saturating_mul(1_000)
|
||||
}
|
||||
|
||||
/// Imports the prototype tables into the contract model.
|
||||
///
|
||||
/// * Every prototype row is an active subscription: it becomes a
|
||||
/// subscription register with `subscribed = true`, generation 0,
|
||||
/// `hlc_ms` = `added_at` and `op_id` = [`LEGACY_IMPORT_OP_PREFIX`] + id,
|
||||
/// so any later contract write orders after it. Other devices learn it
|
||||
/// through the snapshot sections. Nothing is shared (private by default).
|
||||
/// * The podcast id is derived from the canonical feed URL
|
||||
/// ([`FeedUrl::parse`], falling back to [`FeedUrl::from_user_input`]); rows
|
||||
/// whose URL cannot be canonicalized are recorded in `legacy_quarantine`
|
||||
/// and stay in `legacy_podcasts` with their episodes. Rows with the same
|
||||
/// canonical URL merge into one podcast (the first row's subscription and
|
||||
/// fetch state win; the episodes of all of them are imported).
|
||||
/// * Episodes get ids from the contract identity rule over the stored rows.
|
||||
/// The prototype stored the enclosure URL as `guid` when the item had
|
||||
/// none: such rows are imported as GUID-less items and keep the stored
|
||||
/// value in `legacy_guid`. Rows of one GUID-less item seen under several
|
||||
/// enclosure URLs are collapsed into the newest (the others are
|
||||
/// quarantined as duplicates). Because stale rows can still change which
|
||||
/// key is unique, these ids are provisional: the first refresh that lists
|
||||
/// the item re-keys an episode without progress or sessions to the id the
|
||||
/// current feed gives (see `refresh.rs`), including its GUID id when the
|
||||
/// stored value turns out to be a real GUID.
|
||||
/// * ETag, Last-Modified, check time, last error and the cover cache move to
|
||||
/// the local fetch state. The prototype has no progress or history, so no
|
||||
/// progress register or session is created.
|
||||
fn import_prototype(tx: &Transaction<'_>, with_episodes: bool) -> Result<()> {
|
||||
let now = now_ms();
|
||||
let legacy: Vec<LegacyPodcast> = {
|
||||
let mut statement = tx.prepare(
|
||||
"SELECT id, feed_url, title, author, description, link, image_url, cover_url,
|
||||
cover_path, etag, last_modified, added_at, last_checked_at, last_error
|
||||
FROM legacy_podcasts ORDER BY id",
|
||||
)?;
|
||||
statement
|
||||
.query_map([], |row| {
|
||||
Ok(LegacyPodcast {
|
||||
id: row.get(0)?,
|
||||
feed_url: row.get(1)?,
|
||||
title: row.get(2)?,
|
||||
author: row.get(3)?,
|
||||
description: row.get(4)?,
|
||||
link: row.get(5)?,
|
||||
image_url: row.get(6)?,
|
||||
cover_url: row.get(7)?,
|
||||
cover_path: row.get(8)?,
|
||||
etag: row.get(9)?,
|
||||
last_modified: row.get(10)?,
|
||||
added_at: row.get(11)?,
|
||||
last_checked_at: row.get(12)?,
|
||||
last_error: row.get(13)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let mut known_urls: Vec<(PodcastId, FeedUrl)> = Vec::new();
|
||||
for podcast in legacy {
|
||||
let url = match FeedUrl::parse(&podcast.feed_url)
|
||||
.or_else(|_| FeedUrl::from_user_input(&podcast.feed_url))
|
||||
{
|
||||
Ok(url) => url,
|
||||
Err(error) => {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcasts",
|
||||
podcast.id,
|
||||
"invalid feed URL",
|
||||
&error.to_string(),
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let podcast_id = resolve_podcast_id(&url, known_urls.iter().map(|(id, url)| (id, url)));
|
||||
let duplicate = known_urls.iter().any(|(id, _)| *id == podcast_id);
|
||||
known_urls.push((podcast_id.clone(), url.clone()));
|
||||
if duplicate {
|
||||
// Same canonical feed as an earlier row: its episodes join that
|
||||
// podcast; the first row's subscription and fetch state are kept.
|
||||
if with_episodes {
|
||||
import_prototype_episodes(tx, &podcast, &podcast_id, now)?;
|
||||
}
|
||||
apply::refresh_seasons(tx, &podcast_id)?;
|
||||
continue;
|
||||
}
|
||||
let meta = PodcastMeta {
|
||||
feed_url: url.clone(),
|
||||
title: podcast.title.clone(),
|
||||
author: podcast.author.clone(),
|
||||
description: podcast.description.clone(),
|
||||
link: podcast.link.clone(),
|
||||
image_url: podcast.image_url.clone(),
|
||||
}
|
||||
.bounded();
|
||||
let added_ms = secs_to_ms(podcast.added_at);
|
||||
let record = SubscriptionRecord {
|
||||
podcast_id: podcast_id.clone(),
|
||||
subscribed: true,
|
||||
podcast: meta.clone(),
|
||||
feed_aliases: vec![url.clone()],
|
||||
generation: 0,
|
||||
hlc_ms: added_ms,
|
||||
op_id: format!("{LEGACY_IMPORT_OP_PREFIX}{podcast_id}"),
|
||||
};
|
||||
apply::apply_subscription_record(tx, &record, now)?;
|
||||
// The prototype metadata came from the feed itself.
|
||||
tx.execute(
|
||||
"UPDATE podcasts
|
||||
SET title = ?2, author = ?3, description = ?4, link = ?5, image_url = ?6,
|
||||
meta_from_feed = 1, created_at_ms = MIN(created_at_ms, ?7)
|
||||
WHERE podcast_id = ?1 AND meta_from_feed = 0",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
meta.title,
|
||||
meta.author,
|
||||
meta.description,
|
||||
meta.link,
|
||||
meta.image_url,
|
||||
added_ms
|
||||
],
|
||||
)?;
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO podcast_fetch_state
|
||||
(podcast_id, etag, last_modified, checked_at_ms, last_success_at_ms,
|
||||
next_check_at_ms, last_error, consecutive_failures, cover_url, cover_path)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, NULL, ?6, ?7, ?8, ?9)",
|
||||
params![
|
||||
podcast_id.as_str(),
|
||||
podcast.etag,
|
||||
podcast.last_modified,
|
||||
podcast.last_checked_at.map(secs_to_ms),
|
||||
podcast
|
||||
.last_error
|
||||
.is_none()
|
||||
.then(|| podcast.last_checked_at.map(secs_to_ms))
|
||||
.flatten(),
|
||||
podcast.last_error,
|
||||
i64::from(podcast.last_error.is_some()),
|
||||
podcast.cover_url,
|
||||
podcast.cover_path,
|
||||
],
|
||||
)?;
|
||||
if with_episodes {
|
||||
import_prototype_episodes(tx, &podcast, &podcast_id, now)?;
|
||||
}
|
||||
apply::refresh_seasons(tx, &podcast_id)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn import_prototype_episodes(
|
||||
tx: &Transaction<'_>,
|
||||
podcast: &LegacyPodcast,
|
||||
podcast_id: &PodcastId,
|
||||
now: i64,
|
||||
) -> Result<()> {
|
||||
let rows: Vec<LegacyEpisode> = {
|
||||
let mut statement = tx.prepare(
|
||||
"SELECT id, guid, title, description, published_at, duration_seconds, audio_url,
|
||||
mime_type, byte_len, link, image_url, first_seen_at
|
||||
FROM legacy_podcast_episodes WHERE podcast_id = ?1 ORDER BY id",
|
||||
)?;
|
||||
statement
|
||||
.query_map([podcast.id], |row| {
|
||||
Ok(LegacyEpisode {
|
||||
id: row.get(0)?,
|
||||
guid: row.get(1)?,
|
||||
title: row.get(2)?,
|
||||
description: row.get(3)?,
|
||||
published_at: row.get(4)?,
|
||||
duration_seconds: row.get(5)?,
|
||||
audio_url: row.get(6)?,
|
||||
mime_type: row.get(7)?,
|
||||
byte_len: row.get(8)?,
|
||||
link: row.get(9)?,
|
||||
image_url: row.get(10)?,
|
||||
first_seen_at: row.get(11)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
};
|
||||
let mut usable = Vec::new();
|
||||
for row in rows {
|
||||
if !is_http_url(&row.audio_url) {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"invalid audio URL",
|
||||
&row.audio_url,
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
usable.push(row);
|
||||
}
|
||||
// The prototype used the enclosure URL as GUID when the item had none.
|
||||
let real_guid = |row: &LegacyEpisode| {
|
||||
let guid = row.guid.trim();
|
||||
(!guid.is_empty() && guid != row.audio_url.trim()).then(|| guid.to_string())
|
||||
};
|
||||
let usable = collapse_moved_guidless_rows(tx, usable, real_guid)?;
|
||||
let hints: Vec<EpisodeIdentityHints> = usable
|
||||
.iter()
|
||||
.map(|row| EpisodeIdentityHints {
|
||||
guid: real_guid(row),
|
||||
enclosure_url: Some(row.audio_url.clone()),
|
||||
link: row.link.clone(),
|
||||
published_at_ms: row.published_at.map(secs_to_ms),
|
||||
title: Some(row.title.clone()),
|
||||
})
|
||||
.collect();
|
||||
let known: Vec<KnownEpisode> = apply::known_episodes(tx, podcast_id)?;
|
||||
let ids = resolve_episode_ids(podcast_id, podcast.link.as_deref(), &hints, &known);
|
||||
let mut seen: BTreeMap<EpisodeId, i64> = BTreeMap::new();
|
||||
for ((row, hint), id) in usable.iter().zip(&hints).zip(ids) {
|
||||
let Some(episode_id) = id else {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"episode cannot be identified",
|
||||
"no usable identity key or duplicate item",
|
||||
)?;
|
||||
continue;
|
||||
};
|
||||
if let Some(first) = seen.insert(episode_id.clone(), row.id) {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"duplicate episode",
|
||||
&format!("same identity as legacy episode {first}"),
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
let meta = EpisodeMeta {
|
||||
episode_id,
|
||||
podcast_id: podcast_id.clone(),
|
||||
guid: hint.guid.clone(),
|
||||
title: bounded_title(&row.title),
|
||||
description: row
|
||||
.description
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|text| !text.is_empty())
|
||||
.map(|text| bounded_text(text, MAX_DESCRIPTION_CHARS)),
|
||||
season: None,
|
||||
episode_number: None,
|
||||
episode_type: EpisodeType::Full,
|
||||
published_at_ms: row.published_at.map(secs_to_ms),
|
||||
duration_ms: row
|
||||
.duration_seconds
|
||||
.filter(|secs| secs.is_finite() && *secs > 0.0)
|
||||
.map(|secs| (secs * 1_000.0).round() as i64),
|
||||
audio_url: row.audio_url.clone(),
|
||||
mime: row.mime_type.clone().filter(|mime| {
|
||||
!mime.is_empty() && mime.len() <= 128 && mime.bytes().all(|b| b.is_ascii_graphic())
|
||||
}),
|
||||
byte_len: row.byte_len.and_then(|len| u64::try_from(len).ok()),
|
||||
image_url: row.image_url.clone().filter(|url| is_http_url(url)),
|
||||
link: row.link.clone().filter(|link| link.len() <= MAX_URL_BYTES),
|
||||
};
|
||||
if let Err(error) = meta.validate() {
|
||||
quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"invalid episode metadata",
|
||||
&error.to_string(),
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
let legacy_guid = hint.guid.is_none().then(|| row.guid.clone());
|
||||
let exists: bool = tx
|
||||
.query_row(
|
||||
"SELECT 1 FROM episodes WHERE episode_id = ?1",
|
||||
[meta.episode_id.as_str()],
|
||||
|_| Ok(true),
|
||||
)
|
||||
.optional()?
|
||||
.unwrap_or(false);
|
||||
if exists {
|
||||
continue;
|
||||
}
|
||||
apply::insert_episode(
|
||||
tx,
|
||||
&meta,
|
||||
EpisodeSource::Feed,
|
||||
true,
|
||||
legacy_guid.as_deref(),
|
||||
secs_to_ms(row.first_seen_at),
|
||||
now,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Collapses prototype rows that are one GUID-less item seen under several
|
||||
/// enclosure URLs.
|
||||
///
|
||||
/// The prototype keyed GUID-less items by their enclosure URL, so an item
|
||||
/// whose audio URL changed (new CDN, tracking prefix) left one row per URL.
|
||||
/// Those stale rows are not part of the current feed; resolved together with
|
||||
/// it they would make the item's `<link>` and `(date, title)` look
|
||||
/// non-unique and give it an id a fresh device never derives. Rows with the
|
||||
/// same publication second, normalized title and `<link>` (or none) are the
|
||||
/// same item: the newest (`first_seen_at`, then `id`) is imported, the others
|
||||
/// are quarantined as duplicates. The link must match too, so distinct
|
||||
/// episodes that only share a title and second are never merged.
|
||||
fn collapse_moved_guidless_rows(
|
||||
tx: &Transaction<'_>,
|
||||
rows: Vec<LegacyEpisode>,
|
||||
real_guid: impl Fn(&LegacyEpisode) -> Option<String>,
|
||||
) -> Result<Vec<LegacyEpisode>> {
|
||||
type ItemKey = (i64, String, Option<String>);
|
||||
let key = |row: &LegacyEpisode| -> Option<ItemKey> {
|
||||
if real_guid(row).is_some() {
|
||||
return None;
|
||||
}
|
||||
let title = normalize_episode_title(&row.title);
|
||||
let link = row
|
||||
.link
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|link| !link.is_empty())
|
||||
.map(str::to_string);
|
||||
(!title.is_empty()).then_some((row.published_at?, title, link))
|
||||
};
|
||||
let mut newest: BTreeMap<ItemKey, (i64, i64)> = BTreeMap::new();
|
||||
for row in &rows {
|
||||
if let Some(key) = key(row) {
|
||||
let candidate = (row.first_seen_at, row.id);
|
||||
newest
|
||||
.entry(key)
|
||||
.and_modify(|best| *best = (*best).max(candidate))
|
||||
.or_insert(candidate);
|
||||
}
|
||||
}
|
||||
let mut kept = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let keeper = key(&row).and_then(|key| newest.get(&key).map(|(_, id)| *id));
|
||||
match keeper {
|
||||
Some(keeper) if keeper != row.id => quarantine(
|
||||
tx,
|
||||
"legacy_podcast_episodes",
|
||||
row.id,
|
||||
"duplicate episode",
|
||||
&format!(
|
||||
"older enclosure URL of the GUID-less item imported from legacy episode {keeper}"
|
||||
),
|
||||
)?,
|
||||
_ => kept.push(row),
|
||||
}
|
||||
}
|
||||
Ok(kept)
|
||||
}
|
||||
|
||||
fn bounded_title(title: &str) -> String {
|
||||
let title = title.trim();
|
||||
if title.is_empty() {
|
||||
"Untitled episode".to_string()
|
||||
} else {
|
||||
bounded_text(title, MAX_TITLE_CHARS)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "music-dht"
|
||||
version = "0.4.1"
|
||||
version = "0.6.0"
|
||||
description = "Distributed music library search: a Kademlia-style DHT on top of federation-net"
|
||||
readme = "README.md"
|
||||
documentation = "https://docs.rs/music-dht"
|
||||
@@ -26,6 +26,13 @@ thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
data-encoding = { workspace = true }
|
||||
quick-xml = { workspace = true, optional = true }
|
||||
encoding_rs = { workspace = true, optional = true }
|
||||
url = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
# RSS 2.0 / RSS 1.0 / Atom podcast feed parser (`music_dht::podcast::feed`).
|
||||
podcast-feed = ["dep:quick-xml", "dep:encoding_rs", "dep:url"]
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow = { workspace = true }
|
||||
|
||||
@@ -183,7 +183,101 @@ to know that the integration exists.
|
||||
Protocol v1 uses `furumi/sync/1`; v2 uses `furumi/sync/2`. During migration,
|
||||
new clients should accept both ALPNs and only send `ListenRecorded` when
|
||||
`supports_listen_history(peer.protocol_version)` returns true. Likes,
|
||||
playlists, membership and playback coordination remain compatible with v1.
|
||||
playlists and membership remain compatible with v1. Profile version 3 adds
|
||||
versioned personal playback coordination over the existing JSON sync stream;
|
||||
legacy unfenced playback messages are ignored by upgraded clients. Profile
|
||||
version 4 adds podcasts (see below); `op_allowed_for_peer` applies all of
|
||||
these rules to one operation.
|
||||
|
||||
`music_dht::playback` provides the shared pure ownership engine and generic
|
||||
snapshot/command envelopes. Configure startup and failover with `Config`, feed
|
||||
actual output reports, and persist `Checkpoint` before applying decisions.
|
||||
`Config::passive()` is intended for always-running gateways. See the workspace
|
||||
`PLAYBACK_PROTOCOL.md` for invariants, configuration and migration details.
|
||||
|
||||
## Podcasts
|
||||
|
||||
`music_dht::podcast` is the shared, pure podcast contract (no SQLite, HTTP or
|
||||
clock). Clients use it for identity, merge rules and wire DTOs; see the
|
||||
workspace `PODCASTS.md` for the normative rules.
|
||||
|
||||
```rust
|
||||
use music_dht::podcast::{
|
||||
EpisodeIdentityHints, FeedUrl, PodcastId, ProgressRecord, resolve_episode_ids,
|
||||
};
|
||||
|
||||
let url = FeedUrl::from_user_input("feed://example.com/rss?token=abc")?;
|
||||
let podcast = PodcastId::from_feed_url(&url); // "pod1:..."
|
||||
let hints = [EpisodeIdentityHints {
|
||||
guid: Some("episode-1".into()),
|
||||
..Default::default()
|
||||
}];
|
||||
let ids = resolve_episode_ids(&podcast, None, &hints, &[]); // "ep1:..."
|
||||
|
||||
// Explicit actions (seek, start over, new session) bump the generation.
|
||||
let generation = ProgressRecord::explicit_generation(Some(4)); // 5
|
||||
```
|
||||
|
||||
* `FeedUrl` canonicalizes feed URLs and keeps every query parameter.
|
||||
* `PodcastId`, `EpisodeId`, `EpisodeKey`, `resolve_podcast_id`,
|
||||
`merge_feed_aliases`, `resolve_episode_ids`: identity, aliases and the
|
||||
no-GUID rule.
|
||||
* `PodcastMeta`, `EpisodeMeta`, `PodcastEpisodeRef`: bounded metadata and the
|
||||
self-contained queue item.
|
||||
* `SubscriptionRecord`, `ShareRecord`, `ProgressRecord`, `ListenSession` with
|
||||
`merge_subscription`, `merge_share`, `merge_progress`,
|
||||
`merge_listen_session`, `next_podcast_generation`, `effective_share`,
|
||||
`reaches_completion` and `completed_listen_count`.
|
||||
|
||||
Enable the `podcast-feed` feature for `music_dht::podcast::feed`, the shared
|
||||
tolerant RSS 2.0 / RSS 1.0 / Atom parser:
|
||||
|
||||
```toml
|
||||
music-dht = { version = "0.6", features = ["podcast-feed"] }
|
||||
```
|
||||
|
||||
Device-sync protocol v4 adds `podcast_subscription_set`, `podcast_share_set`,
|
||||
`podcast_feed_aliases_added`, `podcast_progress_set` and
|
||||
`podcast_listen_recorded`, podcast snapshot sections, `PlaybackTrack.source`
|
||||
and a reserved `playback_rate`. Old clients cannot parse the new operation
|
||||
kinds and ignore `source`, so always filter with `op_allowed_for_peer`,
|
||||
`playback_allowed_for_peer`, `SyncSnapshot::for_peer` and
|
||||
`playback_command_allowed_from_peer`, using the protocol version each device
|
||||
reported itself (`profile_protocol_version`; relayed profiles go through
|
||||
`relayed_protocol_version`, pairing requests use
|
||||
`InviteWire::inviter_protocol_version`). Bound snapshots with
|
||||
`SyncSnapshot::bound_podcast_state_to(podcast_snapshot_budget(..))`. Jam hosts use
|
||||
`jam::redact_jam_snapshot`, `jam_command_allowed` and
|
||||
`restore_jam_private_items`.
|
||||
|
||||
### Public podcast discovery
|
||||
|
||||
`music_dht::podcast_dht::PodcastDht` is a separate signed overlay (ALPN
|
||||
`furumi-fd/podcast-dht/1`, capability `podcast_dht`) for podcasts the user
|
||||
explicitly shared. Wiring, as for `SimilarityDht`:
|
||||
|
||||
1. declare the ALPN with `schema_independent_stream_protocol(PODCAST_DHT_ALPN)`,
|
||||
open the overlay with its own cache file, take
|
||||
`stream_acceptor(PODCAST_DHT_ALPN)` and spawn `serve` and `maintenance`;
|
||||
2. advertise it with `CapabilityManifest::with_podcast_dht()` (it is not in
|
||||
`CapabilityManifest::frid()`; an application that does not run the overlay
|
||||
simply does not add it);
|
||||
3. pass every peer manifest received to `note_peer_capabilities`, so peers
|
||||
without the overlay are not dialed;
|
||||
4. call `sync_shares` with the merged subscription and share registers on
|
||||
start and after applying sync operations (until the first call, own cards
|
||||
persisted by an earlier run are neither republished nor served): it
|
||||
publishes cards only where `effective_share` holds, tombstones every other
|
||||
own publication, and reports cards it could not build (`failed`) or held
|
||||
back for privacy (`held_back`);
|
||||
5. ask the user before publishing when `PodcastCardSpec::privacy_warnings`
|
||||
(or `feed_url_privacy_warning`) reports a URL that looks private, then
|
||||
call `publish` or set `PodcastShareState::privacy_confirmed`.
|
||||
|
||||
`search` and `lookup_podcast` return `PodcastSearchHit`s; subscribe with the
|
||||
hit's `feed_url` (the URL the id hashes from), never with a
|
||||
`claimed_feed_urls` entry. `merge_search_results` combines hits with
|
||||
`SearchOutcome::network_results` as typed `FederatedSearchHit`s.
|
||||
|
||||
## Verification
|
||||
|
||||
@@ -191,6 +285,7 @@ playlists, membership and playback coordination remain compatible with v1.
|
||||
cargo fmt --check
|
||||
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||
cargo test -p music-dht
|
||||
cargo test -p music-dht --features podcast-feed
|
||||
```
|
||||
|
||||
The integration test starts three real nodes in one process (they use Iroh's
|
||||
|
||||
@@ -35,6 +35,8 @@ pub const CATALOG_ID: &str = "catalog";
|
||||
pub const SIMILARITY_ID: &str = "similarity";
|
||||
/// Stable protocol identifier for DHT-routed similarity discovery.
|
||||
pub const SIMILARITY_DHT_ID: &str = "similarity_dht";
|
||||
/// Stable protocol identifier for the public podcast-discovery overlay.
|
||||
pub const PODCAST_DHT_ID: &str = "podcast_dht";
|
||||
/// Stable protocol identifier for personal-device synchronization.
|
||||
pub const DEVICE_SYNC_ID: &str = "device_sync";
|
||||
/// Stable protocol identifier for Jam playback control.
|
||||
@@ -54,7 +56,13 @@ pub struct CapabilityManifest {
|
||||
}
|
||||
|
||||
impl CapabilityManifest {
|
||||
/// Creates a manifest containing every protocol owned by Frid.
|
||||
/// Creates a manifest containing every protocol owned by Frid that a
|
||||
/// music-DHT application serves.
|
||||
///
|
||||
/// The public podcast-discovery overlay is opt-in: applications that run
|
||||
/// [`crate::podcast_dht::PodcastDht`] add it with
|
||||
/// [`Self::with_podcast_dht`], so an application that does not serve the
|
||||
/// overlay never advertises it.
|
||||
pub fn frid(application: impl Into<String>, application_version: impl Into<String>) -> Self {
|
||||
let mut protocols = BTreeMap::new();
|
||||
protocols.insert(
|
||||
@@ -105,6 +113,29 @@ impl CapabilityManifest {
|
||||
self
|
||||
}
|
||||
|
||||
/// Advertises the public podcast-discovery overlay
|
||||
/// ([`PODCAST_DHT_ID`] at [`crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||
/// Call it only when the application serves
|
||||
/// [`crate::podcast_dht::PODCAST_DHT_ALPN`].
|
||||
pub fn with_podcast_dht(self) -> Self {
|
||||
self.with_protocol(
|
||||
PODCAST_DHT_ID,
|
||||
crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION,
|
||||
)
|
||||
}
|
||||
|
||||
/// Version of protocol `id` advertised by the peer, if any.
|
||||
pub fn protocol_version(&self, id: &str) -> Option<u16> {
|
||||
self.protocols.get(id).copied()
|
||||
}
|
||||
|
||||
/// Whether the peer advertises the public podcast-discovery overlay at
|
||||
/// the version this crate speaks. Peers without it are never queried.
|
||||
pub fn supports_podcast_dht(&self) -> bool {
|
||||
self.protocol_version(PODCAST_DHT_ID)
|
||||
== Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
|
||||
}
|
||||
|
||||
/// Validates bounds and identifiers received from a peer.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.manifest_version != CAPABILITIES_PROTOCOL_VERSION {
|
||||
@@ -227,6 +258,24 @@ mod tests {
|
||||
manifest.validate().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_dht_capability_is_opt_in_and_removable() {
|
||||
let plain = CapabilityManifest::frid("furumi", "1.2.3");
|
||||
assert_eq!(plain.protocol_version(PODCAST_DHT_ID), None);
|
||||
assert!(!plain.supports_podcast_dht());
|
||||
let manifest = plain.with_podcast_dht();
|
||||
assert_eq!(
|
||||
manifest.protocol_version(PODCAST_DHT_ID),
|
||||
Some(crate::podcast_dht::PODCAST_DHT_PROTOCOL_VERSION)
|
||||
);
|
||||
assert!(manifest.supports_podcast_dht());
|
||||
manifest.validate().unwrap();
|
||||
let without = manifest.clone().without_protocol(PODCAST_DHT_ID);
|
||||
assert!(!without.supports_podcast_dht());
|
||||
let other_version = manifest.with_protocol(PODCAST_DHT_ID, 2);
|
||||
assert!(!other_version.supports_podcast_dht());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn application_protocol_is_additive() {
|
||||
let manifest = CapabilityManifest::frid("furumi", "1.2.3").with_protocol("audio", 1);
|
||||
|
||||
+1637
-43
File diff suppressed because it is too large.
Load diff
@@ -50,6 +50,11 @@ pub enum MusicDhtError {
|
||||
/// The service is shutting down and no longer accepts operations.
|
||||
#[error("service is shutting down")]
|
||||
ShuttingDown,
|
||||
|
||||
/// A podcast card cannot be published, for example because the user did
|
||||
/// not share the podcast (see `music_dht::podcast::effective_share`).
|
||||
#[error("podcast publication refused: {0}")]
|
||||
PodcastPublication(String),
|
||||
}
|
||||
|
||||
impl From<federation_net::NetworkError> for MusicDhtError {
|
||||
|
||||
@@ -3,6 +3,12 @@
|
||||
//! Jam is separate from personal-device sync. Possession of a [`JamInvite`]
|
||||
//! capability authorizes playback control for one host process, but never
|
||||
//! grants likes, playlists, history, or trusted-device membership.
|
||||
//!
|
||||
//! Podcast episodes are private. Hosts send [`redact_jam_snapshot`] to
|
||||
//! participants, reject participant commands for which
|
||||
//! [`crate::device_sync::jam_command_allowed`] is false, and map placeholders
|
||||
//! back with [`crate::device_sync::restore_jam_private_items`] before applying
|
||||
//! a participant's state.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
@@ -140,6 +146,17 @@ pub enum JamWireMessage {
|
||||
},
|
||||
}
|
||||
|
||||
/// Returns a playback snapshot safe to send to Jam participants.
|
||||
///
|
||||
/// Podcast episodes become opaque placeholders (see
|
||||
/// [`crate::device_sync::redact_for_jam`]) and the personal coordination
|
||||
/// envelope is removed, since it belongs to the trusted-device scope.
|
||||
pub fn redact_jam_snapshot(mut snapshot: PlaybackSnapshot) -> PlaybackSnapshot {
|
||||
snapshot.state = crate::device_sync::redact_for_jam(snapshot.state);
|
||||
snapshot.coordination = None;
|
||||
snapshot
|
||||
}
|
||||
|
||||
/// Reads a bounded Jam message from an async reader.
|
||||
pub async fn read_message_from<R: AsyncRead + Unpin>(reader: &mut R) -> Result<JamWireMessage> {
|
||||
let mut line = Vec::new();
|
||||
@@ -277,6 +294,54 @@ mod tests {
|
||||
assert!(JamInvite::from_uri("frid://i/abcd").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jam_snapshots_hide_podcast_episodes() {
|
||||
use crate::device_sync::{PlaybackRepeat, PlaybackStateWire, PlaybackTrack};
|
||||
use crate::podcast::{EpisodeId, FeedUrl, PodcastEpisodeRef, PodcastId};
|
||||
|
||||
let feed = FeedUrl::parse("https://private.test/feed").unwrap();
|
||||
let podcast_id = PodcastId::from_feed_url(&feed);
|
||||
let episode = PodcastEpisodeRef {
|
||||
episode_id: EpisodeId::from_guid(&podcast_id, "e").unwrap(),
|
||||
podcast_id,
|
||||
feed_url: feed,
|
||||
podcast_title: "Hidden show".to_string(),
|
||||
episode_title: "Hidden episode".to_string(),
|
||||
audio_url: "https://private.test/e.mp3".to_string(),
|
||||
mime: None,
|
||||
duration_ms: None,
|
||||
published_at_ms: None,
|
||||
season: None,
|
||||
episode_number: None,
|
||||
image_url: None,
|
||||
progress_generation: Some(7),
|
||||
};
|
||||
let snapshot = PlaybackSnapshot {
|
||||
device_id: "host".to_string(),
|
||||
device_name: "Host".to_string(),
|
||||
active: true,
|
||||
updated_at_ms: 1,
|
||||
state: PlaybackStateWire {
|
||||
queue: vec![PlaybackTrack::podcast_episode(episode)],
|
||||
queue_pos: 0,
|
||||
playing: true,
|
||||
paused: false,
|
||||
idle_since_ms: None,
|
||||
position_secs: 1.0,
|
||||
volume: 50,
|
||||
shuffle: false,
|
||||
repeat: PlaybackRepeat::Off,
|
||||
playback_rate: None,
|
||||
},
|
||||
coordination: None,
|
||||
};
|
||||
let redacted = redact_jam_snapshot(snapshot);
|
||||
let json = serde_json::to_string(&redacted).unwrap();
|
||||
assert!(!json.contains("Hidden"));
|
||||
assert!(!json.contains("private.test"));
|
||||
assert!(redacted.state.queue[0].is_private_placeholder());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weak_capability_is_rejected() {
|
||||
let mut invite = invite();
|
||||
|
||||
@@ -23,6 +23,13 @@
|
||||
//! * [`similarity_lsh`] turns those embeddings into deterministic compact
|
||||
//! routing summaries, while [`similarity_dht`] stores and discovers signed
|
||||
//! peer summaries without owning model inference or an embedding database.
|
||||
//! * [`podcast`] is the pure, shared podcast contract (canonical feed URLs,
|
||||
//! podcast/episode ids, metadata DTOs and convergent merge rules), with an
|
||||
//! optional RSS/Atom parser behind the `podcast-feed` feature;
|
||||
//! [`device_sync`] protocol v4 carries it between trusted devices.
|
||||
//! * [`podcast_dht`] is a separate signed overlay for public podcast
|
||||
//! discovery: cards a user explicitly shared, found by title/author tokens
|
||||
//! or podcast id, retracted by signed tombstones.
|
||||
//! * Records are replicated to the `K` nodes whose ids are XOR-closest to
|
||||
//! each key. Publishers pick the targets from their routing table and send
|
||||
//! batched store requests (one pipeline per peer), so even a large library
|
||||
@@ -92,6 +99,8 @@ pub mod jam;
|
||||
mod message;
|
||||
mod node;
|
||||
mod normalization;
|
||||
pub mod podcast;
|
||||
pub mod podcast_dht;
|
||||
mod record;
|
||||
mod request;
|
||||
mod routing;
|
||||
@@ -137,3 +146,5 @@ pub use federation_net::{
|
||||
NetworkHealthSnapshot, NetworkHealthState, NetworkId, PeerTicket, RecvStream, RendezvousConfig,
|
||||
SecretKey, SendStream, StreamAcceptor,
|
||||
};
|
||||
|
||||
pub mod playback;
|
||||
@@ -0,0 +1,624 @@
|
||||
//! Transport-independent personal playback coordination.
|
||||
//!
|
||||
//! This is an eventually consistent register, not a distributed lock. During a
|
||||
//! partition multiple outputs may play. Once announcements are delivered, all
|
||||
//! participants adopt the greatest claim and only its owner may keep playing.
|
||||
//! Applications authenticate membership, persist [`DurableState`] before acting,
|
||||
//! feed monotonic time and actual output heartbeats, and execute ownership changes.
|
||||
//! Relaying an announcement must never manufacture an output heartbeat.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Version of the additive playback coordination envelope.
|
||||
pub const VERSION: u16 = 1;
|
||||
/// Maximum command senders retained within one ownership term.
|
||||
pub const MAX_COMMAND_SENDERS: usize = 128;
|
||||
|
||||
/// Local choices; these do not weaken conflict resolution or command fencing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub struct Config {
|
||||
/// Claim idle playback once after discovery at application startup.
|
||||
pub claim_on_startup: bool,
|
||||
/// Become the output when its owner stops producing heartbeats.
|
||||
pub automatic_failover: bool,
|
||||
/// A newly started output may take over a paused owner.
|
||||
pub take_paused_on_startup: bool,
|
||||
/// Wait for initial peer discovery before considering a startup claim.
|
||||
pub discovery_ms: u64,
|
||||
/// Missing output heartbeats, not server connectivity, trigger failover.
|
||||
pub owner_timeout_ms: u64,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
claim_on_startup: true,
|
||||
automatic_failover: true,
|
||||
take_paused_on_startup: true,
|
||||
discovery_ms: 3_000,
|
||||
owner_timeout_ms: 120_000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Shared candidate policy, also used when a gateway selects a browser.
|
||||
/// `owner` is `(actually_playing, milliseconds_since_fresh_output_report)`.
|
||||
/// Startup intent is a one-shot event, never inferred from reconnection.
|
||||
pub fn should_claim(&self, startup: bool, owner: Option<(bool, u64)>) -> bool {
|
||||
let vacant = owner.is_none();
|
||||
let expired = owner.is_some_and(|(_, age)| age >= self.owner_timeout_ms.max(1));
|
||||
let idle = owner.is_some_and(|(playing, _)| !playing);
|
||||
(startup && (vacant || expired || (self.take_paused_on_startup && idle)))
|
||||
|| (self.automatic_failover && (vacant || expired))
|
||||
}
|
||||
|
||||
/// An always-running gateway only transfers ownership on user intent.
|
||||
pub fn passive() -> Self {
|
||||
Self {
|
||||
claim_on_startup: false,
|
||||
automatic_failover: false,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A Lamport term with a deterministic tie-breaker. No wall-clock ordering.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub struct Claim {
|
||||
/// Greater than every term the issuer has observed.
|
||||
pub counter: u64,
|
||||
/// Authenticated device issuing the claim.
|
||||
pub issuer: String,
|
||||
/// Device allowed to render personal audio in this term.
|
||||
pub owner: String,
|
||||
}
|
||||
|
||||
impl Claim {
|
||||
fn valid(&self) -> bool {
|
||||
self.counter > 0
|
||||
&& [&self.issuer, &self.owner]
|
||||
.into_iter()
|
||||
.all(|id| !id.is_empty() && id.len() <= 256)
|
||||
}
|
||||
}
|
||||
|
||||
/// Persist atomically before publishing a new claim or applying a command.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct DurableState {
|
||||
/// Winning claim, retained even after the owner becomes unavailable.
|
||||
pub claim: Option<Claim>,
|
||||
/// Last accepted command sequence per sender in the winning term.
|
||||
pub commands: BTreeMap<String, u64>,
|
||||
/// Locally authored command sequence; never reset on ownership changes.
|
||||
pub command_sequence: u64,
|
||||
/// Locally authored output heartbeat; retained across process restarts.
|
||||
pub heartbeat_sequence: u64,
|
||||
}
|
||||
|
||||
/// Gossip carried beside a playback snapshot, including by controllers.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Announcement {
|
||||
/// Envelope version; unknown versions are ignored.
|
||||
pub version: u16,
|
||||
/// Greatest observed ownership claim.
|
||||
pub claim: Option<Claim>,
|
||||
/// Advances only when the sender's actual output reports activity.
|
||||
pub heartbeat: u64,
|
||||
/// Sender can currently render audio; a server alone cannot.
|
||||
pub available: bool,
|
||||
/// Sender is actually playing unpaused audio.
|
||||
pub playing: bool,
|
||||
}
|
||||
|
||||
/// Commands are scoped to an ownership term and ordered per sender.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct CommandStamp {
|
||||
/// Coordination envelope version.
|
||||
pub version: u16,
|
||||
/// Ownership term in which the command is valid.
|
||||
pub claim: Claim,
|
||||
/// Increasing sequence of the authenticated operation origin.
|
||||
pub sequence: u64,
|
||||
}
|
||||
|
||||
/// Application checkpoint scoped to a trusted-device group.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Checkpoint {
|
||||
/// Application-defined membership scope. Do not reuse state across groups.
|
||||
pub scope: String,
|
||||
/// Durable coordinator state in that scope.
|
||||
pub state: DurableState,
|
||||
}
|
||||
|
||||
/// Shared snapshot envelope; applications supply their portable queue DTO.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Snapshot<S> {
|
||||
/// Authenticated sender, checked against the transport by the adapter.
|
||||
pub device_id: String,
|
||||
/// Display name only; never an identity or ordering key.
|
||||
pub device_name: String,
|
||||
/// Compatibility/UI projection. Ownership comes only from coordination.
|
||||
pub active: bool,
|
||||
/// Position extrapolation timestamp, never used for claim ordering.
|
||||
pub updated_at_ms: i64,
|
||||
/// Output state or the controller's cached presentation state.
|
||||
pub state: S,
|
||||
/// Absent on older clients and on the independently scoped Jam protocol.
|
||||
#[serde(default)]
|
||||
pub coordination: Option<Announcement>,
|
||||
}
|
||||
|
||||
/// Shared personal/Jam command vocabulary. Personal commands additionally
|
||||
/// require [`CommandStamp`]; Jam keeps its own capability and authority scope.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum Command<S> {
|
||||
/// Update the current owner's output state.
|
||||
SetState {
|
||||
/// Queue and transport state.
|
||||
state: S,
|
||||
/// Also seek the actual audio output.
|
||||
#[serde(default)]
|
||||
seek: bool,
|
||||
},
|
||||
/// Explicit transfer with state for the newly selected output.
|
||||
ActiveChanged {
|
||||
/// Selected output identity.
|
||||
active_device_id: String,
|
||||
/// Selected output display name.
|
||||
active_device_name: String,
|
||||
/// State to render after transfer.
|
||||
state: S,
|
||||
},
|
||||
}
|
||||
|
||||
/// Pure engine. All times are monotonic milliseconds supplied by the adapter.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Engine {
|
||||
id: String,
|
||||
config: Config,
|
||||
durable: DurableState,
|
||||
started_at: u64,
|
||||
startup_pending: bool,
|
||||
owner_seen_at: u64,
|
||||
owner_heartbeat: u64,
|
||||
owner_playing: bool,
|
||||
available: bool,
|
||||
playing: bool,
|
||||
output_report: u64,
|
||||
heartbeat_sequence: u64,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
/// Restore durable state, but never restore persisted liveness timestamps.
|
||||
pub fn new(id: String, config: Config, durable: DurableState, now: u64) -> Self {
|
||||
let heartbeat_sequence = durable.heartbeat_sequence;
|
||||
let owner_playing = durable.claim.is_some();
|
||||
Self {
|
||||
id,
|
||||
startup_pending: config.claim_on_startup,
|
||||
config,
|
||||
durable,
|
||||
started_at: now,
|
||||
owner_seen_at: now,
|
||||
owner_heartbeat: 0,
|
||||
owner_playing,
|
||||
available: false,
|
||||
playing: false,
|
||||
output_report: 0,
|
||||
heartbeat_sequence,
|
||||
}
|
||||
}
|
||||
|
||||
/// Durable checkpoint. Save it before executing decisions.
|
||||
pub fn durable(&self) -> &DurableState {
|
||||
&self.durable
|
||||
}
|
||||
|
||||
/// Winning output, whether currently reachable or not.
|
||||
pub fn owner(&self) -> Option<&str> {
|
||||
self.durable
|
||||
.claim
|
||||
.as_ref()
|
||||
.map(|claim| claim.owner.as_str())
|
||||
}
|
||||
|
||||
/// Whether this device currently owns personal playback.
|
||||
pub fn is_owner(&self) -> bool {
|
||||
self.owner() == Some(self.id.as_str())
|
||||
}
|
||||
|
||||
/// Update local capabilities without interpreting connectivity as playback.
|
||||
pub fn set_output(&mut self, available: bool, playing: bool) {
|
||||
if self.available && !available {
|
||||
self.advance_heartbeat();
|
||||
}
|
||||
self.available = available;
|
||||
self.playing = available && playing;
|
||||
}
|
||||
|
||||
/// Feed one fresh report from the actual local output, even if paused.
|
||||
pub fn heartbeat(&mut self, now: u64) {
|
||||
if self.available {
|
||||
self.advance_heartbeat();
|
||||
if self.is_owner() {
|
||||
self.owner_seen_at = now;
|
||||
self.owner_playing = self.playing;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_heartbeat(&mut self) {
|
||||
self.heartbeat_sequence = self.heartbeat_sequence.saturating_add(1);
|
||||
// Reserve a range before publishing. Restart skips unused values.
|
||||
if self.heartbeat_sequence > self.durable.heartbeat_sequence {
|
||||
self.durable.heartbeat_sequence = self.heartbeat_sequence.saturating_add(1024);
|
||||
}
|
||||
}
|
||||
|
||||
/// Merge gossip received from an authenticated trusted device.
|
||||
/// Returns false for unsupported or malformed envelopes.
|
||||
pub fn observe(&mut self, sender: &str, message: &Announcement, now: u64) -> bool {
|
||||
if message.version != VERSION || message.claim.as_ref().is_some_and(|c| !c.valid()) {
|
||||
return false;
|
||||
}
|
||||
if let Some(claim) = &message.claim {
|
||||
self.merge(claim, now);
|
||||
}
|
||||
if self.owner() == Some(sender)
|
||||
&& self.durable.claim == message.claim
|
||||
&& message.heartbeat > self.owner_heartbeat
|
||||
{
|
||||
self.owner_heartbeat = message.heartbeat;
|
||||
self.owner_playing = message.available && message.playing;
|
||||
if message.available {
|
||||
self.owner_seen_at = now;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn merge(&mut self, claim: &Claim, now: u64) {
|
||||
if self
|
||||
.durable
|
||||
.claim
|
||||
.as_ref()
|
||||
.is_none_or(|current| claim > current)
|
||||
{
|
||||
self.durable.claim = Some(claim.clone());
|
||||
self.durable.commands.clear();
|
||||
self.owner_seen_at = now;
|
||||
self.owner_heartbeat = 0;
|
||||
// A transferred owner gets a full grace period before failover.
|
||||
self.owner_playing = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply explicit user intent. Target eligibility is checked by the adapter.
|
||||
/// Returns false rather than wrapping an exhausted Lamport counter.
|
||||
pub fn transfer(&mut self, target: &str, now: u64) -> bool {
|
||||
if target.is_empty() || target.len() > 256 {
|
||||
return false;
|
||||
}
|
||||
let Some(counter) = self
|
||||
.durable
|
||||
.claim
|
||||
.as_ref()
|
||||
.map_or(Some(1), |c| c.counter.checked_add(1))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let claim = Claim {
|
||||
counter,
|
||||
issuer: self.id.clone(),
|
||||
owner: target.into(),
|
||||
};
|
||||
self.merge(&claim, now);
|
||||
self.startup_pending = false;
|
||||
true
|
||||
}
|
||||
|
||||
/// A real browser/application startup, never a server timer or reconnect.
|
||||
pub fn request_startup(&mut self) {
|
||||
self.startup_pending = true;
|
||||
}
|
||||
|
||||
/// Feed a monotonically increasing output-report identifier. Polling the
|
||||
/// same cached report cannot renew owner liveness.
|
||||
pub fn output_report(&mut self, report: u64, now: u64) {
|
||||
if report > self.output_report {
|
||||
self.output_report = report;
|
||||
self.heartbeat(now);
|
||||
}
|
||||
}
|
||||
|
||||
/// Fence an effect again immediately before executing it in an event loop.
|
||||
pub fn command_is_current(&self, sender: &str, stamp: &CommandStamp) -> bool {
|
||||
self.durable.claim.as_ref() == Some(&stamp.claim)
|
||||
&& self.durable.commands.get(sender) == Some(&stamp.sequence)
|
||||
}
|
||||
|
||||
/// Whether an owner snapshot is at least as recent as accepted liveness.
|
||||
pub fn announcement_is_current(&self, sender: &str, message: &Announcement) -> bool {
|
||||
self.owner() == Some(sender)
|
||||
&& self.durable.claim == message.claim
|
||||
&& message.heartbeat >= self.owner_heartbeat
|
||||
}
|
||||
|
||||
/// Evaluate startup/failover. A losing node cannot immediately reclaim:
|
||||
/// the winning term always receives an entire heartbeat grace period.
|
||||
pub fn tick(&mut self, now: u64) -> bool {
|
||||
if now.saturating_sub(self.started_at) < self.config.discovery_ms {
|
||||
return false;
|
||||
}
|
||||
// A restored or relayed owner has unknown activity until it reports.
|
||||
// Keep startup intent pending through discovery, rather than treating
|
||||
// an unknown playing state as an idle output.
|
||||
if self.startup_pending
|
||||
&& self.owner().is_some()
|
||||
&& !self.is_owner()
|
||||
&& self.owner_heartbeat == 0
|
||||
&& now.saturating_sub(self.owner_seen_at) < self.config.owner_timeout_ms.max(1)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let startup = self.startup_pending;
|
||||
self.startup_pending = false;
|
||||
let eligible = self.available && !self.is_owner();
|
||||
let owner = self
|
||||
.owner()
|
||||
.map(|_| (self.owner_playing, now.saturating_sub(self.owner_seen_at)));
|
||||
if eligible && self.config.should_claim(startup, owner) {
|
||||
return self.transfer(&self.id.clone(), now);
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Current gossip. Controllers repeat the claim, never the owner's heartbeat.
|
||||
pub fn announcement(&self) -> Announcement {
|
||||
Announcement {
|
||||
version: VERSION,
|
||||
claim: self.durable.claim.clone(),
|
||||
heartbeat: self.heartbeat_sequence,
|
||||
available: self.available,
|
||||
playing: self.playing,
|
||||
}
|
||||
}
|
||||
|
||||
/// Stamp a command with the current ownership term.
|
||||
pub fn stamp(&mut self) -> Option<CommandStamp> {
|
||||
let claim = self.durable.claim.clone()?;
|
||||
self.durable.command_sequence = self.durable.command_sequence.checked_add(1)?;
|
||||
Some(CommandStamp {
|
||||
version: VERSION,
|
||||
claim,
|
||||
sequence: self.durable.command_sequence,
|
||||
})
|
||||
}
|
||||
|
||||
/// Merge the referenced term and fence a command. Commands may carry a
|
||||
/// previously issued claim ahead of its gossip (transport can reorder).
|
||||
/// Creating claims remains exclusive to transfer/startup/failover.
|
||||
/// Sender identity comes from the authenticated operation log.
|
||||
pub fn accept_command(
|
||||
&mut self,
|
||||
sender: &str,
|
||||
stamp: &CommandStamp,
|
||||
handoff: bool,
|
||||
now: u64,
|
||||
) -> bool {
|
||||
if stamp.version != VERSION
|
||||
|| !stamp.claim.valid()
|
||||
|| sender.is_empty()
|
||||
|| sender.len() > 256
|
||||
|| stamp.sequence == 0
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if handoff && stamp.claim.issuer != sender {
|
||||
return false;
|
||||
}
|
||||
self.merge(&stamp.claim, now);
|
||||
if self.durable.claim.as_ref() != Some(&stamp.claim) {
|
||||
return false;
|
||||
}
|
||||
if !self.durable.commands.contains_key(sender)
|
||||
&& self.durable.commands.len() >= MAX_COMMAND_SENDERS
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let previous = self.durable.commands.entry(sender.to_string()).or_default();
|
||||
if stamp.sequence <= *previous {
|
||||
return false;
|
||||
}
|
||||
*previous = stamp.sequence;
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
fn node(id: &str, config: Config) -> Engine {
|
||||
let mut e = Engine::new(id.into(), config, DurableState::default(), 0);
|
||||
e.set_output(true, false);
|
||||
e
|
||||
}
|
||||
#[test]
|
||||
fn partition_converges_without_reclaim_loop() {
|
||||
let mut a = node("a", Config::default());
|
||||
let mut b = node("b", Config::default());
|
||||
assert!(a.tick(3_000));
|
||||
assert!(b.tick(3_000));
|
||||
a.set_output(true, true);
|
||||
b.set_output(true, true);
|
||||
a.heartbeat(3_001);
|
||||
b.heartbeat(3_001);
|
||||
let aa = a.announcement();
|
||||
let bb = b.announcement();
|
||||
a.observe("b", &bb, 3_002);
|
||||
b.observe("a", &aa, 3_002);
|
||||
assert_eq!(a.owner(), b.owner());
|
||||
assert!(!a.is_owner());
|
||||
assert!(b.is_owner());
|
||||
assert!(!a.tick(3_003));
|
||||
a.observe("a", &aa, 3_004);
|
||||
assert_eq!(a.owner(), Some("b"));
|
||||
}
|
||||
#[test]
|
||||
fn startup_respects_playing_but_can_take_paused() {
|
||||
for playing in [true, false] {
|
||||
let mut owner = node("a", Config::default());
|
||||
owner.tick(3_000);
|
||||
owner.set_output(true, playing);
|
||||
owner.heartbeat(3_001);
|
||||
let mut newcomer = node("b", Config::default());
|
||||
newcomer.observe("a", &owner.announcement(), 2_000);
|
||||
assert_eq!(newcomer.tick(3_000), !playing);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn passive_gateway_never_claims_without_intent() {
|
||||
let mut web = node("web", Config::passive());
|
||||
assert!(!web.tick(3_000));
|
||||
assert!(!web.tick(1_000_000));
|
||||
assert!(web.transfer("web", 1_000_001));
|
||||
assert!(web.is_owner());
|
||||
}
|
||||
#[test]
|
||||
fn duplicates_and_relays_do_not_extend_liveness() {
|
||||
let mut a = node("a", Config::default());
|
||||
a.tick(3_000);
|
||||
a.set_output(true, true);
|
||||
a.heartbeat(3_001);
|
||||
let mut b = node("b", Config::default());
|
||||
b.observe("a", &a.announcement(), 3_002);
|
||||
b.tick(3_003);
|
||||
b.observe("a", &a.announcement(), 100_000);
|
||||
b.observe("relay", &a.announcement(), 122_000);
|
||||
assert!(b.tick(123_003));
|
||||
}
|
||||
#[test]
|
||||
fn commands_are_fenced_ordered_and_survive_restore() {
|
||||
let mut a = node("a", Config::default());
|
||||
a.tick(3_000);
|
||||
let old = a.stamp().unwrap();
|
||||
assert!(a.accept_command("a", &old, false, 3_001));
|
||||
let mut restored = Engine::new("a".into(), Config::default(), a.durable().clone(), 4_000);
|
||||
assert!(!restored.accept_command("a", &old, false, 4_001));
|
||||
restored.transfer("b", 4_002);
|
||||
assert!(!restored.accept_command("a", &old, false, 4_003));
|
||||
let forged = restored.stamp().unwrap();
|
||||
assert!(!restored.accept_command("spoof", &forged, true, 4_004));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn three_outputs_converge_in_every_delivery_order() {
|
||||
for order in [
|
||||
[0, 1, 2],
|
||||
[0, 2, 1],
|
||||
[1, 0, 2],
|
||||
[1, 2, 0],
|
||||
[2, 0, 1],
|
||||
[2, 1, 0],
|
||||
] {
|
||||
let mut nodes = [
|
||||
node("a", Config::default()),
|
||||
node("b", Config::default()),
|
||||
node("c", Config::default()),
|
||||
];
|
||||
for node in &mut nodes {
|
||||
node.tick(3_000);
|
||||
node.set_output(true, true);
|
||||
node.heartbeat(3_001);
|
||||
}
|
||||
let messages = nodes.each_ref().map(Engine::announcement);
|
||||
for node in &mut nodes {
|
||||
for index in order {
|
||||
node.observe(["a", "b", "c"][index], &messages[index], 4_000);
|
||||
}
|
||||
assert_eq!(node.owner(), Some("c"));
|
||||
assert!(!node.tick(4_001));
|
||||
}
|
||||
assert_eq!(nodes.iter().filter(|node| node.is_owner()).count(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_browser_reports_do_not_keep_a_suspended_output_alive() {
|
||||
let mut web = node("web", Config::passive());
|
||||
web.transfer("web", 0);
|
||||
web.set_output(true, true);
|
||||
web.output_report(1, 1);
|
||||
let mut tui = node("tui", Config::default());
|
||||
tui.observe("web", &web.announcement(), 2);
|
||||
tui.tick(3_000);
|
||||
for now in [30_000, 60_000, 90_000, 120_003] {
|
||||
web.output_report(1, now);
|
||||
tui.observe("web", &web.announcement(), now);
|
||||
}
|
||||
assert!(tui.tick(120_003));
|
||||
web.observe("tui", &tui.announcement(), 120_004);
|
||||
assert!(!web.is_owner());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heartbeat_reservation_survives_restart() {
|
||||
let mut a = node("a", Config::default());
|
||||
a.tick(3_000);
|
||||
a.heartbeat(3_001);
|
||||
let before = a.announcement();
|
||||
let mut restored = Engine::new("a".into(), Config::default(), a.durable().clone(), 0);
|
||||
restored.set_output(true, false);
|
||||
restored.heartbeat(1);
|
||||
assert!(restored.announcement().heartbeat > before.heartbeat);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_can_arrive_before_handoff_and_older_state_cannot_overwrite_it() {
|
||||
let mut a = node("a", Config::default());
|
||||
a.transfer("b", 1);
|
||||
let handoff = a.stamp().unwrap();
|
||||
let play = a.stamp().unwrap();
|
||||
let mut b = node("b", Config::default());
|
||||
assert!(b.accept_command("a", &play, false, 2));
|
||||
assert!(b.is_owner());
|
||||
assert!(!b.accept_command("a", &handoff, true, 3));
|
||||
assert!(b.command_is_current("a", &play));
|
||||
let mut incompatible = a.stamp().unwrap();
|
||||
incompatible.version += 1;
|
||||
assert!(!b.accept_command("a", &incompatible, false, 4));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_automatic_policy_and_unavailable_outputs_never_claim() {
|
||||
let mut disabled = node(
|
||||
"a",
|
||||
Config {
|
||||
claim_on_startup: false,
|
||||
automatic_failover: false,
|
||||
..Config::default()
|
||||
},
|
||||
);
|
||||
assert!(!disabled.tick(1_000_000));
|
||||
let mut unavailable = node("b", Config::default());
|
||||
unavailable.set_output(false, false);
|
||||
assert!(!unavailable.tick(1_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_sender_memory_is_bounded() {
|
||||
let mut a = node("a", Config::default());
|
||||
a.tick(3_000);
|
||||
let stamp = a.stamp().unwrap();
|
||||
for index in 0..MAX_COMMAND_SENDERS {
|
||||
assert!(a.accept_command(&format!("sender{index}"), &stamp, false, 3_001));
|
||||
}
|
||||
assert!(!a.accept_command("overflow", &stamp, false, 3_002));
|
||||
assert_eq!(a.durable().commands.len(), MAX_COMMAND_SENDERS);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,731 @@
|
||||
use super::*;
|
||||
|
||||
const ITUNES_NS: &str = "http://www.itunes.com/dtds/podcast-1.0.dtd";
|
||||
|
||||
fn rss(channel_extra: &str, items: &str) -> String {
|
||||
format!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:itunes="{ITUNES_NS}" xmlns:content="http://purl.org/rss/1.0/modules/content/">
|
||||
<channel>
|
||||
<title>Test Cast</title>
|
||||
<link>https://example.test/cast</link>
|
||||
{channel_extra}
|
||||
{items}
|
||||
</channel>
|
||||
</rss>"#
|
||||
)
|
||||
}
|
||||
|
||||
fn item(guid: &str, title: &str, date: &str) -> String {
|
||||
format!(
|
||||
r#"<item>
|
||||
<title>{title}</title>
|
||||
<guid isPermaLink="false">{guid}</guid>
|
||||
<pubDate>{date}</pubDate>
|
||||
<enclosure url="https://cdn.example.test/{guid}.mp3" length="1234" type="audio/mpeg"/>
|
||||
</item>"#
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rss_with_itunes_tags_cdata_and_fallbacks() {
|
||||
let xml = rss(
|
||||
r#"<description><![CDATA[<p>Weekly <b>talk</b> show.</p><script>evil()</script>]]></description>
|
||||
<itunes:author>Jane Host</itunes:author>
|
||||
<itunes:image href="https://example.test/cover.jpg"/>
|
||||
<image><url>https://example.test/small.png</url><title>ignored</title></image>"#,
|
||||
r#"<item>
|
||||
<title> Episode one </title>
|
||||
<itunes:title>One</itunes:title>
|
||||
<description><p>First &amp; best</p></description>
|
||||
<guid>ep-1</guid>
|
||||
<pubDate>Tue, 02 Jan 2024 10:00:00 GMT</pubDate>
|
||||
<itunes:duration>1:02:03</itunes:duration>
|
||||
<itunes:image href="https://example.test/ep1.jpg"/>
|
||||
<link>https://example.test/ep1</link>
|
||||
<enclosure url="https://cdn.example.test/1.mp3" length="5000" type="audio/mpeg"/>
|
||||
</item>
|
||||
<item>
|
||||
<title></title>
|
||||
<itunes:title>Only itunes title</itunes:title>
|
||||
<content:encoded><![CDATA[<div>Long <i>show</i> notes</div>]]></content:encoded>
|
||||
<guid>ep-2</guid>
|
||||
<itunes:duration>754</itunes:duration>
|
||||
<enclosure url="https://cdn.example.test/2.m4a" type="audio/x-m4a"/>
|
||||
</item>
|
||||
<item>
|
||||
<itunes:summary>Summary only</itunes:summary>
|
||||
<guid>ep-3</guid>
|
||||
<enclosure url="https://cdn.example.test/3.mp3"/>
|
||||
</item>"#,
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(feed.title, "Test Cast");
|
||||
assert_eq!(feed.author.as_deref(), Some("Jane Host"));
|
||||
assert_eq!(feed.description.as_deref(), Some("Weekly talk show."));
|
||||
assert_eq!(feed.link.as_deref(), Some("https://example.test/cast"));
|
||||
assert_eq!(
|
||||
feed.image_url.as_deref(),
|
||||
Some("https://example.test/cover.jpg")
|
||||
);
|
||||
assert_eq!(feed.episodes.len(), 3);
|
||||
|
||||
let first = &feed.episodes[0];
|
||||
assert_eq!(first.guid.as_deref(), Some("ep-1"));
|
||||
assert_eq!(first.title, "Episode one");
|
||||
assert_eq!(first.description.as_deref(), Some("First & best"));
|
||||
assert_eq!(first.published_at_ms, Some(1_704_189_600_000));
|
||||
assert_eq!(first.duration_ms, Some(3_723_000));
|
||||
assert_eq!(first.audio_url, "https://cdn.example.test/1.mp3");
|
||||
assert_eq!(first.mime_type.as_deref(), Some("audio/mpeg"));
|
||||
assert_eq!(first.byte_len, Some(5000));
|
||||
assert_eq!(first.link.as_deref(), Some("https://example.test/ep1"));
|
||||
assert_eq!(
|
||||
first.image_url.as_deref(),
|
||||
Some("https://example.test/ep1.jpg")
|
||||
);
|
||||
|
||||
let second = &feed.episodes[1];
|
||||
assert_eq!(second.title, "Only itunes title");
|
||||
assert_eq!(second.description.as_deref(), Some("Long show notes"));
|
||||
assert_eq!(second.duration_ms, Some(754_000));
|
||||
assert_eq!(second.byte_len, None);
|
||||
assert_eq!(second.published_at_ms, None);
|
||||
|
||||
let third = &feed.episodes[2];
|
||||
assert_eq!(third.title, "Untitled episode");
|
||||
assert_eq!(third.description.as_deref(), Some("Summary only"));
|
||||
assert_eq!(third.mime_type, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entities_numeric_refs_and_unknown_html_entities() {
|
||||
let xml = rss(
|
||||
"",
|
||||
r#"<item>
|
||||
<title>A & B — AП — &bogus; R&D ©</title>
|
||||
<description>x < y && z</description>
|
||||
<guid>e</guid>
|
||||
<enclosure url="https://cdn.example.test/a.mp3?x=1&y=2" type="audio/mpeg"/>
|
||||
</item>"#,
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
let episode = &feed.episodes[0];
|
||||
assert_eq!(episode.title, "A & B — AП — &bogus; R&D ©");
|
||||
assert_eq!(episode.description.as_deref(), Some("x < y && z"));
|
||||
assert_eq!(episode.audio_url, "https://cdn.example.test/a.mp3?x=1&y=2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atom_feed() {
|
||||
let xml = r#"<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title type="text">Atom Cast</title>
|
||||
<subtitle>About <b>things</b></subtitle>
|
||||
<link rel="self" href="https://example.test/atom.xml"/>
|
||||
<link rel="alternate" href="https://example.test/"/>
|
||||
<logo>https://example.test/logo.png</logo>
|
||||
<author><name>Atom Author</name></author>
|
||||
<entry>
|
||||
<title>Entry one</title>
|
||||
<id>urn:uuid:1</id>
|
||||
<published>2024-03-01T12:30:00+02:00</published>
|
||||
<updated>2024-03-05T00:00:00Z</updated>
|
||||
<summary type="html"><p>Entry summary</p></summary>
|
||||
<link rel="alternate" href="https://example.test/e1"/>
|
||||
<link rel="enclosure" href="https://cdn.example.test/e1.ogg" type="audio/ogg" length="42"/>
|
||||
</entry>
|
||||
<entry>
|
||||
<title>Entry two</title>
|
||||
<id>urn:uuid:2</id>
|
||||
<updated>2024-03-06T00:00:00Z</updated>
|
||||
<content type="xhtml"><div xmlns="http://www.w3.org/1999/xhtml"><p>Inline</p><p>xhtml</p></div></content>
|
||||
<link rel="enclosure" href="https://cdn.example.test/e2.mp3"/>
|
||||
</entry>
|
||||
<entry>
|
||||
<title>No audio</title>
|
||||
<id>urn:uuid:3</id>
|
||||
<link rel="alternate" href="https://example.test/e3"/>
|
||||
</entry>
|
||||
</feed>"#;
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(feed.title, "Atom Cast");
|
||||
assert_eq!(feed.description.as_deref(), Some("About things"));
|
||||
assert_eq!(feed.link.as_deref(), Some("https://example.test/"));
|
||||
assert_eq!(
|
||||
feed.image_url.as_deref(),
|
||||
Some("https://example.test/logo.png")
|
||||
);
|
||||
assert_eq!(feed.author.as_deref(), Some("Atom Author"));
|
||||
assert_eq!(feed.episodes.len(), 2);
|
||||
let first = &feed.episodes[0];
|
||||
assert_eq!(first.guid.as_deref(), Some("urn:uuid:1"));
|
||||
assert_eq!(first.published_at_ms, Some(1_709_289_000_000));
|
||||
assert_eq!(first.description.as_deref(), Some("Entry summary"));
|
||||
assert_eq!(first.link.as_deref(), Some("https://example.test/e1"));
|
||||
assert_eq!(first.audio_url, "https://cdn.example.test/e1.ogg");
|
||||
assert_eq!(first.byte_len, Some(42));
|
||||
let second = &feed.episodes[1];
|
||||
assert_eq!(second.published_at_ms, Some(1_709_683_200_000));
|
||||
assert_eq!(second.description.as_deref(), Some("Inline xhtml"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn items_without_audio_are_skipped_and_media_content_is_a_fallback() {
|
||||
let xml = r#"<rss xmlns:media="http://search.yahoo.com/mrss/"><channel><title>T</title>
|
||||
<item><title>No enclosure</title><guid>a</guid></item>
|
||||
<item><title>Image enclosure</title><guid>b</guid><enclosure url="https://x.test/b.jpg" type="image/jpeg"/></item>
|
||||
<item><title>Empty url</title><guid>c</guid><enclosure url="" type="audio/mpeg"/></item>
|
||||
<item><title>Media</title><guid>d</guid>
|
||||
<media:content url="https://x.test/d.png" medium="image"/>
|
||||
<media:group><media:content url="https://x.test/d.mp3" type="audio/mpeg" fileSize="77"/></media:group>
|
||||
<media:thumbnail url="https://x.test/d-thumb.jpg"/>
|
||||
</item>
|
||||
<item><title>Video</title><guid>e</guid><enclosure url="https://x.test/e.mp4" type="video/mp4"/></item>
|
||||
</channel></rss>"#;
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
let titles: Vec<_> = feed.episodes.iter().map(|e| e.title.as_str()).collect();
|
||||
assert_eq!(titles, ["Media", "Video"]);
|
||||
assert_eq!(feed.episodes[0].audio_url, "https://x.test/d.mp3");
|
||||
assert_eq!(feed.episodes[0].byte_len, Some(77));
|
||||
assert_eq!(
|
||||
feed.episodes[0].image_url.as_deref(),
|
||||
Some("https://x.test/d-thumb.jpg")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_guids_keep_the_first_and_missing_guids_stay_absent() {
|
||||
let items = [
|
||||
item("dup", "First", "Mon, 01 Jan 2024 00:00:00 +0000"),
|
||||
item("dup", "Second", "Tue, 02 Jan 2024 00:00:00 +0000"),
|
||||
r#"<item><title>No guid</title><enclosure url="https://cdn.example.test/noguid.mp3" type="audio/mpeg"/></item>"#.to_string(),
|
||||
r#"<item><title>Blank guid</title><guid> </guid><enclosure url="https://cdn.example.test/blank.mp3" type="audio/mpeg"/></item>"#.to_string(),
|
||||
]
|
||||
.concat();
|
||||
let feed = parse_feed(rss("", &items).as_bytes()).unwrap();
|
||||
let pairs: Vec<_> = feed
|
||||
.episodes
|
||||
.iter()
|
||||
.map(|e| (e.guid.as_deref(), e.title.as_str(), e.audio_url.as_str()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
pairs,
|
||||
[
|
||||
(Some("dup"), "First", "https://cdn.example.test/dup.mp3"),
|
||||
(None, "No guid", "https://cdn.example.test/noguid.mp3"),
|
||||
(None, "Blank guid", "https://cdn.example.test/blank.mp3"),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durations() {
|
||||
let cases = [
|
||||
("01:02:03", Some(3723.0)),
|
||||
("1:02:03", Some(3723.0)),
|
||||
("62:03", Some(3723.0)),
|
||||
("3723", Some(3723.0)),
|
||||
("3723.5", Some(3723.5)),
|
||||
("00:00:01.25", Some(1.25)),
|
||||
(" 45:00 ", Some(2700.0)),
|
||||
("0", None),
|
||||
("00:00:00", None),
|
||||
("", None),
|
||||
("abc", None),
|
||||
("1:2:3:4", None),
|
||||
("-5", None),
|
||||
("1e3", None),
|
||||
("NaN", None),
|
||||
("12:", None),
|
||||
];
|
||||
for (input, expected) in cases {
|
||||
assert_eq!(parse_duration(input), expected, "{input:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dates() {
|
||||
let reference = Some(1_704_189_600); // 2024-01-02 10:00:00 UTC
|
||||
let cases = [
|
||||
("Tue, 02 Jan 2024 10:00:00 GMT", reference),
|
||||
("Tue, 02 Jan 2024 10:00:00 +0000", reference),
|
||||
("02 Jan 2024 10:00:00 UT", reference),
|
||||
("Tue, 2 Jan 2024 13:00:00 +0300", reference),
|
||||
("Tue, 02 Jan 2024 05:00:00 EST", reference),
|
||||
("Tue, 02 Jan 2024 02:00:00 PST", reference),
|
||||
("Tuesday, 02 January 2024 10:00 GMT", reference),
|
||||
("Tue, 02 Jan 24 10:00:00 GMT", reference),
|
||||
("Tue, 02 Jan 2024 13:00:00 +03:00", reference),
|
||||
("Tue, 02 Jan 2024 10:00:00", reference),
|
||||
("Вт, 02 янв 2024 13:00:00 +0300", reference),
|
||||
("2024-01-02T10:00:00Z", reference),
|
||||
("2024-01-02T13:00:00+03:00", reference),
|
||||
("2024-01-02T10:00:00.123Z", reference),
|
||||
("2024-01-02 10:00:00", reference),
|
||||
("2024-01-02", Some(1_704_153_600)),
|
||||
("not a date", None),
|
||||
("Tue, 31 Feb 2024 10:00:00 GMT", None),
|
||||
("Tue, 02 Foo 2024 10:00:00 GMT", None),
|
||||
("2024-13-02T10:00:00Z", None),
|
||||
("", None),
|
||||
];
|
||||
for (input, expected) in cases {
|
||||
assert_eq!(parse_date(input), expected, "{input:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_ascii_zones_do_not_panic() {
|
||||
for input in [
|
||||
"Mon, 07 Nov 2022 10:00:00 +й0",
|
||||
"Mon, 07 Nov 2022 10:00:00 -йй",
|
||||
"Mon, 07 Nov 2022 10:00+й0",
|
||||
"Mon, 07 Nov 2022 10:00:00 +0й00",
|
||||
"2022-11-07T10:00:00+0й0",
|
||||
"2022-11-07T10:00:00+й",
|
||||
"2022-11-07T10:00:00-€:00",
|
||||
"2022-11-07T10:00:00+日本",
|
||||
] {
|
||||
assert_eq!(parse_date(input), None, "{input:?}");
|
||||
}
|
||||
// Exhaustive-ish: every short suffix of mixed ASCII / multi-byte
|
||||
// characters after a sign.
|
||||
let alphabet = ['0', '9', ':', 'й', '€', '日', '+', '-'];
|
||||
for a in alphabet {
|
||||
for b in alphabet {
|
||||
for c in alphabet {
|
||||
for prefix in ["Mon, 07 Nov 2022 10:00:00 ", "2022-11-07T10:00:00"] {
|
||||
for sign in ['+', '-'] {
|
||||
let _ = parse_date(&format!("{prefix}{sign}{a}{b}{c}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// In a feed, the bad date only loses the date.
|
||||
let xml = rss("", &item("z", "Zone", "Mon, 07 Nov 2022 10:00:00 +й0"));
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(feed.episodes.len(), 1);
|
||||
assert_eq!(feed.episodes[0].published_at_ms, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_urls_are_resolved_and_must_be_http() {
|
||||
let xml = rss(
|
||||
r#"<itunes:image href="/img/cover.jpg"/>"#,
|
||||
r#"<item><guid>rel</guid><title>Relative</title>
|
||||
<itunes:image href="ep.png"/>
|
||||
<enclosure url="/audio/1.mp3" type="audio/mpeg"/></item>
|
||||
<item><guid>proto</guid><title>Protocol relative</title>
|
||||
<enclosure url="//cdn.example.test/2.mp3" type="audio/mpeg"/></item>
|
||||
<item><guid>file</guid><title>Local file</title>
|
||||
<enclosure url="file:///etc/passwd" type="audio/mpeg"/></item>
|
||||
<item><guid>ftp</guid><title>FTP</title>
|
||||
<enclosure url="ftp://example.test/3.mp3" type="audio/mpeg"/></item>
|
||||
<item><guid>fallback</guid><title>Media fallback</title>
|
||||
<enclosure url="file:///tmp/x.mp3" type="audio/mpeg"/>
|
||||
<media:content xmlns:media="http://search.yahoo.com/mrss/" url="https://cdn.example.test/4.mp3" type="audio/mpeg"/></item>
|
||||
<item><guid>abs</guid><title>Absolute</title>
|
||||
<enclosure url="HTTPS://cdn.example.test/5.mp3" type="audio/mpeg"/></item>"#,
|
||||
);
|
||||
let feed =
|
||||
parse_feed_with_base(xml.as_bytes(), Some("https://host.test/feeds/rss.xml")).unwrap();
|
||||
let urls: Vec<(&str, &str)> = feed
|
||||
.episodes
|
||||
.iter()
|
||||
.map(|e| (e.guid.as_deref().unwrap_or_default(), e.audio_url.as_str()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
urls,
|
||||
[
|
||||
("rel", "https://host.test/audio/1.mp3"),
|
||||
("proto", "https://cdn.example.test/2.mp3"),
|
||||
("fallback", "https://cdn.example.test/4.mp3"),
|
||||
("abs", "HTTPS://cdn.example.test/5.mp3"),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
feed.image_url.as_deref(),
|
||||
Some("https://host.test/img/cover.jpg")
|
||||
);
|
||||
assert_eq!(
|
||||
feed.episodes[0].image_url.as_deref(),
|
||||
Some("https://host.test/feeds/ep.png")
|
||||
);
|
||||
for episode in &feed.episodes {
|
||||
assert!(is_remote_url(&episode.audio_url), "{}", episode.audio_url);
|
||||
}
|
||||
|
||||
// Without a base, relative media cannot be resolved and is dropped;
|
||||
// protocol-relative URLs default to https.
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
let guids: Vec<&str> = feed
|
||||
.episodes
|
||||
.iter()
|
||||
.map(|e| e.guid.as_deref().unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(guids, ["proto", "fallback", "abs"]);
|
||||
assert_eq!(feed.image_url, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rss_with_an_unrelated_default_namespace() {
|
||||
for ns in ["http://example.test/x", "http://www.w3.org/2005/Atom"] {
|
||||
let xml = format!(
|
||||
r#"<rss version="2.0" xmlns="{ns}" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>T</title>
|
||||
<atom:link href="https://a.test/feed" rel="self"/>
|
||||
<item><guid>1</guid><title>One</title><enclosure url="https://a.test/1.mp3" type="audio/mpeg"/></item>
|
||||
</channel></rss>"#
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(feed.title, "T", "{ns}");
|
||||
assert_eq!(feed.episodes.len(), 1, "{ns}");
|
||||
assert_eq!(feed.episodes[0].title, "One");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_pages_and_garbage_are_rejected() {
|
||||
let html = br#"<!DOCTYPE html><html><head><title>Vivat</title></head><body><p>Hi there</p></body></html>"#;
|
||||
let err = parse_feed(html).unwrap_err().to_string();
|
||||
assert!(err.contains("not an RSS or Atom feed"), "{err}");
|
||||
for garbage in [
|
||||
&b""[..],
|
||||
b"hello world",
|
||||
b"\x00\xff\xfe garbage",
|
||||
b"{\"json\": true}",
|
||||
b"<",
|
||||
b"<<<>>>",
|
||||
] {
|
||||
let err = parse_feed(garbage).unwrap_err().to_string();
|
||||
assert!(
|
||||
err.contains("not an RSS or Atom feed"),
|
||||
"{garbage:?}: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_and_malformed_xml_never_panics() {
|
||||
let items: String = (0..5)
|
||||
.map(|i| item(&format!("g{i}"), &format!("T {i} & "), "bad date"))
|
||||
.collect();
|
||||
let xml = rss(
|
||||
"<description><![CDATA[cdata]]></description><itunes:image href=\"x\"/>",
|
||||
&items,
|
||||
);
|
||||
let bytes = xml.as_bytes();
|
||||
for len in 0..=bytes.len() {
|
||||
let _ = parse_feed(&bytes[..len]);
|
||||
}
|
||||
// Mismatched end tags are tolerated.
|
||||
let sloppy = r#"<rss><channel><title>S</title><item><title>A</b></title><guid>1</guid><enclosure url="https://x.test/1.mp3"/></item></channel></rss>"#;
|
||||
let feed = parse_feed(sloppy.as_bytes()).unwrap();
|
||||
assert_eq!(feed.episodes.len(), 1);
|
||||
assert_eq!(feed.episodes[0].title, "A");
|
||||
// Truncation after complete items keeps them.
|
||||
let third = xml.find("<title>T 3").unwrap();
|
||||
let cut = &bytes[..xml[..third].rfind("<item>").unwrap()];
|
||||
let feed = parse_feed(cut).unwrap();
|
||||
assert_eq!(feed.episodes.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_nesting_does_not_overflow() {
|
||||
let depth = 100_000;
|
||||
let mut xml = String::from("<rss><channel><title>Deep</title>");
|
||||
xml.push_str(&"<x>".repeat(depth));
|
||||
xml.push_str(&"</x>".repeat(depth));
|
||||
xml.push_str(&item("after", "After", ""));
|
||||
xml.push_str("</channel></rss>");
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(feed.episodes.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cyrillic_titles_in_utf8_and_windows_1251() {
|
||||
let xml = rss(
|
||||
"<itunes:author>Иван Петров</itunes:author>",
|
||||
&item(
|
||||
"1",
|
||||
"Выпуск №1: «Привет»",
|
||||
"Mon, 01 Jan 2024 00:00:00 +0300",
|
||||
),
|
||||
)
|
||||
.replace("Test Cast", "Подкаст");
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(feed.title, "Подкаст");
|
||||
assert_eq!(feed.author.as_deref(), Some("Иван Петров"));
|
||||
assert_eq!(feed.episodes[0].title, "Выпуск №1: «Привет»");
|
||||
|
||||
let cp1251 = xml.replace("encoding=\"UTF-8\"", "encoding=\"windows-1251\"");
|
||||
let (encoded, _, unmappable) = encoding_rs::WINDOWS_1251.encode(&cp1251);
|
||||
assert!(!unmappable);
|
||||
let feed = parse_feed(&encoded).unwrap();
|
||||
assert_eq!(feed.title, "Подкаст");
|
||||
assert_eq!(feed.episodes[0].title, "Выпуск №1: «Привет»");
|
||||
|
||||
// Invalid UTF-8 without a declaration is decoded lossily.
|
||||
let mut broken = xml.into_bytes();
|
||||
let at = broken
|
||||
.windows(7)
|
||||
.position(|w| w == b"Test Ca")
|
||||
.unwrap_or(10);
|
||||
broken.insert(at, 0xff);
|
||||
assert!(parse_feed(&broken).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_prefixes_are_matched_tolerantly() {
|
||||
// Undeclared `itunes:` prefix and a custom prefix bound to the
|
||||
// iTunes namespace.
|
||||
let xml = format!(
|
||||
r#"<rss><channel><title>P</title><itunes:author>Undeclared</itunes:author>
|
||||
<item xmlns:it="{ITUNES_NS}"><guid>1</guid><it:title>Custom prefix</it:title><it:duration>90</it:duration>
|
||||
<enclosure url="https://x.test/1.mp3" type="audio/mpeg"/></item></channel></rss>"#
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(feed.author.as_deref(), Some("Undeclared"));
|
||||
assert_eq!(feed.episodes[0].title, "Custom prefix");
|
||||
assert_eq!(feed.episodes[0].duration_ms, Some(90_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_descriptions_are_truncated() {
|
||||
let long = "word ".repeat(2000);
|
||||
let text = html_to_text(&long, 4000);
|
||||
assert!(text.chars().count() <= 4000);
|
||||
assert!(text.ends_with('…'));
|
||||
let xml = rss(&format!("<description>{long}</description>"), "");
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert!(feed.description.unwrap().chars().count() <= 4000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn itunes_season_episode_and_type() {
|
||||
let xml = rss(
|
||||
"",
|
||||
r#"<item><guid>a</guid><title>A</title>
|
||||
<itunes:season>2</itunes:season><itunes:episode> 7 </itunes:episode>
|
||||
<itunes:episodeType>Trailer</itunes:episodeType>
|
||||
<enclosure url="https://cdn.example.test/a.mp3" type="audio/mpeg"/></item>
|
||||
<item><guid>b</guid><title>B</title>
|
||||
<itunes:season>two</itunes:season><itunes:episode>-1</itunes:episode>
|
||||
<itunes:episodeType>mystery</itunes:episodeType>
|
||||
<enclosure url="https://cdn.example.test/b.mp3" type="audio/mpeg"/></item>
|
||||
<item><guid>c</guid><title>C</title><itunes:episodeType>bonus</itunes:episodeType>
|
||||
<itunes:episode>99999999999</itunes:episode>
|
||||
<enclosure url="https://cdn.example.test/c.mp3" type="audio/mpeg"/></item>"#,
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
let a = &feed.episodes[0];
|
||||
assert_eq!((a.season, a.episode_number), (Some(2), Some(7)));
|
||||
assert_eq!(a.episode_type, EpisodeType::Trailer);
|
||||
let b = &feed.episodes[1];
|
||||
assert_eq!((b.season, b.episode_number), (None, None));
|
||||
assert_eq!(b.episode_type, EpisodeType::Full);
|
||||
let c = &feed.episodes[2];
|
||||
assert_eq!((c.season, c.episode_number), (None, None));
|
||||
assert_eq!(c.episode_type, EpisodeType::Bonus);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guid_permalink_flag() {
|
||||
let xml = rss(
|
||||
"",
|
||||
r#"<item><guid isPermaLink="false">x</guid><enclosure url="https://c.test/x.mp3"/></item>
|
||||
<item><guid>https://e.test/y</guid><enclosure url="https://c.test/y.mp3"/></item>
|
||||
<item><guid isPermaLink=" FALSE ">z</guid><enclosure url="https://c.test/z.mp3"/></item>
|
||||
<item><enclosure url="https://c.test/n.mp3"/></item>"#,
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
let flags: Vec<Option<bool>> = feed.episodes.iter().map(|e| e.guid_is_permalink).collect();
|
||||
assert_eq!(flags, [Some(false), Some(true), Some(false), None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_feed_url_is_reported() {
|
||||
let xml = rss(
|
||||
"<itunes:new-feed-url> https://new.example.test/feed?x=1 </itunes:new-feed-url>",
|
||||
"",
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(
|
||||
feed.new_feed_url.as_deref(),
|
||||
Some("https://new.example.test/feed?x=1")
|
||||
);
|
||||
let alias = FeedUrl::parse(feed.new_feed_url.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(alias.as_str(), "https://new.example.test/feed?x=1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_guid_is_treated_as_absent() {
|
||||
let long = "g".repeat(MAX_GUID_BYTES + 1);
|
||||
let xml = rss(
|
||||
"",
|
||||
&format!(r#"<item><guid>{long}</guid><enclosure url="https://c.test/1.mp3"/></item>"#),
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(feed.episodes[0].guid, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_urls_with_spaces_are_percent_encoded() {
|
||||
let xml = rss(
|
||||
"",
|
||||
r#"<item><guid>s</guid><enclosure url="https://c.test/my file.mp3" type="audio/mpeg"/></item>"#,
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert_eq!(feed.episodes[0].audio_url, "https://c.test/my%20file.mp3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsed_feed_maps_to_valid_contract_metadata() {
|
||||
let base = FeedUrl::parse("https://example.test/feed.xml").unwrap();
|
||||
let items = [
|
||||
item("g1", "One", "Tue, 02 Jan 2024 10:00:00 GMT"),
|
||||
r#"<item><title>No guid</title><link>https://example.test/no-guid</link>
|
||||
<pubDate>Wed, 03 Jan 2024 10:00:00 GMT</pubDate>
|
||||
<enclosure url="https://cdn.example.test/ng.mp3" type="audio/mpeg"/></item>"#
|
||||
.to_string(),
|
||||
// Same page as the channel link: not identifying.
|
||||
r#"<item><title>Home link</title><link>https://example.test/cast</link>
|
||||
<enclosure url="https://cdn.example.test/home.mp3" type="audio/mpeg"/></item>"#
|
||||
.to_string(),
|
||||
]
|
||||
.concat();
|
||||
let feed = parse_feed_with_base(rss("", &items).as_bytes(), Some(base.as_str())).unwrap();
|
||||
let podcast_id = PodcastId::from_feed_url(&base);
|
||||
let ids = feed.resolve_episode_ids(&podcast_id, &[]);
|
||||
assert_eq!(ids[0], EpisodeId::from_guid(&podcast_id, "g1"));
|
||||
assert_eq!(
|
||||
ids[1],
|
||||
Some(EpisodeId::derive(
|
||||
&podcast_id,
|
||||
&crate::podcast::EpisodeKey::link("https://example.test/no-guid").unwrap()
|
||||
))
|
||||
);
|
||||
assert_eq!(
|
||||
ids[2],
|
||||
Some(EpisodeId::derive(
|
||||
&podcast_id,
|
||||
&crate::podcast::EpisodeKey::enclosure("https://cdn.example.test/home.mp3").unwrap()
|
||||
))
|
||||
);
|
||||
|
||||
let podcast = feed.to_podcast_meta(base.clone());
|
||||
podcast.validate().unwrap();
|
||||
for (episode, id) in feed.episodes.iter().zip(ids) {
|
||||
let meta = episode.to_meta(&podcast_id, id.unwrap());
|
||||
meta.validate().unwrap();
|
||||
meta.to_ref(&podcast).validate().unwrap();
|
||||
}
|
||||
|
||||
// A later refresh with a changed title and audio URL keeps every id,
|
||||
// using the stored hints of the previous refresh.
|
||||
let known: Vec<KnownEpisode> = feed
|
||||
.episodes
|
||||
.iter()
|
||||
.zip(feed.resolve_episode_ids(&podcast_id, &[]))
|
||||
.map(|(episode, id)| KnownEpisode {
|
||||
episode_id: id.unwrap(),
|
||||
hints: episode.identity_hints(),
|
||||
})
|
||||
.collect();
|
||||
let refreshed = items
|
||||
.replace("Home link", "Home link (edited)")
|
||||
.replace("cdn.example.test/ng.mp3", "cdn2.example.test/ng.mp3");
|
||||
let feed2 = parse_feed_with_base(rss("", &refreshed).as_bytes(), Some(base.as_str())).unwrap();
|
||||
let ids2 = feed2.resolve_episode_ids(&podcast_id, &known);
|
||||
let expected: Vec<Option<EpisodeId>> =
|
||||
known.iter().map(|k| Some(k.episode_id.clone())).collect();
|
||||
assert_eq!(ids2, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_sizes_stay_bounded() {
|
||||
let huge_title = "t".repeat(MAX_CAPTURE_BYTES * 2);
|
||||
let xml = rss(
|
||||
"",
|
||||
&format!(
|
||||
r#"<item><guid>h</guid><title>{huge_title}</title><enclosure url="https://c.test/h.mp3"/></item>"#
|
||||
),
|
||||
);
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert!(feed.episodes[0].title.chars().count() <= MAX_TITLE_CHARS);
|
||||
let duration = rss(
|
||||
"",
|
||||
r#"<item><guid>d</guid><itunes:duration>99999999999999999999</itunes:duration><enclosure url="https://c.test/d.mp3"/></item>"#,
|
||||
);
|
||||
assert_eq!(
|
||||
parse_feed(duration.as_bytes()).unwrap().episodes[0].duration_ms,
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_full_of_unclosed_angle_brackets_is_linear() {
|
||||
// '<a' repeated up to the capture cap, with no '>' anywhere: the old
|
||||
// scanner searched for '>' to the end of the text at every '<'.
|
||||
let body = "<a".repeat(MAX_CAPTURE_BYTES / 2);
|
||||
let xml = rss(
|
||||
"",
|
||||
&format!(
|
||||
r#"<item><guid>lt</guid><title>{body}</title><description>{body}</description><enclosure url="https://c.test/lt.mp3"/></item>"#
|
||||
),
|
||||
);
|
||||
let started = std::time::Instant::now();
|
||||
let feed = parse_feed(xml.as_bytes()).unwrap();
|
||||
assert!(
|
||||
started.elapsed() < std::time::Duration::from_secs(2),
|
||||
"took {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
let episode = &feed.episodes[0];
|
||||
assert!(episode.title.starts_with("<a<a"));
|
||||
assert!(episode.title.chars().count() <= MAX_TITLE_CHARS);
|
||||
assert_eq!(
|
||||
html_to_text("a < b and <b>bold</b> <c", 100),
|
||||
"a < b and bold <c"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn links_are_identity_material_and_kept_verbatim() {
|
||||
// Entities are decoded once by the XML reader and never again; tag-like
|
||||
// text, inner whitespace and long links are not rewritten by the display
|
||||
// cleaner.
|
||||
let long_path = "p".repeat(1_500);
|
||||
let items = format!(
|
||||
r#"<item><title>A</title><link> https://e.test/a?x=1&amp;y=<b> </link><enclosure url="https://c.test/a.mp3"/></item>
|
||||
<item><title>B</title><link>https://e.test/{long_path}</link><enclosure url="https://c.test/b.mp3"/></item>"#
|
||||
);
|
||||
let feed = parse_feed(rss("", &items).as_bytes()).unwrap();
|
||||
assert_eq!(
|
||||
feed.episodes[0].link.as_deref(),
|
||||
Some("https://e.test/a?x=1&y=<b>")
|
||||
);
|
||||
let long = format!("https://e.test/{long_path}");
|
||||
assert_eq!(feed.episodes[1].link.as_deref(), Some(long.as_str()));
|
||||
let podcast_id =
|
||||
PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed").unwrap());
|
||||
let ids = feed.resolve_episode_ids(&podcast_id, &[]);
|
||||
assert_eq!(
|
||||
ids[1],
|
||||
Some(EpisodeId::derive(
|
||||
&podcast_id,
|
||||
&crate::podcast::EpisodeKey::link(&long).unwrap()
|
||||
))
|
||||
);
|
||||
let atom = r#"<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>T</title>
|
||||
<link href=" https://e.test/home?a=1&b=2 "/>
|
||||
<entry><title>E</title><link rel="alternate" href="https://e.test/e?q=<x>"/>
|
||||
<link rel="enclosure" href="https://c.test/e.mp3" type="audio/mpeg"/></entry></feed>"#;
|
||||
let feed = parse_feed(atom.as_bytes()).unwrap();
|
||||
assert_eq!(feed.link.as_deref(), Some("https://e.test/home?a=1&b=2"));
|
||||
assert_eq!(
|
||||
feed.episodes[0].link.as_deref(),
|
||||
Some("https://e.test/e?q=<x>")
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,530 @@
|
||||
//! Feed URL canonicalization and feed aliases.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{MAX_FEED_ALIASES, MAX_URL_BYTES, PodcastError, PodcastId, PodcastResult};
|
||||
|
||||
/// Canonical podcast feed URL.
|
||||
///
|
||||
/// Canonicalization ([`FeedUrl::parse`]) is deliberately small so that every
|
||||
/// client, whatever its URL library, produces the same bytes:
|
||||
///
|
||||
/// 1. Only `http://` and `https://` (any case) are accepted; the scheme is
|
||||
/// lowercased.
|
||||
/// 2. Userinfo (`user:pass@`), whitespace, control characters and `\` are
|
||||
/// rejected.
|
||||
/// 3. The host is lowercased. Non-ASCII hosts are rejected: pass the IDNA
|
||||
/// (`xn--`) form, as produced by WHATWG URL parsers. IPv6 literals are
|
||||
/// accepted in brackets.
|
||||
/// 4. An empty port and the scheme's default port (80, 443) are dropped;
|
||||
/// other ports lose leading zeros.
|
||||
/// 5. The fragment is dropped.
|
||||
/// 6. An empty path becomes `/`. The path and the **whole** query string are
|
||||
/// otherwise kept byte-for-byte, including parameter order, case and
|
||||
/// percent-encoding; only non-ASCII bytes are percent-encoded (`%XX`,
|
||||
/// uppercase hex) so the result is ASCII.
|
||||
///
|
||||
/// The canonical form is at most [`MAX_URL_BYTES`] bytes and canonicalizing it
|
||||
/// again is the identity. Deserialization accepts only canonical strings.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct FeedUrl(String);
|
||||
|
||||
impl FeedUrl {
|
||||
/// Canonicalizes an absolute `http(s)` feed URL.
|
||||
pub fn parse(input: &str) -> PodcastResult<Self> {
|
||||
canonicalize(input).map(Self)
|
||||
}
|
||||
|
||||
/// Canonicalizes a URL typed or pasted by a user.
|
||||
///
|
||||
/// Surrounding whitespace is trimmed, `feed:`, `podcast:`, `pcast:` and
|
||||
/// `itpc:` prefixes map to `https://` (or to the embedded `http(s)` URL),
|
||||
/// and input without a scheme defaults to `https://`. `http:` or `https:`
|
||||
/// not followed by `//` (for example `https:/host/feed`) is rejected.
|
||||
pub fn from_user_input(input: &str) -> PodcastResult<Self> {
|
||||
let trimmed = input.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(invalid("feed URL is empty"));
|
||||
}
|
||||
let lower = trimmed.to_ascii_lowercase();
|
||||
if is_http_scheme_typo(&lower) {
|
||||
return Err(invalid("feed URL must start with http:// or https://"));
|
||||
}
|
||||
let mut candidate = trimmed.to_string();
|
||||
for scheme in ["feed:", "podcast:", "pcast:", "itpc:"] {
|
||||
if lower.starts_with(scheme) {
|
||||
let rest = &trimmed[scheme.len()..];
|
||||
let rest_lower = rest.to_ascii_lowercase();
|
||||
if is_http_scheme_typo(&rest_lower) {
|
||||
return Err(invalid("feed URL must start with http:// or https://"));
|
||||
}
|
||||
candidate =
|
||||
if rest_lower.starts_with("http://") || rest_lower.starts_with("https://") {
|
||||
rest.to_string()
|
||||
} else {
|
||||
format!("https://{}", rest.trim_start_matches('/'))
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !has_scheme(&candidate) {
|
||||
candidate = format!("https://{candidate}");
|
||||
}
|
||||
Self::parse(&candidate)
|
||||
}
|
||||
|
||||
/// Canonical string form.
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FeedUrl {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<String> for FeedUrl {
|
||||
type Error = PodcastError;
|
||||
|
||||
fn try_from(value: String) -> PodcastResult<Self> {
|
||||
if canonicalize(&value)? == value {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(invalid("feed URL is not in canonical form"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<FeedUrl> for String {
|
||||
fn from(value: FeedUrl) -> String {
|
||||
value.0
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<str> for FeedUrl {
|
||||
fn as_ref(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid(reason: &str) -> PodcastError {
|
||||
PodcastError::InvalidFeedUrl(reason.to_string())
|
||||
}
|
||||
|
||||
/// `http:` or `https:` not followed by `//`, such as `https:/host/feed`.
|
||||
fn is_http_scheme_typo(lower: &str) -> bool {
|
||||
["http:", "https:"]
|
||||
.iter()
|
||||
.any(|scheme| lower.starts_with(scheme) && !lower[scheme.len()..].starts_with("//"))
|
||||
}
|
||||
|
||||
fn has_scheme(text: &str) -> bool {
|
||||
let Some((scheme, rest)) = text.split_once(':') else {
|
||||
return false;
|
||||
};
|
||||
let mut chars = scheme.chars();
|
||||
chars.next().is_some_and(|c| c.is_ascii_alphabetic())
|
||||
&& chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
|
||||
&& rest.starts_with("//")
|
||||
}
|
||||
|
||||
fn canonicalize(input: &str) -> PodcastResult<String> {
|
||||
if input.len() > MAX_URL_BYTES * 3 {
|
||||
return Err(invalid("feed URL is too long"));
|
||||
}
|
||||
if input.chars().any(|c| c.is_whitespace() || c.is_control()) {
|
||||
return Err(invalid("feed URL must not contain whitespace"));
|
||||
}
|
||||
if input.contains('\\') {
|
||||
return Err(invalid("feed URL must not contain '\\'"));
|
||||
}
|
||||
let (scheme, rest) = input
|
||||
.split_once("://")
|
||||
.ok_or_else(|| invalid("feed URL must start with http:// or https://"))?;
|
||||
let (scheme, default_port) = if scheme.eq_ignore_ascii_case("http") {
|
||||
("http", 80)
|
||||
} else if scheme.eq_ignore_ascii_case("https") {
|
||||
("https", 443)
|
||||
} else {
|
||||
return Err(invalid("only http and https feed URLs are supported"));
|
||||
};
|
||||
let authority_end = rest.find(['/', '?', '#']).unwrap_or(rest.len());
|
||||
let (authority, tail) = rest.split_at(authority_end);
|
||||
if authority.contains('@') {
|
||||
return Err(invalid("feed URL must not contain credentials"));
|
||||
}
|
||||
let (host, port) = split_host_port(authority)?;
|
||||
let host = canonical_host(host)?;
|
||||
let port = match port {
|
||||
None | Some("") => None,
|
||||
Some(port) => {
|
||||
if port.len() > 5 || !port.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(invalid("invalid feed URL port"));
|
||||
}
|
||||
match port.parse::<u32>() {
|
||||
Ok(value) if (1..=65_535).contains(&value) => {
|
||||
(value != default_port).then_some(value)
|
||||
}
|
||||
_ => return Err(invalid("invalid feed URL port")),
|
||||
}
|
||||
}
|
||||
};
|
||||
let tail = tail.split('#').next().unwrap_or_default();
|
||||
let (path, query) = match tail.find('?') {
|
||||
Some(at) => tail.split_at(at),
|
||||
None => (tail, ""),
|
||||
};
|
||||
let mut out = String::with_capacity(input.len() + 8);
|
||||
out.push_str(scheme);
|
||||
out.push_str("://");
|
||||
out.push_str(&host);
|
||||
if let Some(port) = port {
|
||||
out.push(':');
|
||||
out.push_str(&port.to_string());
|
||||
}
|
||||
if path.is_empty() {
|
||||
out.push('/');
|
||||
} else {
|
||||
push_ascii(&mut out, path);
|
||||
}
|
||||
push_ascii(&mut out, query);
|
||||
if out.len() > MAX_URL_BYTES {
|
||||
return Err(invalid("feed URL is too long"));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn split_host_port(authority: &str) -> PodcastResult<(&str, Option<&str>)> {
|
||||
if authority.starts_with('[') {
|
||||
let close = authority
|
||||
.find(']')
|
||||
.ok_or_else(|| invalid("unterminated IPv6 host"))?;
|
||||
let (host, after) = authority.split_at(close + 1);
|
||||
return match after {
|
||||
"" => Ok((host, None)),
|
||||
_ => after
|
||||
.strip_prefix(':')
|
||||
.map(|port| (host, Some(port)))
|
||||
.ok_or_else(|| invalid("invalid feed URL host")),
|
||||
};
|
||||
}
|
||||
Ok(match authority.rsplit_once(':') {
|
||||
Some((host, port)) => (host, Some(port)),
|
||||
None => (authority, None),
|
||||
})
|
||||
}
|
||||
|
||||
fn canonical_host(host: &str) -> PodcastResult<String> {
|
||||
if host.is_empty() {
|
||||
return Err(invalid("feed URL has no host"));
|
||||
}
|
||||
if !host.is_ascii() {
|
||||
return Err(invalid(
|
||||
"feed URL host must be ASCII (use the IDNA xn-- form)",
|
||||
));
|
||||
}
|
||||
if let Some(inner) = host.strip_prefix('[').and_then(|h| h.strip_suffix(']')) {
|
||||
if inner.is_empty()
|
||||
|| !inner
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_hexdigit() || matches!(b, b':' | b'.'))
|
||||
{
|
||||
return Err(invalid("invalid IPv6 host"));
|
||||
}
|
||||
return Ok(host.to_ascii_lowercase());
|
||||
}
|
||||
if !host
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_'))
|
||||
|| !host.bytes().any(|b| b.is_ascii_alphanumeric())
|
||||
{
|
||||
return Err(invalid("invalid feed URL host"));
|
||||
}
|
||||
Ok(host.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
fn push_ascii(out: &mut String, value: &str) {
|
||||
for ch in value.chars() {
|
||||
if ch.is_ascii() {
|
||||
out.push(ch);
|
||||
} else {
|
||||
let mut buf = [0u8; 4];
|
||||
for byte in ch.encode_utf8(&mut buf).bytes() {
|
||||
out.push_str(&format!("%{byte:02X}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the podcast a canonical feed URL belongs to.
|
||||
///
|
||||
/// `known` yields `(podcast_id, url)` pairs of every URL already associated
|
||||
/// with a podcast: the URL of its first subscription, its current fetch URL
|
||||
/// and its feed aliases ([`super::SubscriptionRecord::feed_urls`]; the bounded
|
||||
/// alias set alone can miss the current fetch URL). When `url` matches, the existing id is returned (the
|
||||
/// smallest id if, after a concurrent offline subscription, several podcasts
|
||||
/// share the URL). Otherwise a new id is derived from `url`.
|
||||
///
|
||||
/// Re-subscribing through a redirect target or an `itunes:new-feed-url`
|
||||
/// therefore reuses the original podcast, its progress and its history.
|
||||
pub fn resolve_podcast_id<'a>(
|
||||
url: &FeedUrl,
|
||||
known: impl IntoIterator<Item = (&'a PodcastId, &'a FeedUrl)>,
|
||||
) -> PodcastId {
|
||||
known
|
||||
.into_iter()
|
||||
.filter(|(_, alias)| *alias == url)
|
||||
.map(|(id, _)| id)
|
||||
.min()
|
||||
.cloned()
|
||||
.unwrap_or_else(|| PodcastId::from_feed_url(url))
|
||||
}
|
||||
|
||||
/// Merges two feed alias sets of `podcast` as a bounded grow-only set.
|
||||
///
|
||||
/// The result is sorted and duplicate-free. The URL from which `podcast` was
|
||||
/// derived is always kept; of the other URLs the [`MAX_FEED_ALIASES`]
|
||||
/// lexicographically smallest are kept. Taking the smallest `k` elements of a
|
||||
/// union is commutative, associative and idempotent, so devices converge
|
||||
/// regardless of delivery order. Other URLs, including a podcast's current
|
||||
/// fetch URL, can fall outside the bound; the materialized subscription keeps
|
||||
/// its `podcast.feed_url` separately.
|
||||
pub fn merge_feed_aliases(podcast: &PodcastId, a: &[FeedUrl], b: &[FeedUrl]) -> Vec<FeedUrl> {
|
||||
let mut origin = None;
|
||||
let mut rest = BTreeSet::new();
|
||||
for url in a.iter().chain(b) {
|
||||
if PodcastId::from_feed_url(url) == *podcast {
|
||||
origin = Some(url.clone());
|
||||
} else {
|
||||
rest.insert(url.clone());
|
||||
}
|
||||
}
|
||||
let mut out: Vec<FeedUrl> = rest.into_iter().take(MAX_FEED_ALIASES).collect();
|
||||
if let Some(origin) = origin {
|
||||
out.push(origin);
|
||||
out.sort();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn canon(input: &str) -> String {
|
||||
FeedUrl::parse(input).unwrap().as_str().to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalization_rules() {
|
||||
let cases = [
|
||||
(
|
||||
"https://example.com/feed.xml",
|
||||
"https://example.com/feed.xml",
|
||||
),
|
||||
(
|
||||
"HTTPS://Example.COM/Feed.XML",
|
||||
"https://example.com/Feed.XML",
|
||||
),
|
||||
("http://example.com:80/rss", "http://example.com/rss"),
|
||||
("https://example.com:443/rss", "https://example.com/rss"),
|
||||
("http://example.com:443/rss", "http://example.com:443/rss"),
|
||||
(
|
||||
"https://example.com:08080/rss",
|
||||
"https://example.com:8080/rss",
|
||||
),
|
||||
("https://example.com:/rss", "https://example.com/rss"),
|
||||
("https://example.com", "https://example.com/"),
|
||||
("https://example.com?x=1", "https://example.com/?x=1"),
|
||||
("https://example.com/rss#latest", "https://example.com/rss"),
|
||||
("https://example.com/rss?#x", "https://example.com/rss?"),
|
||||
(
|
||||
"https://example.com/a/../b/./c?b=2&a=1&A=%2f&token=XyZ",
|
||||
"https://example.com/a/../b/./c?b=2&a=1&A=%2f&token=XyZ",
|
||||
),
|
||||
(
|
||||
"https://example.com/rss?url=https://other.test/x?y#frag",
|
||||
"https://example.com/rss?url=https://other.test/x?y",
|
||||
),
|
||||
(
|
||||
"https://example.com/подкаст?q=ё",
|
||||
"https://example.com/%D0%BF%D0%BE%D0%B4%D0%BA%D0%B0%D1%81%D1%82?q=%D1%91",
|
||||
),
|
||||
(
|
||||
"https://[2001:DB8::1]:443/feed",
|
||||
"https://[2001:db8::1]/feed",
|
||||
),
|
||||
("https://[::1]:8443/feed", "https://[::1]:8443/feed"),
|
||||
(
|
||||
"https://xn--d1acufc.xn--p1ai/rss",
|
||||
"https://xn--d1acufc.xn--p1ai/rss",
|
||||
),
|
||||
(
|
||||
"https://my_host.example./rss",
|
||||
"https://my_host.example./rss",
|
||||
),
|
||||
];
|
||||
for (input, expected) in cases {
|
||||
assert_eq!(canon(input), expected, "{input:?}");
|
||||
// Idempotent.
|
||||
assert_eq!(canon(expected), expected, "{expected:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejected_urls() {
|
||||
for bad in [
|
||||
"",
|
||||
"example.com/feed",
|
||||
"ftp://example.com/feed",
|
||||
"file:///etc/passwd",
|
||||
"javascript:alert(1)",
|
||||
"https://",
|
||||
"https:///path",
|
||||
"https://user:pass@example.com/feed",
|
||||
"https://token@example.com/feed",
|
||||
"https://example.com/my feed",
|
||||
" https://example.com/feed",
|
||||
"https://example.com/feed\n",
|
||||
"https://example.com\\feed",
|
||||
"https://пример.рф/rss",
|
||||
"https://example.com:0/rss",
|
||||
"https://example.com:65536/rss",
|
||||
"https://example.com:12a/rss",
|
||||
"https://[::1/feed",
|
||||
"https://[zz]/feed",
|
||||
"https://exa%6dple.com/feed",
|
||||
"https://.../feed",
|
||||
] {
|
||||
assert!(FeedUrl::parse(bad).is_err(), "{bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn length_limit_is_inclusive() {
|
||||
let prefix = "https://a.test/";
|
||||
let at = format!("{prefix}{}", "x".repeat(MAX_URL_BYTES - prefix.len()));
|
||||
assert_eq!(canon(&at).len(), MAX_URL_BYTES);
|
||||
let over = format!("{at}x");
|
||||
assert!(FeedUrl::parse(&over).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_input_forms() {
|
||||
let ok = [
|
||||
(
|
||||
"https://podster.test/rss.xml?pid=778",
|
||||
"https://podster.test/rss.xml?pid=778",
|
||||
),
|
||||
(" http://a.test/feed ", "http://a.test/feed"),
|
||||
("a.test/feed.xml", "https://a.test/feed.xml"),
|
||||
("feed://a.test/rss", "https://a.test/rss"),
|
||||
("feed:https://a.test/rss", "https://a.test/rss"),
|
||||
("podcast://a.test/rss", "https://a.test/rss"),
|
||||
("itpc://a.test/rss", "https://a.test/rss"),
|
||||
("HTTPS://A.TEST/Feed", "https://a.test/Feed"),
|
||||
("a.test:8080/rss", "https://a.test:8080/rss"),
|
||||
(
|
||||
"example.test/rss?src=https://other.test",
|
||||
"https://example.test/rss?src=https://other.test",
|
||||
),
|
||||
(
|
||||
"localhost:8080/feed?u=http://x.test/",
|
||||
"https://localhost:8080/feed?u=http://x.test/",
|
||||
),
|
||||
];
|
||||
for (input, expected) in ok {
|
||||
assert_eq!(
|
||||
FeedUrl::from_user_input(input).unwrap().as_str(),
|
||||
expected,
|
||||
"{input:?}"
|
||||
);
|
||||
}
|
||||
for bad in [
|
||||
"",
|
||||
" ",
|
||||
"ftp://a.test/feed",
|
||||
"https://a.test/my feed",
|
||||
"https://",
|
||||
"file:///etc/passwd",
|
||||
"javascript:alert(1)",
|
||||
// One-slash typos must not become a host named "https".
|
||||
"https:/a.test/feed",
|
||||
"http:/a.test/feed",
|
||||
"HTTPS:a.test/feed",
|
||||
"https:///a.test/feed",
|
||||
"feed:https:/a.test/feed",
|
||||
] {
|
||||
assert!(FeedUrl::from_user_input(bad).is_err(), "{bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialization_requires_canonical_form() {
|
||||
let url = FeedUrl::parse("https://example.com/rss").unwrap();
|
||||
let json = serde_json::to_string(&url).unwrap();
|
||||
assert_eq!(serde_json::from_str::<FeedUrl>(&json).unwrap(), url);
|
||||
assert!(serde_json::from_str::<FeedUrl>("\"HTTPS://example.com/rss\"").is_err());
|
||||
assert!(serde_json::from_str::<FeedUrl>("\"https://example.com/rss#x\"").is_err());
|
||||
assert!(serde_json::from_str::<FeedUrl>("\"ftp://example.com/rss\"").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_parameters_change_identity() {
|
||||
let a = PodcastId::from_feed_url(&FeedUrl::parse("https://a.test/rss?id=1").unwrap());
|
||||
let b = PodcastId::from_feed_url(&FeedUrl::parse("https://a.test/rss?id=2").unwrap());
|
||||
let c = PodcastId::from_feed_url(&FeedUrl::parse("HTTPS://A.test:443/rss?id=1#x").unwrap());
|
||||
assert_ne!(a, b);
|
||||
assert_eq!(a, c);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aliases_resolve_to_the_original_podcast() {
|
||||
let original = FeedUrl::parse("http://old.test/feed").unwrap();
|
||||
let moved = FeedUrl::parse("https://new.test/feed").unwrap();
|
||||
let id = PodcastId::from_feed_url(&original);
|
||||
let known = [(&id, &original), (&id, &moved)];
|
||||
assert_eq!(resolve_podcast_id(&moved, known), id);
|
||||
assert_eq!(resolve_podcast_id(&original, known), id);
|
||||
let unrelated = FeedUrl::parse("https://other.test/feed").unwrap();
|
||||
assert_eq!(
|
||||
resolve_podcast_id(&unrelated, known),
|
||||
PodcastId::from_feed_url(&unrelated)
|
||||
);
|
||||
// Ambiguity after concurrent offline subscriptions resolves to the
|
||||
// smallest id, independent of iteration order.
|
||||
let other = PodcastId::from_feed_url(&moved);
|
||||
let ambiguous = [(&id, &moved), (&other, &moved)];
|
||||
let reversed = [(&other, &moved), (&id, &moved)];
|
||||
let expected = (&id).min(&other).clone();
|
||||
assert_eq!(resolve_podcast_id(&moved, ambiguous), expected);
|
||||
assert_eq!(resolve_podcast_id(&moved, reversed), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alias_merge_is_a_bounded_semilattice() {
|
||||
let origin = FeedUrl::parse("https://zzz.test/origin").unwrap();
|
||||
let id = PodcastId::from_feed_url(&origin);
|
||||
let urls: Vec<FeedUrl> = (0..(MAX_FEED_ALIASES + 4))
|
||||
.map(|i| FeedUrl::parse(&format!("https://a.test/{i:02}")).unwrap())
|
||||
.collect();
|
||||
let a: Vec<FeedUrl> = urls[..5].to_vec();
|
||||
let b: Vec<FeedUrl> = urls[3..].iter().cloned().chain([origin.clone()]).collect();
|
||||
let c: Vec<FeedUrl> = urls[7..].to_vec();
|
||||
let ab = merge_feed_aliases(&id, &a, &b);
|
||||
assert_eq!(ab, merge_feed_aliases(&id, &b, &a));
|
||||
assert_eq!(ab, merge_feed_aliases(&id, &ab, &ab));
|
||||
let left = merge_feed_aliases(&id, &ab, &c);
|
||||
let right = merge_feed_aliases(&id, &a, &merge_feed_aliases(&id, &b, &c));
|
||||
assert_eq!(left, right);
|
||||
assert_eq!(left.len(), MAX_FEED_ALIASES + 1);
|
||||
assert!(left.contains(&origin));
|
||||
assert!(left.windows(2).all(|w| w[0] < w[1]));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
//! Cross-device podcast and episode identifiers.
|
||||
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{FeedUrl, MAX_GUID_BYTES, PodcastError, PodcastResult};
|
||||
|
||||
/// BLAKE3 derivation domain of [`PodcastId`]. Changing it changes every id.
|
||||
pub const PODCAST_ID_DOMAIN: &str = "furumi-podcast:podcast:v1";
|
||||
/// BLAKE3 derivation domain of [`EpisodeId`]. Changing it changes every id.
|
||||
pub const EPISODE_ID_DOMAIN: &str = "furumi-podcast:episode:v1";
|
||||
/// String prefix of a [`PodcastId`].
|
||||
pub const PODCAST_ID_PREFIX: &str = "pod1:";
|
||||
/// String prefix of an [`EpisodeId`].
|
||||
pub const EPISODE_ID_PREFIX: &str = "ep1:";
|
||||
|
||||
/// Length of the base32 body: 32 bytes, RFC 4648 lowercase, no padding.
|
||||
const DIGEST_CHARS: usize = 52;
|
||||
|
||||
fn digest(domain: &str, parts: &[&[u8]]) -> [u8; 32] {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
hasher.update(domain.as_bytes());
|
||||
hasher.update(&[0]);
|
||||
for part in parts {
|
||||
hasher.update(part);
|
||||
}
|
||||
*hasher.finalize().as_bytes()
|
||||
}
|
||||
|
||||
fn encode(prefix: &str, digest: &[u8; 32]) -> String {
|
||||
let mut body = data_encoding::BASE32_NOPAD.encode(digest);
|
||||
body.make_ascii_lowercase();
|
||||
format!("{prefix}{body}")
|
||||
}
|
||||
|
||||
fn decode(prefix: &str, value: &str) -> Option<[u8; 32]> {
|
||||
let body = value.strip_prefix(prefix)?;
|
||||
if body.len() != DIGEST_CHARS
|
||||
|| !body
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_lowercase() || (b'2'..=b'7').contains(&b))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let bytes = data_encoding::BASE32_NOPAD
|
||||
.decode(body.to_ascii_uppercase().as_bytes())
|
||||
.ok()?;
|
||||
let digest: [u8; 32] = bytes.try_into().ok()?;
|
||||
// Reject non-canonical trailing bits so every id has one spelling.
|
||||
(encode(prefix, &digest) == value).then_some(digest)
|
||||
}
|
||||
|
||||
macro_rules! string_id {
|
||||
($name:ident, $prefix:expr, $kind:literal) => {
|
||||
impl $name {
|
||||
/// Parses and validates the canonical string form.
|
||||
pub fn parse(value: &str) -> PodcastResult<Self> {
|
||||
decode($prefix, value)
|
||||
.map(|_| Self(value.to_string()))
|
||||
.ok_or(PodcastError::InvalidId { kind: $kind })
|
||||
}
|
||||
|
||||
/// Canonical string form.
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
|
||||
/// The 32-byte BLAKE3 digest encoded in the id.
|
||||
pub fn digest(&self) -> [u8; 32] {
|
||||
decode($prefix, &self.0).expect("validated on construction")
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for $name {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for $name {
|
||||
type Err = PodcastError;
|
||||
|
||||
fn from_str(value: &str) -> PodcastResult<Self> {
|
||||
Self::parse(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<String> for $name {
|
||||
type Error = PodcastError;
|
||||
|
||||
fn try_from(value: String) -> PodcastResult<Self> {
|
||||
decode($prefix, &value)
|
||||
.map(|_| Self(value))
|
||||
.ok_or(PodcastError::InvalidId { kind: $kind })
|
||||
}
|
||||
}
|
||||
|
||||
impl From<$name> for String {
|
||||
fn from(value: $name) -> String {
|
||||
value.0
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<str> for $name {
|
||||
fn as_ref(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Stable cross-device podcast identity: `pod1:<52 lowercase base32 chars>`.
|
||||
///
|
||||
/// Derived as `BLAKE3("furumi-podcast:podcast:v1" || 0x00 || canonical_feed_url)`
|
||||
/// from the URL used by the **first** subscription. The id never changes when
|
||||
/// the feed moves; new URLs are recorded as aliases (see
|
||||
/// [`super::resolve_podcast_id`]). Deserialization validates the format.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct PodcastId(String);
|
||||
|
||||
string_id!(PodcastId, PODCAST_ID_PREFIX, "podcast");
|
||||
|
||||
impl PodcastId {
|
||||
/// Derives the id of a podcast first subscribed through `url`.
|
||||
pub fn from_feed_url(url: &FeedUrl) -> Self {
|
||||
Self(encode(
|
||||
PODCAST_ID_PREFIX,
|
||||
&digest(PODCAST_ID_DOMAIN, &[url.as_str().as_bytes()]),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Stable cross-device episode identity: `ep1:<52 lowercase base32 chars>`.
|
||||
///
|
||||
/// Derived as `BLAKE3("furumi-podcast:episode:v1" || 0x00 || podcast_id || key)`
|
||||
/// where `key` is the material of an [`EpisodeKey`]. Title, artwork or audio
|
||||
/// URL changes never change the id of an existing episode.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(try_from = "String", into = "String")]
|
||||
pub struct EpisodeId(String);
|
||||
|
||||
string_id!(EpisodeId, EPISODE_ID_PREFIX, "episode");
|
||||
|
||||
impl EpisodeId {
|
||||
/// Derives an episode id from a key inside `podcast`.
|
||||
pub fn derive(podcast: &PodcastId, key: &EpisodeKey) -> Self {
|
||||
let material = key.material();
|
||||
Self(encode(
|
||||
EPISODE_ID_PREFIX,
|
||||
&digest(
|
||||
EPISODE_ID_DOMAIN,
|
||||
&[podcast.as_str().as_bytes(), material.as_bytes()],
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
/// Derives the id of an episode with an RSS GUID (or Atom `<id>`).
|
||||
///
|
||||
/// Returns `None` when the GUID is blank or longer than [`MAX_GUID_BYTES`].
|
||||
pub fn from_guid(podcast: &PodcastId, guid: &str) -> Option<Self> {
|
||||
EpisodeKey::guid(guid).map(|key| Self::derive(podcast, &key))
|
||||
}
|
||||
}
|
||||
|
||||
/// Key material from which an [`EpisodeId`] is derived.
|
||||
///
|
||||
/// | Variant | Material |
|
||||
/// | --- | --- |
|
||||
/// | `Guid` | `guid:` + GUID trimmed of surrounding whitespace |
|
||||
/// | `Link` | `link:` + `<link>` trimmed |
|
||||
/// | `DateTitle` | `date-title:` + unix seconds + `\0` + [`super::normalize_episode_title`] |
|
||||
/// | `Enclosure` | `enclosure:` + enclosure URL trimmed |
|
||||
///
|
||||
/// GUIDs and URLs are otherwise byte-for-byte: GUIDs are case-sensitive.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum EpisodeKey {
|
||||
/// RSS `<guid>` or Atom `<id>`.
|
||||
Guid(String),
|
||||
/// Item `<link>`.
|
||||
Link(String),
|
||||
/// Publication time (whole unix seconds) and normalized title.
|
||||
DateTitle {
|
||||
/// Publication time in unix seconds.
|
||||
published_at_secs: i64,
|
||||
/// Title normalized by [`super::normalize_episode_title`].
|
||||
normalized_title: String,
|
||||
},
|
||||
/// Enclosure (audio) URL.
|
||||
Enclosure(String),
|
||||
}
|
||||
|
||||
impl EpisodeKey {
|
||||
/// GUID key; `None` for a blank or oversized GUID.
|
||||
pub fn guid(guid: &str) -> Option<Self> {
|
||||
let guid = guid.trim();
|
||||
(!guid.is_empty() && guid.len() <= MAX_GUID_BYTES).then(|| Self::Guid(guid.to_string()))
|
||||
}
|
||||
|
||||
/// Link key; `None` for a blank or oversized link.
|
||||
pub fn link(link: &str) -> Option<Self> {
|
||||
let link = link.trim();
|
||||
(!link.is_empty() && link.len() <= super::MAX_URL_BYTES)
|
||||
.then(|| Self::Link(link.to_string()))
|
||||
}
|
||||
|
||||
/// Date + title key; `None` when the title normalizes to nothing.
|
||||
pub fn date_title(published_at_ms: i64, title: &str) -> Option<Self> {
|
||||
let normalized_title = super::normalize_episode_title(title);
|
||||
(!normalized_title.is_empty()).then(|| Self::DateTitle {
|
||||
published_at_secs: published_at_ms.div_euclid(1_000),
|
||||
normalized_title,
|
||||
})
|
||||
}
|
||||
|
||||
/// Enclosure key; `None` for a blank or oversized URL.
|
||||
pub fn enclosure(url: &str) -> Option<Self> {
|
||||
let url = url.trim();
|
||||
(!url.is_empty() && url.len() <= super::MAX_URL_BYTES)
|
||||
.then(|| Self::Enclosure(url.to_string()))
|
||||
}
|
||||
|
||||
/// Exact bytes hashed after the podcast id.
|
||||
pub fn material(&self) -> String {
|
||||
match self {
|
||||
Self::Guid(guid) => format!("guid:{guid}"),
|
||||
Self::Link(link) => format!("link:{link}"),
|
||||
Self::DateTitle {
|
||||
published_at_secs,
|
||||
normalized_title,
|
||||
} => format!("date-title:{published_at_secs}\0{normalized_title}"),
|
||||
Self::Enclosure(url) => format!("enclosure:{url}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn podcast() -> PodcastId {
|
||||
PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed.xml").unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_id_is_domain_separated_blake3() {
|
||||
let url = FeedUrl::parse("https://example.test/feed.xml").unwrap();
|
||||
let mut input = b"furumi-podcast:podcast:v1\0".to_vec();
|
||||
input.extend_from_slice(url.as_str().as_bytes());
|
||||
let expected = blake3::hash(&input);
|
||||
let id = PodcastId::from_feed_url(&url);
|
||||
assert_eq!(&id.digest(), expected.as_bytes());
|
||||
assert_eq!(id.as_str().len(), PODCAST_ID_PREFIX.len() + DIGEST_CHARS);
|
||||
assert!(id.as_str()[5..].bytes().all(|b| !b.is_ascii_uppercase()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn podcast_id_golden_value() {
|
||||
// Golden vector: changing it breaks every stored and synced id.
|
||||
let id = podcast();
|
||||
let mut input = b"furumi-podcast:podcast:v1\0".to_vec();
|
||||
input.extend_from_slice(b"https://example.test/feed.xml");
|
||||
let mut body = data_encoding::BASE32_NOPAD.encode(blake3::hash(&input).as_bytes());
|
||||
body.make_ascii_lowercase();
|
||||
assert_eq!(id.as_str(), format!("pod1:{body}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn episode_id_follows_the_plan_formula() {
|
||||
let podcast = podcast();
|
||||
let id = EpisodeId::from_guid(&podcast, " abc-123 ").unwrap();
|
||||
let mut input = b"furumi-podcast:episode:v1\0".to_vec();
|
||||
input.extend_from_slice(podcast.as_str().as_bytes());
|
||||
input.extend_from_slice(b"guid:abc-123");
|
||||
assert_eq!(&id.digest(), blake3::hash(&input).as_bytes());
|
||||
// GUIDs are case-sensitive and podcast-scoped.
|
||||
assert_ne!(EpisodeId::from_guid(&podcast, "ABC-123"), Some(id.clone()));
|
||||
let other = PodcastId::from_feed_url(&FeedUrl::parse("https://other.test/").unwrap());
|
||||
assert_ne!(EpisodeId::from_guid(&other, "abc-123"), Some(id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keys_are_distinct_per_kind() {
|
||||
let podcast = podcast();
|
||||
let value = "https://example.test/e1";
|
||||
let link = EpisodeId::derive(&podcast, &EpisodeKey::link(value).unwrap());
|
||||
let guid = EpisodeId::derive(&podcast, &EpisodeKey::guid(value).unwrap());
|
||||
let enclosure = EpisodeId::derive(&podcast, &EpisodeKey::enclosure(value).unwrap());
|
||||
assert_ne!(link, guid);
|
||||
assert_ne!(link, enclosure);
|
||||
assert_ne!(guid, enclosure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blank_and_oversized_keys_are_rejected() {
|
||||
assert!(EpisodeKey::guid(" ").is_none());
|
||||
assert!(EpisodeKey::guid(&"g".repeat(MAX_GUID_BYTES)).is_some());
|
||||
assert!(EpisodeKey::guid(&"g".repeat(MAX_GUID_BYTES + 1)).is_none());
|
||||
assert!(EpisodeKey::date_title(0, " -- ").is_none());
|
||||
assert!(EpisodeKey::link("").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn date_title_uses_whole_seconds_and_normalized_title() {
|
||||
let a = EpisodeKey::date_title(1_700_000_000_123, "Hello, World!").unwrap();
|
||||
let b = EpisodeKey::date_title(1_700_000_000_999, "hello world").unwrap();
|
||||
assert_eq!(a, b);
|
||||
let c = EpisodeKey::date_title(-1, "x").unwrap();
|
||||
assert_eq!(
|
||||
c,
|
||||
EpisodeKey::DateTitle {
|
||||
published_at_secs: -1,
|
||||
normalized_title: "x".to_string()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsing_validates_and_round_trips() {
|
||||
let id = podcast();
|
||||
assert_eq!(PodcastId::parse(id.as_str()).unwrap(), id);
|
||||
let json = serde_json::to_string(&id).unwrap();
|
||||
assert_eq!(serde_json::from_str::<PodcastId>(&json).unwrap(), id);
|
||||
|
||||
let upper = id.as_str().to_ascii_uppercase().replace("POD1:", "pod1:");
|
||||
assert!(PodcastId::parse(&upper).is_err());
|
||||
assert!(PodcastId::parse(&id.as_str()[..id.as_str().len() - 1]).is_err());
|
||||
assert!(PodcastId::parse(&format!("{id}a")).is_err());
|
||||
assert!(PodcastId::parse("pod1:").is_err());
|
||||
assert!(PodcastId::parse("").is_err());
|
||||
assert!(EpisodeId::parse(id.as_str()).is_err());
|
||||
assert!(serde_json::from_str::<PodcastId>("\"pod1:xyz\"").is_err());
|
||||
|
||||
// Non-canonical trailing bits: last char encodes 1 data bit + 4 zero bits.
|
||||
let mut chars: Vec<char> = id.as_str().chars().collect();
|
||||
let last = chars.len() - 1;
|
||||
chars[last] = if chars[last] == 'b' { 'c' } else { 'b' };
|
||||
let tampered: String = chars.into_iter().collect();
|
||||
assert!(PodcastId::parse(&tampered).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,604 @@
|
||||
//! Episode identity resolution, including the documented no-GUID rule.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{EpisodeId, EpisodeKey, PodcastId};
|
||||
|
||||
/// Normalizes an episode title for the `(date, title)` identity key.
|
||||
///
|
||||
/// This is [`crate::normalize_name`]: NFKC, lowercase, non-alphanumeric runs
|
||||
/// collapsed to one space, trimmed. It is part of the episode-id derivation.
|
||||
pub fn normalize_episode_title(title: &str) -> String {
|
||||
crate::normalize_name(title)
|
||||
}
|
||||
|
||||
/// Fields of a feed item (or of a stored episode) that identify it.
|
||||
///
|
||||
/// The parser in `podcast::feed` produces these hints; stores keep them for
|
||||
/// known episodes so that a later refresh can recognize items without GUID.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct EpisodeIdentityHints {
|
||||
/// RSS `<guid>` / Atom `<id>`, when present.
|
||||
#[serde(default)]
|
||||
pub guid: Option<String>,
|
||||
/// Enclosure (audio) URL.
|
||||
#[serde(default)]
|
||||
pub enclosure_url: Option<String>,
|
||||
/// Item `<link>`.
|
||||
#[serde(default)]
|
||||
pub link: Option<String>,
|
||||
/// Publication time in unix milliseconds.
|
||||
#[serde(default)]
|
||||
pub published_at_ms: Option<i64>,
|
||||
/// Item title.
|
||||
#[serde(default)]
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
impl EpisodeIdentityHints {
|
||||
fn guid_key(&self) -> Option<EpisodeKey> {
|
||||
self.guid.as_deref().and_then(EpisodeKey::guid)
|
||||
}
|
||||
|
||||
fn enclosure(&self) -> Option<&str> {
|
||||
trimmed(self.enclosure_url.as_deref())
|
||||
}
|
||||
|
||||
fn link(&self) -> Option<&str> {
|
||||
trimmed(self.link.as_deref())
|
||||
}
|
||||
|
||||
fn date_title(&self) -> Option<(i64, String)> {
|
||||
let published = self.published_at_ms?.div_euclid(1_000);
|
||||
let title = normalize_episode_title(self.title.as_deref()?);
|
||||
(!title.is_empty()).then_some((published, title))
|
||||
}
|
||||
}
|
||||
|
||||
/// One matching stage of the no-GUID rule: does `known` match `item`?
|
||||
type StageMatcher<'f> = &'f dyn Fn(&Usable<'_>, &EpisodeIdentityHints) -> bool;
|
||||
|
||||
struct Usable<'a> {
|
||||
link: Option<&'a str>,
|
||||
enclosure: Option<&'a str>,
|
||||
date_title: Option<(i64, String)>,
|
||||
}
|
||||
|
||||
fn trimmed(value: Option<&str>) -> Option<&str> {
|
||||
value.map(str::trim).filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
/// An episode already stored for the podcast, with the hints it was last seen with.
|
||||
///
|
||||
/// Include episodes learned through sync (progress operations, queue items)
|
||||
/// as well as episodes from earlier RSS refreshes.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct KnownEpisode {
|
||||
/// Existing stable id.
|
||||
pub episode_id: EpisodeId,
|
||||
/// Identity hints recorded for it.
|
||||
pub hints: EpisodeIdentityHints,
|
||||
}
|
||||
|
||||
/// Resolves the id of a single feed item. See [`resolve_episode_ids`].
|
||||
///
|
||||
/// A single item has no feed context, so its `<link>` is only used when it
|
||||
/// differs from `channel_link`. Prefer [`resolve_episode_ids`] for a whole
|
||||
/// feed refresh.
|
||||
pub fn resolve_episode_id(
|
||||
podcast: &PodcastId,
|
||||
channel_link: Option<&str>,
|
||||
item: &EpisodeIdentityHints,
|
||||
known: &[KnownEpisode],
|
||||
) -> Option<EpisodeId> {
|
||||
resolve_episode_ids(podcast, channel_link, std::slice::from_ref(item), known)
|
||||
.pop()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Resolves stable ids for the items of one feed document.
|
||||
///
|
||||
/// Returns one entry per item, in order; `None` marks an item that cannot be
|
||||
/// identified (no usable key) or that duplicates an earlier item of the same
|
||||
/// document. The rule (normative, see `PODCASTS.md`):
|
||||
///
|
||||
/// 1. An item with a GUID gets `EpisodeId::from_guid`. A GUID never matches
|
||||
/// a different known episode, so the id is independent of local history.
|
||||
/// 2. Items without a GUID then look for known episodes in three stages run
|
||||
/// over the whole document: same enclosure URL, then same `<link>`, then
|
||||
/// same publication second and [`normalize_episode_title`]. A stage
|
||||
/// assigns matches for all remaining items before the next stage starts;
|
||||
/// among candidates the smallest id is reused. A known episode already
|
||||
/// claimed (by a GUID item or an earlier stage) is not a candidate, so the
|
||||
/// result does not depend on item order.
|
||||
/// 3. Otherwise a new id is derived from the first available key: `<link>`,
|
||||
/// then `(date, title)`, then enclosure URL.
|
||||
///
|
||||
/// A `<link>` is only identifying when it is unique within the document and
|
||||
/// differs from the channel link; many feeds repeat the show homepage in every
|
||||
/// item. The same applies to enclosure URLs and `(date, title)` pairs shared
|
||||
/// by several GUID-less items.
|
||||
pub fn resolve_episode_ids(
|
||||
podcast: &PodcastId,
|
||||
channel_link: Option<&str>,
|
||||
items: &[EpisodeIdentityHints],
|
||||
known: &[KnownEpisode],
|
||||
) -> Vec<Option<EpisodeId>> {
|
||||
let channel_link = trimmed(channel_link);
|
||||
let mut link_counts: BTreeMap<&str, usize> = BTreeMap::new();
|
||||
let mut enclosure_counts: BTreeMap<&str, usize> = BTreeMap::new();
|
||||
let mut date_title_counts: BTreeMap<(i64, String), usize> = BTreeMap::new();
|
||||
for item in items {
|
||||
if let Some(link) = item.link() {
|
||||
*link_counts.entry(link).or_default() += 1;
|
||||
}
|
||||
if item.guid_key().is_none() {
|
||||
if let Some(url) = item.enclosure() {
|
||||
*enclosure_counts.entry(url).or_default() += 1;
|
||||
}
|
||||
if let Some(pair) = item.date_title() {
|
||||
*date_title_counts.entry(pair).or_default() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let usable: Vec<Usable<'_>> = items
|
||||
.iter()
|
||||
.map(|item| Usable {
|
||||
link: item
|
||||
.link()
|
||||
.filter(|link| Some(*link) != channel_link && link_counts.get(link) == Some(&1)),
|
||||
enclosure: item
|
||||
.enclosure()
|
||||
.filter(|url| enclosure_counts.get(url) == Some(&1)),
|
||||
date_title: item
|
||||
.date_title()
|
||||
.filter(|pair| date_title_counts.get(pair) == Some(&1)),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut claimed: BTreeSet<EpisodeId> = BTreeSet::new();
|
||||
let mut out: Vec<Option<EpisodeId>> = vec![None; items.len()];
|
||||
|
||||
// Pass 1: GUID items claim their derived ids first.
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if let Some(key) = item.guid_key() {
|
||||
let id = EpisodeId::derive(podcast, &key);
|
||||
if claimed.insert(id.clone()) {
|
||||
out[index] = Some(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2: GUID-less items reuse a matching known episode. Stages run
|
||||
// across the whole document: every enclosure match is assigned before any
|
||||
// link match, and every link match before any (date, title) match, so a
|
||||
// weak match of an earlier item cannot steal the known episode that a
|
||||
// later item matches strongly. Within one stage an item's key is unique in
|
||||
// the document, so two items never compete for the same known episode
|
||||
// and the result does not depend on item order.
|
||||
let mut pending: Vec<usize> = (0..items.len())
|
||||
.filter(|index| items[*index].guid_key().is_none())
|
||||
.collect();
|
||||
let stages: [StageMatcher<'_>; 3] = [
|
||||
&|own, candidate| {
|
||||
own.enclosure
|
||||
.is_some_and(|url| candidate.enclosure() == Some(url))
|
||||
},
|
||||
&|own, candidate| own.link.is_some_and(|link| candidate.link() == Some(link)),
|
||||
&|own, candidate| {
|
||||
own.date_title
|
||||
.as_ref()
|
||||
.is_some_and(|pair| candidate.date_title().as_ref() == Some(pair))
|
||||
},
|
||||
];
|
||||
for matches in stages {
|
||||
let mut unmatched = Vec::with_capacity(pending.len());
|
||||
for index in pending {
|
||||
let own = &usable[index];
|
||||
let reused = known
|
||||
.iter()
|
||||
.filter(|known| !claimed.contains(&known.episode_id) && matches(own, &known.hints))
|
||||
.map(|known| &known.episode_id)
|
||||
.min()
|
||||
.cloned();
|
||||
match reused {
|
||||
Some(id) => {
|
||||
claimed.insert(id.clone());
|
||||
out[index] = Some(id);
|
||||
}
|
||||
None => unmatched.push(index),
|
||||
}
|
||||
}
|
||||
pending = unmatched;
|
||||
}
|
||||
|
||||
// Pass 3: derive new ids from the first available fallback key.
|
||||
for index in pending {
|
||||
let own = &usable[index];
|
||||
let key = own
|
||||
.link
|
||||
.and_then(EpisodeKey::link)
|
||||
.or_else(|| {
|
||||
own.date_title
|
||||
.as_ref()
|
||||
.map(|(secs, title)| EpisodeKey::DateTitle {
|
||||
published_at_secs: *secs,
|
||||
normalized_title: title.clone(),
|
||||
})
|
||||
})
|
||||
.or_else(|| own.enclosure.and_then(EpisodeKey::enclosure));
|
||||
if let Some(key) = key {
|
||||
let id = EpisodeId::derive(podcast, &key);
|
||||
if claimed.insert(id.clone()) {
|
||||
out[index] = Some(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::podcast::FeedUrl;
|
||||
|
||||
fn podcast() -> PodcastId {
|
||||
PodcastId::from_feed_url(&FeedUrl::parse("https://example.test/feed").unwrap())
|
||||
}
|
||||
|
||||
fn hints(
|
||||
guid: Option<&str>,
|
||||
enclosure: Option<&str>,
|
||||
link: Option<&str>,
|
||||
date: Option<i64>,
|
||||
title: Option<&str>,
|
||||
) -> EpisodeIdentityHints {
|
||||
EpisodeIdentityHints {
|
||||
guid: guid.map(str::to_string),
|
||||
enclosure_url: enclosure.map(str::to_string),
|
||||
link: link.map(str::to_string),
|
||||
published_at_ms: date,
|
||||
title: title.map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
fn derived(key: EpisodeKey) -> Option<EpisodeId> {
|
||||
Some(EpisodeId::derive(&podcast(), &key))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guid_wins_and_ignores_history() {
|
||||
let item = hints(Some("g1"), Some("https://cdn/1.mp3"), None, None, Some("T"));
|
||||
let known = [KnownEpisode {
|
||||
episode_id: EpisodeId::from_guid(&podcast(), "other").unwrap(),
|
||||
hints: hints(None, Some("https://cdn/1.mp3"), None, None, None),
|
||||
}];
|
||||
assert_eq!(
|
||||
resolve_episode_id(&podcast(), None, &item, &known),
|
||||
derived(EpisodeKey::guid("g1").unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn title_cover_and_audio_url_changes_keep_the_guid_id() {
|
||||
let before = hints(
|
||||
Some("g1"),
|
||||
Some("https://cdn/a.mp3"),
|
||||
None,
|
||||
Some(1),
|
||||
Some("Old"),
|
||||
);
|
||||
let after = hints(
|
||||
Some("g1"),
|
||||
Some("https://cdn/b.mp3"),
|
||||
None,
|
||||
Some(2),
|
||||
Some("New"),
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_episode_id(&podcast(), None, &before, &[]),
|
||||
resolve_episode_id(&podcast(), None, &after, &[])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_key_order_for_new_items() {
|
||||
let p = podcast();
|
||||
let with_link = hints(
|
||||
None,
|
||||
Some("https://cdn/1.mp3"),
|
||||
Some("https://e/1"),
|
||||
Some(5_000),
|
||||
Some("A"),
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_episode_id(&p, None, &with_link, &[]),
|
||||
derived(EpisodeKey::link("https://e/1").unwrap())
|
||||
);
|
||||
let date_title = hints(
|
||||
None,
|
||||
Some("https://cdn/1.mp3"),
|
||||
None,
|
||||
Some(5_000),
|
||||
Some("A"),
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_episode_id(&p, None, &date_title, &[]),
|
||||
derived(EpisodeKey::date_title(5_000, "A").unwrap())
|
||||
);
|
||||
let enclosure_only = hints(None, Some("https://cdn/1.mp3"), None, None, Some("A"));
|
||||
assert_eq!(
|
||||
resolve_episode_id(&p, None, &enclosure_only, &[]),
|
||||
derived(EpisodeKey::enclosure("https://cdn/1.mp3").unwrap())
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_episode_id(&p, None, &EpisodeIdentityHints::default(), &[]),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guidless_episode_keeps_its_id_after_title_and_url_changes() {
|
||||
let p = podcast();
|
||||
// First seen: no link, so the id comes from (date, title).
|
||||
let first = hints(
|
||||
None,
|
||||
Some("https://cdn/a.mp3"),
|
||||
None,
|
||||
Some(9_000),
|
||||
Some("Pilot"),
|
||||
);
|
||||
let id = resolve_episode_id(&p, None, &first, &[]).unwrap();
|
||||
let known = [KnownEpisode {
|
||||
episode_id: id.clone(),
|
||||
hints: first.clone(),
|
||||
}];
|
||||
// Title changed: matched by enclosure.
|
||||
let retitled = hints(
|
||||
None,
|
||||
Some("https://cdn/a.mp3"),
|
||||
None,
|
||||
Some(9_000),
|
||||
Some("Pilot (remastered)"),
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_episode_id(&p, None, &retitled, &known),
|
||||
Some(id.clone())
|
||||
);
|
||||
// Audio URL changed: matched by (date, title).
|
||||
let moved = hints(
|
||||
None,
|
||||
Some("https://cdn2/a.mp3"),
|
||||
None,
|
||||
Some(9_999),
|
||||
Some("PILOT!"),
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_episode_id(&p, None, &moved, &known),
|
||||
Some(id.clone())
|
||||
);
|
||||
// Link appeared later: the enclosure still matches first.
|
||||
let linked = hints(
|
||||
None,
|
||||
Some("https://cdn/a.mp3"),
|
||||
Some("https://e/pilot"),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
assert_eq!(resolve_episode_id(&p, None, &linked, &known), Some(id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_channel_links_are_not_identifying() {
|
||||
let p = podcast();
|
||||
let home = "https://show.test/";
|
||||
let items = [
|
||||
hints(None, Some("https://cdn/1.mp3"), Some(home), None, None),
|
||||
hints(None, Some("https://cdn/2.mp3"), Some(home), None, None),
|
||||
];
|
||||
let ids = resolve_episode_ids(&p, Some(home), &items, &[]);
|
||||
assert_eq!(
|
||||
ids[0],
|
||||
derived(EpisodeKey::enclosure("https://cdn/1.mp3").unwrap())
|
||||
);
|
||||
assert_eq!(
|
||||
ids[1],
|
||||
derived(EpisodeKey::enclosure("https://cdn/2.mp3").unwrap())
|
||||
);
|
||||
// The same holds without the channel link: the link repeats.
|
||||
assert_eq!(resolve_episode_ids(&p, None, &items, &[]), ids);
|
||||
// A single item whose link equals the channel link.
|
||||
assert_eq!(resolve_episode_id(&p, Some(home), &items[0], &[]), ids[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicates_in_one_document_are_dropped() {
|
||||
let p = podcast();
|
||||
let items = [
|
||||
hints(Some("g"), Some("https://cdn/1.mp3"), None, None, None),
|
||||
hints(Some("g"), Some("https://cdn/2.mp3"), None, None, None),
|
||||
hints(None, Some("https://cdn/3.mp3"), None, None, None),
|
||||
hints(None, Some("https://cdn/3.mp3"), None, None, None),
|
||||
];
|
||||
let ids = resolve_episode_ids(&p, None, &items, &[]);
|
||||
assert!(ids[0].is_some());
|
||||
assert_eq!(ids[1], None);
|
||||
// The shared enclosure is not identifying: both GUID-less items have
|
||||
// no usable key.
|
||||
assert_eq!(ids[2], None);
|
||||
assert_eq!(ids[3], None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_known_episode_is_claimed_once() {
|
||||
let p = podcast();
|
||||
let known_id = derived(EpisodeKey::enclosure("https://cdn/old.mp3").unwrap()).unwrap();
|
||||
let known = [KnownEpisode {
|
||||
episode_id: known_id.clone(),
|
||||
hints: hints(
|
||||
None,
|
||||
Some("https://cdn/old.mp3"),
|
||||
None,
|
||||
Some(1_000),
|
||||
Some("Same"),
|
||||
),
|
||||
}];
|
||||
// Two new items both match by (date, title) only if unique; here
|
||||
// they share it, so neither matches; the enclosure one matches.
|
||||
let items = [
|
||||
hints(
|
||||
None,
|
||||
Some("https://cdn/x.mp3"),
|
||||
None,
|
||||
Some(1_000),
|
||||
Some("Same"),
|
||||
),
|
||||
hints(
|
||||
None,
|
||||
Some("https://cdn/old.mp3"),
|
||||
None,
|
||||
Some(1_000),
|
||||
Some("Same"),
|
||||
),
|
||||
];
|
||||
let ids = resolve_episode_ids(&p, None, &items, &known);
|
||||
assert_eq!(ids[1], Some(known_id.clone()));
|
||||
assert_eq!(
|
||||
ids[0],
|
||||
derived(EpisodeKey::enclosure("https://cdn/x.mp3").unwrap())
|
||||
);
|
||||
assert_ne!(ids[0], ids[1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguous_known_matches_pick_the_smallest_id_in_any_order() {
|
||||
let p = podcast();
|
||||
let a = EpisodeId::from_guid(&p, "a").unwrap();
|
||||
let b = EpisodeId::from_guid(&p, "b").unwrap();
|
||||
let same = hints(None, Some("https://cdn/1.mp3"), None, None, None);
|
||||
let k1 = KnownEpisode {
|
||||
episode_id: a.clone(),
|
||||
hints: same.clone(),
|
||||
};
|
||||
let k2 = KnownEpisode {
|
||||
episode_id: b.clone(),
|
||||
hints: same.clone(),
|
||||
};
|
||||
let expected = Some(a.clone().min(b.clone()));
|
||||
assert_eq!(
|
||||
resolve_episode_id(&p, None, &same, &[k1.clone(), k2.clone()]),
|
||||
expected
|
||||
);
|
||||
assert_eq!(resolve_episode_id(&p, None, &same, &[k2, k1]), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn result_is_independent_of_known_order() {
|
||||
let p = podcast();
|
||||
let items: Vec<EpisodeIdentityHints> = (0..6)
|
||||
.map(|i| {
|
||||
hints(
|
||||
None,
|
||||
Some(&format!("https://cdn/{i}.mp3")),
|
||||
Some(&format!("https://e/{i}")),
|
||||
Some(i * 1_000),
|
||||
Some(&format!("Ep {i}")),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let mut known: Vec<KnownEpisode> = items
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, h)| KnownEpisode {
|
||||
episode_id: EpisodeId::from_guid(&p, &format!("legacy-{i}")).unwrap(),
|
||||
hints: h.clone(),
|
||||
})
|
||||
.collect();
|
||||
let forward = resolve_episode_ids(&p, None, &items, &known);
|
||||
known.reverse();
|
||||
assert_eq!(resolve_episode_ids(&p, None, &items, &known), forward);
|
||||
assert!(forward.iter().all(Option::is_some));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strong_matches_beat_earlier_weak_matches_in_any_item_order() {
|
||||
let p = podcast();
|
||||
let k1 = EpisodeId::from_guid(&p, "legacy-k1").unwrap();
|
||||
let known = [KnownEpisode {
|
||||
episode_id: k1.clone(),
|
||||
hints: hints(
|
||||
None,
|
||||
Some("https://cdn/e1.mp3"),
|
||||
None,
|
||||
Some(5_000),
|
||||
Some("Same"),
|
||||
),
|
||||
}];
|
||||
// A new item that only matches K1 by (date, title), and the real K1
|
||||
// (same enclosure) renamed and re-dated by the publisher.
|
||||
let weak = hints(
|
||||
None,
|
||||
Some("https://cdn/new.mp3"),
|
||||
None,
|
||||
Some(5_000),
|
||||
Some("Same"),
|
||||
);
|
||||
let strong = hints(
|
||||
None,
|
||||
Some("https://cdn/e1.mp3"),
|
||||
None,
|
||||
Some(9_000),
|
||||
Some("Renamed"),
|
||||
);
|
||||
let forward = resolve_episode_ids(&p, None, &[weak.clone(), strong.clone()], &known);
|
||||
assert_eq!(forward[1].as_ref(), Some(&k1), "enclosure match keeps K1");
|
||||
assert_ne!(forward[0].as_ref(), Some(&k1));
|
||||
let reverse = resolve_episode_ids(&p, None, &[strong, weak], &known);
|
||||
assert_eq!(reverse, vec![forward[1].clone(), forward[0].clone()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn result_is_independent_of_item_order() {
|
||||
let p = podcast();
|
||||
let items: Vec<EpisodeIdentityHints> = (0..5)
|
||||
.map(|i| {
|
||||
hints(
|
||||
None,
|
||||
Some(&format!("https://cdn/{i}.mp3")),
|
||||
(i % 2 == 0).then(|| format!("https://e/{i}")).as_deref(),
|
||||
Some((i % 3) * 1_000),
|
||||
Some(&format!("Ep {}", i % 2)),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
// Known episodes whose hints overlap different items at different
|
||||
// stages.
|
||||
let known: Vec<KnownEpisode> = (0..5)
|
||||
.map(|i| KnownEpisode {
|
||||
episode_id: EpisodeId::from_guid(&p, &format!("legacy-{i}")).unwrap(),
|
||||
hints: hints(
|
||||
None,
|
||||
Some(&format!("https://cdn/{}.mp3", (i + 2) % 7)),
|
||||
Some(&format!("https://e/{}", (i + 1) % 5)),
|
||||
Some((i % 3) * 1_000),
|
||||
Some(&format!("Ep {}", i % 2)),
|
||||
),
|
||||
})
|
||||
.collect();
|
||||
let reference = resolve_episode_ids(&p, None, &items, &known);
|
||||
let mut order: Vec<usize> = (0..items.len()).collect();
|
||||
for _ in 0..20 {
|
||||
order.rotate_left(1);
|
||||
let middle = order.len() / 2;
|
||||
order.swap(0, middle);
|
||||
let shuffled: Vec<EpisodeIdentityHints> =
|
||||
order.iter().map(|i| items[*i].clone()).collect();
|
||||
let result = resolve_episode_ids(&p, None, &shuffled, &known);
|
||||
for (position, index) in order.iter().enumerate() {
|
||||
assert_eq!(result[position], reference[*index], "order {order:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,493 @@
|
||||
//! Podcast and episode metadata DTOs.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{
|
||||
EpisodeId, EpisodeIdentityHints, FeedUrl, MAX_DESCRIPTION_CHARS, MAX_GUID_BYTES,
|
||||
MAX_MIME_BYTES, MAX_TITLE_CHARS, MAX_URL_BYTES, PodcastError, PodcastId, PodcastResult,
|
||||
bounded_text, check_opt_text, check_opt_url, check_text, check_url,
|
||||
};
|
||||
|
||||
/// `itunes:episodeType`.
|
||||
///
|
||||
/// Unknown values deserialize (and parse) as [`EpisodeType::Full`], the
|
||||
/// iTunes default, so publishers cannot break the wire format.
|
||||
#[derive(
|
||||
Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize,
|
||||
)]
|
||||
#[serde(rename_all = "snake_case", from = "String")]
|
||||
pub enum EpisodeType {
|
||||
/// Regular episode.
|
||||
#[default]
|
||||
Full,
|
||||
/// Trailer or teaser.
|
||||
Trailer,
|
||||
/// Bonus content.
|
||||
Bonus,
|
||||
}
|
||||
|
||||
impl EpisodeType {
|
||||
/// Parses an `itunes:episodeType` value, case-insensitively.
|
||||
pub fn parse(value: &str) -> Self {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"trailer" => Self::Trailer,
|
||||
"bonus" => Self::Bonus,
|
||||
_ => Self::Full,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for EpisodeType {
|
||||
fn from(value: String) -> Self {
|
||||
Self::parse(&value)
|
||||
}
|
||||
}
|
||||
|
||||
/// Channel metadata of a podcast, as last fetched from its feed.
|
||||
///
|
||||
/// Synced inside subscription operations so that another trusted device can
|
||||
/// show and refresh the podcast without the user typing the URL again. It is
|
||||
/// private user data: never publish it outside the trusted-device group.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
pub struct PodcastMeta {
|
||||
/// Canonical URL the feed is currently fetched from.
|
||||
pub feed_url: FeedUrl,
|
||||
/// Podcast title.
|
||||
pub title: String,
|
||||
/// Author.
|
||||
#[serde(default)]
|
||||
pub author: Option<String>,
|
||||
/// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`].
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
/// Publisher web page.
|
||||
#[serde(default)]
|
||||
pub link: Option<String>,
|
||||
/// Artwork URL.
|
||||
#[serde(default)]
|
||||
pub image_url: Option<String>,
|
||||
}
|
||||
|
||||
impl PodcastMeta {
|
||||
/// Validates contract bounds.
|
||||
pub fn validate(&self) -> PodcastResult<()> {
|
||||
check_text("podcast title", &self.title, MAX_TITLE_CHARS, true)?;
|
||||
check_opt_text("podcast author", self.author.as_deref(), MAX_TITLE_CHARS)?;
|
||||
check_opt_text(
|
||||
"podcast description",
|
||||
self.description.as_deref(),
|
||||
MAX_DESCRIPTION_CHARS,
|
||||
)?;
|
||||
if self
|
||||
.link
|
||||
.as_ref()
|
||||
.is_some_and(|link| link.len() > MAX_URL_BYTES)
|
||||
{
|
||||
return Err(PodcastError::InvalidField {
|
||||
field: "podcast link",
|
||||
reason: "is too long",
|
||||
});
|
||||
}
|
||||
check_opt_url("podcast image_url", self.image_url.as_deref())
|
||||
}
|
||||
|
||||
/// Returns a copy that fits the contract bounds.
|
||||
///
|
||||
/// Text is truncated with [`bounded_text`]; invalid optional URLs are
|
||||
/// dropped; an empty title becomes `"Untitled podcast"`.
|
||||
pub fn bounded(&self) -> Self {
|
||||
Self {
|
||||
feed_url: self.feed_url.clone(),
|
||||
title: bounded_title(&self.title, "Untitled podcast"),
|
||||
author: bounded_opt(self.author.as_deref(), MAX_TITLE_CHARS),
|
||||
description: bounded_opt(self.description.as_deref(), MAX_DESCRIPTION_CHARS),
|
||||
link: self
|
||||
.link
|
||||
.clone()
|
||||
.filter(|link| link.len() <= MAX_URL_BYTES && !link.trim().is_empty()),
|
||||
image_url: valid_url(self.image_url.as_deref()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Full episode metadata, as stored by clients and refreshed from RSS.
|
||||
///
|
||||
/// Publisher metadata never touches personal progress or history: those are
|
||||
/// separate records keyed by [`EpisodeId`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct EpisodeMeta {
|
||||
/// Stable episode id.
|
||||
pub episode_id: EpisodeId,
|
||||
/// Owning podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// RSS GUID / Atom id, when present.
|
||||
#[serde(default)]
|
||||
pub guid: Option<String>,
|
||||
/// Episode title.
|
||||
pub title: String,
|
||||
/// Plain-text description, at most [`MAX_DESCRIPTION_CHARS`].
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
/// `itunes:season`. Episodes without a season form their own group.
|
||||
#[serde(default)]
|
||||
pub season: Option<u32>,
|
||||
/// `itunes:episode`.
|
||||
#[serde(default)]
|
||||
pub episode_number: Option<u32>,
|
||||
/// `itunes:episodeType`.
|
||||
#[serde(default)]
|
||||
pub episode_type: EpisodeType,
|
||||
/// Publication time in unix milliseconds.
|
||||
#[serde(default)]
|
||||
pub published_at_ms: Option<i64>,
|
||||
/// Duration announced by the publisher, in milliseconds.
|
||||
#[serde(default)]
|
||||
pub duration_ms: Option<i64>,
|
||||
/// Absolute http(s) audio URL at the publisher.
|
||||
pub audio_url: String,
|
||||
/// Enclosure MIME type.
|
||||
#[serde(default)]
|
||||
pub mime: Option<String>,
|
||||
/// Enclosure length in bytes.
|
||||
#[serde(default)]
|
||||
pub byte_len: Option<u64>,
|
||||
/// Episode artwork URL.
|
||||
#[serde(default)]
|
||||
pub image_url: Option<String>,
|
||||
/// Episode web page.
|
||||
#[serde(default)]
|
||||
pub link: Option<String>,
|
||||
}
|
||||
|
||||
impl EpisodeMeta {
|
||||
/// Validates contract bounds.
|
||||
pub fn validate(&self) -> PodcastResult<()> {
|
||||
if self
|
||||
.guid
|
||||
.as_ref()
|
||||
.is_some_and(|guid| guid.len() > MAX_GUID_BYTES)
|
||||
{
|
||||
return Err(PodcastError::InvalidField {
|
||||
field: "episode guid",
|
||||
reason: "is too long",
|
||||
});
|
||||
}
|
||||
check_text("episode title", &self.title, MAX_TITLE_CHARS, true)?;
|
||||
check_opt_text(
|
||||
"episode description",
|
||||
self.description.as_deref(),
|
||||
MAX_DESCRIPTION_CHARS,
|
||||
)?;
|
||||
check_duration(self.duration_ms)?;
|
||||
check_url("episode audio_url", &self.audio_url)?;
|
||||
check_mime(self.mime.as_deref())?;
|
||||
check_opt_url("episode image_url", self.image_url.as_deref())?;
|
||||
if self
|
||||
.link
|
||||
.as_ref()
|
||||
.is_some_and(|link| link.len() > MAX_URL_BYTES)
|
||||
{
|
||||
return Err(PodcastError::InvalidField {
|
||||
field: "episode link",
|
||||
reason: "is too long",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Identity hints to keep for the no-GUID matching rule.
|
||||
pub fn identity_hints(&self) -> EpisodeIdentityHints {
|
||||
EpisodeIdentityHints {
|
||||
guid: self.guid.clone(),
|
||||
enclosure_url: Some(self.audio_url.clone()),
|
||||
link: self.link.clone(),
|
||||
published_at_ms: self.published_at_ms,
|
||||
title: Some(self.title.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the self-contained queue reference for this episode.
|
||||
///
|
||||
/// The episode artwork falls back to the podcast artwork.
|
||||
pub fn to_ref(&self, podcast: &PodcastMeta) -> PodcastEpisodeRef {
|
||||
PodcastEpisodeRef {
|
||||
podcast_id: self.podcast_id.clone(),
|
||||
episode_id: self.episode_id.clone(),
|
||||
feed_url: podcast.feed_url.clone(),
|
||||
podcast_title: bounded_title(&podcast.title, "Untitled podcast"),
|
||||
episode_title: bounded_title(&self.title, "Untitled episode"),
|
||||
audio_url: self.audio_url.clone(),
|
||||
mime: self
|
||||
.mime
|
||||
.clone()
|
||||
.filter(|mime| check_mime(Some(mime)).is_ok()),
|
||||
duration_ms: self.duration_ms.filter(|ms| *ms > 0),
|
||||
published_at_ms: self.published_at_ms,
|
||||
season: self.season,
|
||||
episode_number: self.episode_number,
|
||||
image_url: valid_url(self.image_url.as_deref())
|
||||
.or_else(|| valid_url(podcast.image_url.as_deref())),
|
||||
progress_generation: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Self-contained, bounded reference to a podcast episode.
|
||||
///
|
||||
/// Carried by playback queues (`PlaybackSource::PodcastEpisode`) and by
|
||||
/// progress operations, so a receiving trusted device can display, resume and
|
||||
/// play the episode without being subscribed and without its own copy of the
|
||||
/// feed. Apart from the optional handoff hint
|
||||
/// [`progress_generation`](Self::progress_generation) it contains no personal
|
||||
/// state. It is private: Jam snapshots replace it with an opaque placeholder.
|
||||
///
|
||||
/// If the audio URL went stale, refresh the feed at `feed_url` and look the
|
||||
/// episode up by `episode_id`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastEpisodeRef {
|
||||
/// Owning podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Episode.
|
||||
pub episode_id: EpisodeId,
|
||||
/// Canonical feed URL used to refresh the episode.
|
||||
pub feed_url: FeedUrl,
|
||||
/// Podcast title for display.
|
||||
pub podcast_title: String,
|
||||
/// Episode title for display.
|
||||
pub episode_title: String,
|
||||
/// Absolute http(s) audio URL at the publisher.
|
||||
pub audio_url: String,
|
||||
/// Enclosure MIME type.
|
||||
#[serde(default)]
|
||||
pub mime: Option<String>,
|
||||
/// Announced duration in milliseconds.
|
||||
#[serde(default)]
|
||||
pub duration_ms: Option<i64>,
|
||||
/// Publication time in unix milliseconds.
|
||||
#[serde(default)]
|
||||
pub published_at_ms: Option<i64>,
|
||||
/// `itunes:season`.
|
||||
#[serde(default)]
|
||||
pub season: Option<u32>,
|
||||
/// `itunes:episode`.
|
||||
#[serde(default)]
|
||||
pub episode_number: Option<u32>,
|
||||
/// Episode or podcast artwork URL.
|
||||
#[serde(default)]
|
||||
pub image_url: Option<String>,
|
||||
/// Greatest progress generation the sender has seen for this episode.
|
||||
///
|
||||
/// Set on queue items of a playback state (in particular on handoff),
|
||||
/// because the playback state can arrive before the sender's progress
|
||||
/// operations. A device that starts playing the item writes its explicit
|
||||
/// action with
|
||||
/// [`ProgressRecord::explicit_generation`](super::ProgressRecord::explicit_generation)`(max(local, progress_generation))`.
|
||||
/// Ignored in progress and listening operations, whose own generation is
|
||||
/// authoritative. Not part of the episode identity.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub progress_generation: Option<u64>,
|
||||
}
|
||||
|
||||
impl PodcastEpisodeRef {
|
||||
/// Validates contract bounds. Receivers must drop invalid references.
|
||||
pub fn validate(&self) -> PodcastResult<()> {
|
||||
check_text("podcast title", &self.podcast_title, MAX_TITLE_CHARS, true)?;
|
||||
check_text("episode title", &self.episode_title, MAX_TITLE_CHARS, true)?;
|
||||
check_url("episode audio_url", &self.audio_url)?;
|
||||
check_mime(self.mime.as_deref())?;
|
||||
check_duration(self.duration_ms)?;
|
||||
check_opt_url("episode image_url", self.image_url.as_deref())
|
||||
}
|
||||
|
||||
/// Identity hints known from this reference.
|
||||
///
|
||||
/// Add them to the known-episode set before resolving ids from a feed so
|
||||
/// a GUID-less episode learned through sync keeps its id.
|
||||
pub fn identity_hints(&self) -> EpisodeIdentityHints {
|
||||
EpisodeIdentityHints {
|
||||
guid: None,
|
||||
enclosure_url: Some(self.audio_url.clone()),
|
||||
link: None,
|
||||
published_at_ms: self.published_at_ms,
|
||||
title: Some(self.episode_title.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check_duration(duration_ms: Option<i64>) -> PodcastResult<()> {
|
||||
if duration_ms.is_some_and(|ms| ms < 0) {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field: "duration_ms",
|
||||
reason: "must not be negative",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_mime(mime: Option<&str>) -> PodcastResult<()> {
|
||||
let Some(mime) = mime else { return Ok(()) };
|
||||
if mime.is_empty() || mime.len() > MAX_MIME_BYTES || !mime.bytes().all(|b| b.is_ascii_graphic())
|
||||
{
|
||||
return Err(PodcastError::InvalidField {
|
||||
field: "mime",
|
||||
reason: "must be a short printable ASCII token",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bounded_title(title: &str, fallback: &str) -> String {
|
||||
let title = title.trim();
|
||||
if title.is_empty() {
|
||||
fallback.to_string()
|
||||
} else {
|
||||
bounded_text(title, MAX_TITLE_CHARS)
|
||||
}
|
||||
}
|
||||
|
||||
fn bounded_opt(value: Option<&str>, max_chars: usize) -> Option<String> {
|
||||
value
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(|value| bounded_text(value, max_chars))
|
||||
}
|
||||
|
||||
fn valid_url(value: Option<&str>) -> Option<String> {
|
||||
value
|
||||
.filter(|value| super::is_http_url(value))
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn podcast_meta() -> PodcastMeta {
|
||||
PodcastMeta {
|
||||
feed_url: FeedUrl::parse("https://example.test/feed").unwrap(),
|
||||
title: "Show".to_string(),
|
||||
author: Some("Host".to_string()),
|
||||
description: None,
|
||||
link: Some("https://example.test/".to_string()),
|
||||
image_url: Some("https://example.test/cover.jpg".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn episode() -> EpisodeMeta {
|
||||
let podcast_id = PodcastId::from_feed_url(&podcast_meta().feed_url);
|
||||
EpisodeMeta {
|
||||
episode_id: EpisodeId::from_guid(&podcast_id, "g1").unwrap(),
|
||||
podcast_id,
|
||||
guid: Some("g1".to_string()),
|
||||
title: "Pilot".to_string(),
|
||||
description: Some("Notes".to_string()),
|
||||
season: Some(2),
|
||||
episode_number: Some(7),
|
||||
episode_type: EpisodeType::Bonus,
|
||||
published_at_ms: Some(1_700_000_000_000),
|
||||
duration_ms: Some(3_600_000),
|
||||
audio_url: "https://cdn.example.test/1.mp3?sig=abc".to_string(),
|
||||
mime: Some("audio/mpeg".to_string()),
|
||||
byte_len: Some(1234),
|
||||
image_url: None,
|
||||
link: Some("https://example.test/1".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn episode_type_is_tolerant() {
|
||||
assert_eq!(EpisodeType::parse(" Trailer "), EpisodeType::Trailer);
|
||||
assert_eq!(EpisodeType::parse("BONUS"), EpisodeType::Bonus);
|
||||
assert_eq!(EpisodeType::parse("full"), EpisodeType::Full);
|
||||
assert_eq!(EpisodeType::parse("teaser"), EpisodeType::Full);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&EpisodeType::Trailer).unwrap(),
|
||||
"\"trailer\""
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<EpisodeType>("\"something-new\"").unwrap(),
|
||||
EpisodeType::Full
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_round_trips_and_validates() {
|
||||
let episode = episode();
|
||||
episode.validate().unwrap();
|
||||
let json = serde_json::to_string(&episode).unwrap();
|
||||
assert_eq!(serde_json::from_str::<EpisodeMeta>(&json).unwrap(), episode);
|
||||
podcast_meta().validate().unwrap();
|
||||
|
||||
let mut bad = episode.clone();
|
||||
bad.audio_url = "file:///tmp/x.mp3".to_string();
|
||||
assert!(bad.validate().is_err());
|
||||
let mut bad = episode.clone();
|
||||
bad.description = Some("x".repeat(MAX_DESCRIPTION_CHARS + 1));
|
||||
assert!(bad.validate().is_err());
|
||||
let mut bad = episode;
|
||||
bad.mime = Some("audio/mpeg; codecs=\"mp3\"".to_string());
|
||||
assert!(bad.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimal_legacy_json_uses_defaults() {
|
||||
let episode = episode();
|
||||
let json = format!(
|
||||
r#"{{"episode_id":"{}","podcast_id":"{}","title":"T","audio_url":"https://a.test/1.mp3"}}"#,
|
||||
episode.episode_id, episode.podcast_id
|
||||
);
|
||||
let parsed: EpisodeMeta = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed.episode_type, EpisodeType::Full);
|
||||
assert_eq!(parsed.season, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn episode_ref_is_self_contained_and_bounded() {
|
||||
let podcast = podcast_meta();
|
||||
let mut episode = episode();
|
||||
episode.title = "x".repeat(MAX_TITLE_CHARS + 10);
|
||||
let reference = episode.to_ref(&podcast);
|
||||
reference.validate().unwrap();
|
||||
assert_eq!(reference.episode_title.chars().count(), MAX_TITLE_CHARS);
|
||||
assert_eq!(reference.podcast_title, "Show");
|
||||
assert_eq!(
|
||||
reference.image_url.as_deref(),
|
||||
Some("https://example.test/cover.jpg")
|
||||
);
|
||||
assert_eq!(reference.feed_url, podcast.feed_url);
|
||||
let json = serde_json::to_string(&reference).unwrap();
|
||||
assert!(!json.contains("description"));
|
||||
assert_eq!(
|
||||
serde_json::from_str::<PodcastEpisodeRef>(&json).unwrap(),
|
||||
reference
|
||||
);
|
||||
|
||||
let mut bad = reference.clone();
|
||||
bad.episode_title = " ".to_string();
|
||||
assert!(bad.validate().is_err());
|
||||
let mut bad = reference.clone();
|
||||
bad.audio_url = "javascript:alert(1)".to_string();
|
||||
assert!(bad.validate().is_err());
|
||||
let mut bad = reference;
|
||||
bad.duration_ms = Some(-1);
|
||||
assert!(bad.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refs_reject_invalid_ids_on_deserialization() {
|
||||
let json = r#"{"podcast_id":"pod1:bad","episode_id":"ep1:bad","feed_url":"https://a.test/","podcast_title":"P","episode_title":"E","audio_url":"https://a.test/1.mp3"}"#;
|
||||
assert!(serde_json::from_str::<PodcastEpisodeRef>(json).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_podcast_meta_fits_the_contract() {
|
||||
let mut meta = podcast_meta();
|
||||
meta.title = " ".to_string();
|
||||
meta.description = Some("d".repeat(MAX_DESCRIPTION_CHARS * 2));
|
||||
meta.image_url = Some("ftp://x".to_string());
|
||||
let bounded = meta.bounded();
|
||||
bounded.validate().unwrap();
|
||||
assert_eq!(bounded.title, "Untitled podcast");
|
||||
assert_eq!(bounded.image_url, None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
//! Shared podcast contract for Furumi clients.
|
||||
//!
|
||||
//! This module is pure: it has no SQLite, HTTP, clock, audio or UI
|
||||
//! dependency. Every Furumi client (TUI, web, desktop) uses it so that
|
||||
//! podcast identity, normalization, merge rules and wire DTOs are defined
|
||||
//! exactly once. The normative description lives in the workspace
|
||||
//! `PODCASTS.md`; this documentation summarizes the API.
|
||||
//!
|
||||
//! # Identity
|
||||
//!
|
||||
//! * [`FeedUrl`] is a canonical feed URL. [`FeedUrl::parse`] lowercases the
|
||||
//! scheme and host, drops a default port and the fragment, and keeps the
|
||||
//! path and **every** query parameter byte-for-byte (only non-ASCII bytes
|
||||
//! are percent-encoded). Non-`http(s)` URLs, userinfo and whitespace are
|
||||
//! rejected. [`FeedUrl::from_user_input`] additionally accepts the
|
||||
//! `feed:`/`podcast:`/`pcast:`/`itpc:` forms and a missing scheme.
|
||||
//! * [`PodcastId`] is `pod1:` followed by the lowercase base32 BLAKE3 hash of
|
||||
//! `"furumi-podcast:podcast:v1\0"` and the canonical feed URL. It is
|
||||
//! derived once, from the URL of the first subscription; later URLs
|
||||
//! (redirects, `itunes:new-feed-url`) become feed aliases resolved by
|
||||
//! [`resolve_podcast_id`].
|
||||
//! * [`EpisodeId`] is `ep1:` followed by the lowercase base32 BLAKE3 hash of
|
||||
//! `"furumi-podcast:episode:v1\0"`, the podcast id and a key. The key is
|
||||
//! the RSS GUID when present. Without a GUID, [`resolve_episode_ids`] first
|
||||
//! reuses the id of a known episode that matches by enclosure URL, `<link>`
|
||||
//! or (date, normalized title) and only then derives a new id from the
|
||||
//! first available fallback key.
|
||||
//!
|
||||
//! # Personal state
|
||||
//!
|
||||
//! * [`SubscriptionRecord`]: last-writer-wins by `(generation, hlc_ms, op_id)`
|
||||
//! where `generation` is a per-podcast Lamport counter
|
||||
//! ([`next_podcast_generation`]); unsubscribing is state, not a tombstone.
|
||||
//! Feed aliases merge as a bounded grow-only set.
|
||||
//! * [`ShareRecord`]: last-writer-wins with the same ordering; `shared = false`
|
||||
//! is register state that is never compacted. See [`effective_share`] for
|
||||
//! the rule that a (re)subscription is private.
|
||||
//! * [`ProgressRecord`]: register ordered by
|
||||
//! `(generation, hlc_ms, device_id, op_id)`. Explicit user actions bump the
|
||||
//! generation to `max_seen + 1`; periodic checkpoints keep the generation of
|
||||
//! their session, so a late checkpoint of an old session always loses.
|
||||
//! * [`ListenSession`]: grow-only set keyed by `session_id`; one completion per
|
||||
//! session, so re-delivery never double counts.
|
||||
//!
|
||||
//! # Queue items
|
||||
//!
|
||||
//! [`PodcastEpisodeRef`] is the bounded, self-contained reference carried by
|
||||
//! playback queues and progress operations. A receiving device can show and
|
||||
//! play the episode without being subscribed to the podcast.
|
||||
//!
|
||||
//! # Feed parser
|
||||
//!
|
||||
//! With the `podcast-feed` cargo feature, `feed` provides the shared tolerant
|
||||
//! RSS 2.0 / RSS 1.0 / Atom parser that produces the identity hints consumed
|
||||
//! by [`resolve_episode_ids`].
|
||||
//!
|
||||
//! ```
|
||||
//! use music_dht::podcast::{EpisodeId, FeedUrl, PodcastId};
|
||||
//!
|
||||
//! let url = FeedUrl::parse("HTTPS://Example.COM:443/feed.xml?token=Ab#top").unwrap();
|
||||
//! assert_eq!(url.as_str(), "https://example.com/feed.xml?token=Ab");
|
||||
//! let podcast = PodcastId::from_feed_url(&url);
|
||||
//! assert!(podcast.as_str().starts_with("pod1:"));
|
||||
//! let episode = EpisodeId::from_guid(&podcast, "episode-42").unwrap();
|
||||
//! assert!(episode.as_str().starts_with("ep1:"));
|
||||
//! ```
|
||||
|
||||
mod feed_url;
|
||||
mod id;
|
||||
mod identity;
|
||||
mod meta;
|
||||
mod state;
|
||||
|
||||
#[cfg(feature = "podcast-feed")]
|
||||
pub mod feed;
|
||||
|
||||
pub use feed_url::{FeedUrl, merge_feed_aliases, resolve_podcast_id};
|
||||
pub use id::{
|
||||
EPISODE_ID_DOMAIN, EPISODE_ID_PREFIX, EpisodeId, EpisodeKey, PODCAST_ID_DOMAIN,
|
||||
PODCAST_ID_PREFIX, PodcastId,
|
||||
};
|
||||
pub use identity::{
|
||||
EpisodeIdentityHints, KnownEpisode, normalize_episode_title, resolve_episode_id,
|
||||
resolve_episode_ids,
|
||||
};
|
||||
pub use meta::{EpisodeMeta, EpisodeType, PodcastEpisodeRef, PodcastMeta};
|
||||
pub use state::{
|
||||
COMPLETION_TAIL_MAX_MS, ListenSession, MIN_RECORDED_EPISODE_LISTEN_MS, ProgressRecord,
|
||||
ProgressStatus, ShareRecord, SubscriptionRecord, completed_listen_count, effective_share,
|
||||
merge_listen_session, merge_progress, merge_share, merge_subscription, next_podcast_generation,
|
||||
reaches_completion,
|
||||
};
|
||||
|
||||
/// Maximum characters of a title (podcast, episode, author).
|
||||
pub const MAX_TITLE_CHARS: usize = 1_000;
|
||||
/// Maximum characters of a podcast or episode description.
|
||||
pub const MAX_DESCRIPTION_CHARS: usize = 4_000;
|
||||
/// Maximum bytes of any URL carried by the contract.
|
||||
pub const MAX_URL_BYTES: usize = 2_048;
|
||||
/// Maximum bytes of an RSS GUID. Longer GUIDs are treated as absent.
|
||||
pub const MAX_GUID_BYTES: usize = 2_048;
|
||||
/// Maximum bytes of a MIME type.
|
||||
pub const MAX_MIME_BYTES: usize = 128;
|
||||
/// Maximum bytes of a device, session or operation id.
|
||||
pub const MAX_TOKEN_BYTES: usize = 256;
|
||||
/// Maximum feed aliases kept per podcast, besides the URL the id was derived from.
|
||||
pub const MAX_FEED_ALIASES: usize = 8;
|
||||
|
||||
/// Maximum subscription records in one sync snapshot.
|
||||
pub const MAX_SNAPSHOT_SUBSCRIPTIONS: usize = 1_000;
|
||||
/// Maximum share records in one sync snapshot.
|
||||
pub const MAX_SNAPSHOT_SHARES: usize = 1_000;
|
||||
/// Maximum progress registers in one sync snapshot.
|
||||
pub const MAX_SNAPSHOT_PROGRESS: usize = 5_000;
|
||||
/// Maximum listening sessions in one sync snapshot.
|
||||
pub const MAX_SNAPSHOT_LISTEN_SESSIONS: usize = 2_000;
|
||||
/// Serialized JSON budget shared by all podcast snapshot sections.
|
||||
///
|
||||
/// It leaves room for music state inside the 8 MiB sync line limit.
|
||||
pub const MAX_PODCAST_SNAPSHOT_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Errors produced by podcast contract validation and parsing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[non_exhaustive]
|
||||
pub enum PodcastError {
|
||||
/// A feed URL cannot be canonicalized.
|
||||
#[error("invalid feed URL: {0}")]
|
||||
InvalidFeedUrl(String),
|
||||
/// A string is not a valid podcast or episode id.
|
||||
#[error("invalid {kind} id")]
|
||||
InvalidId {
|
||||
/// `podcast` or `episode`.
|
||||
kind: &'static str,
|
||||
},
|
||||
/// A field violates the contract (missing, too long, malformed).
|
||||
#[error("invalid {field}: {reason}")]
|
||||
InvalidField {
|
||||
/// Field name.
|
||||
field: &'static str,
|
||||
/// Human-readable reason.
|
||||
reason: &'static str,
|
||||
},
|
||||
/// Two records that must describe the same entity do not.
|
||||
#[error("cannot merge records of different {0}")]
|
||||
Mismatch(&'static str),
|
||||
/// The document is not an RSS or Atom feed.
|
||||
#[error("not an RSS or Atom feed{0}")]
|
||||
NotAFeed(String),
|
||||
/// The feed is malformed and nothing usable could be parsed.
|
||||
#[error("malformed feed: {0}")]
|
||||
MalformedFeed(String),
|
||||
}
|
||||
|
||||
/// Result alias for this module.
|
||||
pub type PodcastResult<T> = std::result::Result<T, PodcastError>;
|
||||
|
||||
/// Truncates `text` to at most `max_chars` characters, ending with `…` when cut.
|
||||
///
|
||||
/// Clients use it to fit publisher metadata into the contract bounds.
|
||||
pub fn bounded_text(text: &str, max_chars: usize) -> String {
|
||||
if text.chars().count() <= max_chars {
|
||||
return text.to_string();
|
||||
}
|
||||
if max_chars == 0 {
|
||||
return String::new();
|
||||
}
|
||||
let mut out: String = text.chars().take(max_chars - 1).collect();
|
||||
out.push('…');
|
||||
out
|
||||
}
|
||||
|
||||
pub(crate) fn check_text(
|
||||
field: &'static str,
|
||||
value: &str,
|
||||
max_chars: usize,
|
||||
required: bool,
|
||||
) -> PodcastResult<()> {
|
||||
if required && value.trim().is_empty() {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field,
|
||||
reason: "must not be empty",
|
||||
});
|
||||
}
|
||||
if value.len() > max_chars * 4 || value.chars().count() > max_chars {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field,
|
||||
reason: "is too long",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn check_opt_text(
|
||||
field: &'static str,
|
||||
value: Option<&str>,
|
||||
max_chars: usize,
|
||||
) -> PodcastResult<()> {
|
||||
value.map_or(Ok(()), |value| check_text(field, value, max_chars, false))
|
||||
}
|
||||
|
||||
pub(crate) fn check_token(field: &'static str, value: &str) -> PodcastResult<()> {
|
||||
if value.is_empty() {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field,
|
||||
reason: "must not be empty",
|
||||
});
|
||||
}
|
||||
if value.len() > MAX_TOKEN_BYTES {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field,
|
||||
reason: "is too long",
|
||||
});
|
||||
}
|
||||
if value.chars().any(|c| c.is_control()) {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field,
|
||||
reason: "contains control characters",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns whether `value` is an absolute `http`/`https` URL with a host,
|
||||
/// without whitespace or control characters and within [`MAX_URL_BYTES`].
|
||||
///
|
||||
/// Media URLs keep the publisher's spelling; only feed URLs are canonicalized.
|
||||
pub fn is_http_url(value: &str) -> bool {
|
||||
if value.is_empty()
|
||||
|| value.len() > MAX_URL_BYTES
|
||||
|| value.chars().any(|c| c.is_whitespace() || c.is_control())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let lower_prefix = |prefix: &str| {
|
||||
value.len() >= prefix.len() && value[..prefix.len()].eq_ignore_ascii_case(prefix)
|
||||
};
|
||||
let rest = if lower_prefix("http://") {
|
||||
&value[7..]
|
||||
} else if lower_prefix("https://") {
|
||||
&value[8..]
|
||||
} else {
|
||||
return false;
|
||||
};
|
||||
let authority = rest.split(['/', '?', '#']).next().unwrap_or_default();
|
||||
let host = authority.rsplit('@').next().unwrap_or_default();
|
||||
!host.is_empty() && !host.starts_with(':')
|
||||
}
|
||||
|
||||
pub(crate) fn check_url(field: &'static str, value: &str) -> PodcastResult<()> {
|
||||
if is_http_url(value) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(PodcastError::InvalidField {
|
||||
field,
|
||||
reason: "must be an absolute http(s) URL",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn check_opt_url(field: &'static str, value: Option<&str>) -> PodcastResult<()> {
|
||||
value.map_or(Ok(()), |value| check_url(field, value))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn bounded_text_cuts_on_char_boundaries() {
|
||||
assert_eq!(bounded_text("hello", 5), "hello");
|
||||
assert_eq!(bounded_text("hello!", 5), "hell…");
|
||||
assert_eq!(bounded_text("привет", 3), "пр…");
|
||||
assert_eq!(bounded_text("x", 0), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_limits_accept_the_limit_and_reject_one_more() {
|
||||
let at = "я".repeat(MAX_TITLE_CHARS);
|
||||
let over = "я".repeat(MAX_TITLE_CHARS + 1);
|
||||
assert!(check_text("title", &at, MAX_TITLE_CHARS, true).is_ok());
|
||||
assert!(check_text("title", &over, MAX_TITLE_CHARS, true).is_err());
|
||||
assert!(check_text("title", " ", MAX_TITLE_CHARS, true).is_err());
|
||||
assert!(check_text("title", "", MAX_TITLE_CHARS, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_limits() {
|
||||
assert!(check_token("op_id", &"a".repeat(MAX_TOKEN_BYTES)).is_ok());
|
||||
assert!(check_token("op_id", &"a".repeat(MAX_TOKEN_BYTES + 1)).is_err());
|
||||
assert!(check_token("op_id", "").is_err());
|
||||
assert!(check_token("op_id", "a\nb").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_urls() {
|
||||
assert!(is_http_url("https://cdn.example.test/a.mp3?x=1"));
|
||||
assert!(is_http_url("HTTP://cdn.example.test"));
|
||||
assert!(!is_http_url("ftp://cdn.example.test/a.mp3"));
|
||||
assert!(!is_http_url("file:///etc/passwd"));
|
||||
assert!(!is_http_url("https://"));
|
||||
assert!(!is_http_url("https:///path"));
|
||||
assert!(!is_http_url("https://a.test/my file.mp3"));
|
||||
assert!(!is_http_url(""));
|
||||
let long = format!("https://a.test/{}", "x".repeat(MAX_URL_BYTES));
|
||||
assert!(!is_http_url(&long));
|
||||
let at = format!("https://a.test/{}", "x".repeat(MAX_URL_BYTES - 15));
|
||||
assert_eq!(at.len(), MAX_URL_BYTES);
|
||||
assert!(is_http_url(&at));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,526 @@
|
||||
//! Personal podcast state and its convergent merge rules.
|
||||
//!
|
||||
//! Every merge function here is commutative, associative and idempotent over
|
||||
//! records of the same entity, so trusted devices that receive the same set of
|
||||
//! operations converge to the same state regardless of delivery order and
|
||||
//! duplication. All ties are broken by a total order over every field.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{
|
||||
EpisodeId, FeedUrl, MAX_FEED_ALIASES, PodcastError, PodcastId, PodcastMeta, PodcastResult,
|
||||
check_token, merge_feed_aliases,
|
||||
};
|
||||
|
||||
/// Largest tail of an episode that still counts as finished, in milliseconds.
|
||||
///
|
||||
/// See [`reaches_completion`].
|
||||
pub const COMPLETION_TAIL_MAX_MS: i64 = 30_000;
|
||||
|
||||
/// Minimum actual listening time for a non-completed session to enter history.
|
||||
pub const MIN_RECORDED_EPISODE_LISTEN_MS: i64 = 30_000;
|
||||
|
||||
/// Shared automatic completion criterion for every client.
|
||||
///
|
||||
/// A position completes an episode when it is within the smaller of 30 s and
|
||||
/// 5 % of the duration from the end. Natural end of stream always completes,
|
||||
/// whatever the announced duration. Without a known positive duration only the
|
||||
/// end of stream completes.
|
||||
pub fn reaches_completion(position_ms: i64, duration_ms: Option<i64>) -> bool {
|
||||
let Some(duration) = duration_ms.filter(|duration| *duration > 0) else {
|
||||
return false;
|
||||
};
|
||||
let tail = COMPLETION_TAIL_MAX_MS.min(duration / 20);
|
||||
position_ms >= duration - tail
|
||||
}
|
||||
|
||||
/// Listening status of one episode.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ProgressStatus {
|
||||
/// Never started, or explicitly marked unplayed.
|
||||
NotStarted,
|
||||
/// Started and not finished.
|
||||
InProgress,
|
||||
/// Finished, automatically or by an explicit "mark played".
|
||||
Completed,
|
||||
}
|
||||
|
||||
/// Durable resume register of one episode.
|
||||
///
|
||||
/// The winner among records of the same episode is the greatest
|
||||
/// `(generation, hlc_ms, device_id, op_id)`; remaining fields only break ties
|
||||
/// between malformed duplicates. Progress is **never** the maximum position:
|
||||
/// seeking backwards and "start over" are real changes.
|
||||
///
|
||||
/// * Explicit user actions (start of a new listening session, seek, "start
|
||||
/// over", "mark played", "mark unplayed", taking over playback through a
|
||||
/// handoff) use [`ProgressRecord::explicit_generation`]: one more than the
|
||||
/// greatest generation the device has seen for the episode. This is a
|
||||
/// Lamport clock, so wall-clock skew cannot undo an explicit action that was
|
||||
/// observed. An explicit action during a session becomes the generation of
|
||||
/// that session.
|
||||
/// * Periodic checkpoints keep the generation of the latest explicit action of
|
||||
/// their session. A late checkpoint of an older session therefore loses
|
||||
/// against any newer explicit action, whatever its `hlc_ms`.
|
||||
/// * Concurrent explicit actions on offline devices receive the same
|
||||
/// generation and are ordered by `hlc_ms`, then `device_id`, then `op_id`.
|
||||
/// * Handoff: only the receiving device writes the explicit action, with
|
||||
/// `explicit_generation(max(local, carried))` where `carried` is
|
||||
/// [`PodcastEpisodeRef::progress_generation`](super::PodcastEpisodeRef::progress_generation)
|
||||
/// of the queue item. The sender writes a final checkpoint at its session
|
||||
/// generation.
|
||||
/// * A device that observes a greater generation for the episode it is
|
||||
/// playing stops writing at its old generation: either playback moved away,
|
||||
/// or the user keeps listening locally and the device writes its next
|
||||
/// record as an explicit action.
|
||||
/// * Outgoing-log coalescing may drop a pending progress operation only when
|
||||
/// another pending operation of the same episode [supersedes](Self::supersedes)
|
||||
/// it; dropping by sequence order could discard an explicit action.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ProgressRecord {
|
||||
/// Episode the register belongs to.
|
||||
pub episode_id: EpisodeId,
|
||||
/// Owning podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Resume position in milliseconds.
|
||||
pub position_ms: i64,
|
||||
/// Listening status.
|
||||
pub status: ProgressStatus,
|
||||
/// Lamport generation of the latest explicit action (see type docs).
|
||||
pub generation: u64,
|
||||
/// Listening session (or explicit action) that wrote the record.
|
||||
pub session_id: String,
|
||||
/// Logical write time, normally the sync operation `hlc_ms`.
|
||||
pub hlc_ms: i64,
|
||||
/// Writing device, normally the sync operation origin.
|
||||
pub device_id: String,
|
||||
/// Writing operation id.
|
||||
pub op_id: String,
|
||||
}
|
||||
|
||||
impl ProgressRecord {
|
||||
/// Generation for an explicit user action, given the greatest generation
|
||||
/// already seen for the episode (`None` when none was seen).
|
||||
pub fn explicit_generation(max_seen: Option<u64>) -> u64 {
|
||||
max_seen.map_or(1, |generation| generation.saturating_add(1))
|
||||
}
|
||||
|
||||
/// Primary ordering key.
|
||||
pub fn order_key(&self) -> (u64, i64, &str, &str) {
|
||||
(
|
||||
self.generation,
|
||||
self.hlc_ms,
|
||||
self.device_id.as_str(),
|
||||
self.op_id.as_str(),
|
||||
)
|
||||
}
|
||||
|
||||
fn total_cmp(&self, other: &Self) -> Ordering {
|
||||
self.order_key()
|
||||
.cmp(&other.order_key())
|
||||
.then_with(|| self.position_ms.cmp(&other.position_ms))
|
||||
.then_with(|| self.status.cmp(&other.status))
|
||||
.then_with(|| self.session_id.cmp(&other.session_id))
|
||||
.then_with(|| self.podcast_id.cmp(&other.podcast_id))
|
||||
}
|
||||
|
||||
/// Returns whether this record wins over `other` (same episode).
|
||||
pub fn supersedes(&self, other: &Self) -> bool {
|
||||
self.total_cmp(other) == Ordering::Greater
|
||||
}
|
||||
|
||||
/// Position from which ordinary Play resumes.
|
||||
///
|
||||
/// Completed and not-started episodes start from the beginning; use the
|
||||
/// explicit "start over" action to restart an in-progress episode.
|
||||
pub fn resume_position_ms(&self) -> i64 {
|
||||
match self.status {
|
||||
ProgressStatus::InProgress => self.position_ms,
|
||||
ProgressStatus::NotStarted | ProgressStatus::Completed => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates contract bounds.
|
||||
pub fn validate(&self) -> PodcastResult<()> {
|
||||
if self.position_ms < 0 {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field: "position_ms",
|
||||
reason: "must not be negative",
|
||||
});
|
||||
}
|
||||
check_token("session_id", &self.session_id)?;
|
||||
check_token("device_id", &self.device_id)?;
|
||||
check_token("op_id", &self.op_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges two progress registers of the same episode.
|
||||
pub fn merge_progress(
|
||||
current: &ProgressRecord,
|
||||
incoming: &ProgressRecord,
|
||||
) -> PodcastResult<ProgressRecord> {
|
||||
if current.episode_id != incoming.episode_id {
|
||||
return Err(PodcastError::Mismatch("episodes"));
|
||||
}
|
||||
Ok(if incoming.supersedes(current) {
|
||||
incoming.clone()
|
||||
} else {
|
||||
current.clone()
|
||||
})
|
||||
}
|
||||
|
||||
/// Personal subscription state of one podcast.
|
||||
///
|
||||
/// `subscribed` and `podcast` form one last-writer-wins register ordered by
|
||||
/// `(generation, hlc_ms, op_id)`. `generation` is a Lamport counter shared by
|
||||
/// the subscription and share registers of the podcast (see
|
||||
/// [`next_podcast_generation`]), so an action taken after observing another
|
||||
/// device's write wins whatever the device clocks say. Unsubscribing writes
|
||||
/// `subscribed = false`: it is state, not a tombstone, is never compacted, and
|
||||
/// never deletes progress or history, so subscribing again continues where
|
||||
/// the user stopped.
|
||||
///
|
||||
/// Only explicit user actions (subscribe, unsubscribe) write this register.
|
||||
/// Background RSS refreshes keep their metadata locally and must not emit a
|
||||
/// subscription operation; otherwise a stale device could resurrect an
|
||||
/// unsubscribed podcast. Feed aliases learned by refreshes travel in their
|
||||
/// own grow-only operation.
|
||||
///
|
||||
/// `feed_aliases` is a bounded grow-only set merged independently of the
|
||||
/// register (see [`merge_feed_aliases`]): the URL the id was derived from
|
||||
/// plus the [`MAX_FEED_ALIASES`] smallest other URLs. The current
|
||||
/// `podcast.feed_url` may fall outside that bound, so look podcasts up by
|
||||
/// [`SubscriptionRecord::feed_urls`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SubscriptionRecord {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Whether the user is subscribed.
|
||||
pub subscribed: bool,
|
||||
/// Channel metadata at the time of the write.
|
||||
pub podcast: PodcastMeta,
|
||||
/// Known feed URLs of the podcast.
|
||||
#[serde(default)]
|
||||
pub feed_aliases: Vec<FeedUrl>,
|
||||
/// Lamport generation shared with the share register (see type docs).
|
||||
#[serde(default)]
|
||||
pub generation: u64,
|
||||
/// Logical write time.
|
||||
pub hlc_ms: i64,
|
||||
/// Writing operation id.
|
||||
pub op_id: String,
|
||||
}
|
||||
|
||||
impl SubscriptionRecord {
|
||||
/// Register ordering key: `(generation, hlc_ms, op_id)`.
|
||||
pub fn order_key(&self) -> (u64, i64, &str) {
|
||||
(self.generation, self.hlc_ms, self.op_id.as_str())
|
||||
}
|
||||
|
||||
fn total_cmp(&self, other: &Self) -> Ordering {
|
||||
self.order_key()
|
||||
.cmp(&other.order_key())
|
||||
.then_with(|| self.subscribed.cmp(&other.subscribed))
|
||||
.then_with(|| self.podcast.cmp(&other.podcast))
|
||||
}
|
||||
|
||||
/// Returns whether this record's register wins over `other`'s.
|
||||
pub fn supersedes(&self, other: &Self) -> bool {
|
||||
self.total_cmp(other) == Ordering::Greater
|
||||
}
|
||||
|
||||
/// Merges `podcast.feed_url` into `feed_aliases` and bounds the set.
|
||||
///
|
||||
/// The result is sorted and bounded by [`merge_feed_aliases`]; it may not
|
||||
/// contain `podcast.feed_url` when many smaller aliases are known.
|
||||
pub fn normalize(&mut self) {
|
||||
self.feed_aliases = merge_feed_aliases(
|
||||
&self.podcast_id,
|
||||
&self.feed_aliases,
|
||||
std::slice::from_ref(&self.podcast.feed_url),
|
||||
);
|
||||
}
|
||||
|
||||
/// Adds aliases learned through a `podcast_feed_aliases_added` operation.
|
||||
///
|
||||
/// Only the grow-only alias set changes; the register is untouched.
|
||||
pub fn add_feed_aliases(&mut self, aliases: &[FeedUrl]) {
|
||||
self.feed_aliases = merge_feed_aliases(&self.podcast_id, &self.feed_aliases, aliases);
|
||||
}
|
||||
|
||||
/// Every URL that identifies this podcast: the aliases and the current
|
||||
/// `podcast.feed_url`, sorted and without duplicates.
|
||||
///
|
||||
/// Stores pass these pairs to [`super::resolve_podcast_id`].
|
||||
pub fn feed_urls(&self) -> Vec<FeedUrl> {
|
||||
let mut urls: BTreeSet<FeedUrl> = self.feed_aliases.iter().cloned().collect();
|
||||
urls.insert(self.podcast.feed_url.clone());
|
||||
urls.into_iter().collect()
|
||||
}
|
||||
|
||||
/// Validates contract bounds.
|
||||
pub fn validate(&self) -> PodcastResult<()> {
|
||||
self.podcast.validate()?;
|
||||
if self.feed_aliases.len() > MAX_FEED_ALIASES + 1 {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field: "feed_aliases",
|
||||
reason: "has too many entries",
|
||||
});
|
||||
}
|
||||
check_token("op_id", &self.op_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges two subscription records of the same podcast.
|
||||
///
|
||||
/// The result is normalized (see [`SubscriptionRecord::normalize`]).
|
||||
pub fn merge_subscription(
|
||||
current: &SubscriptionRecord,
|
||||
incoming: &SubscriptionRecord,
|
||||
) -> PodcastResult<SubscriptionRecord> {
|
||||
if current.podcast_id != incoming.podcast_id {
|
||||
return Err(PodcastError::Mismatch("podcasts"));
|
||||
}
|
||||
let mut merged = if incoming.supersedes(current) {
|
||||
incoming.clone()
|
||||
} else {
|
||||
current.clone()
|
||||
};
|
||||
let mut left = current.feed_aliases.clone();
|
||||
left.push(current.podcast.feed_url.clone());
|
||||
let mut right = incoming.feed_aliases.clone();
|
||||
right.push(incoming.podcast.feed_url.clone());
|
||||
merged.feed_aliases = merge_feed_aliases(&merged.podcast_id, &left, &right);
|
||||
Ok(merged)
|
||||
}
|
||||
|
||||
/// Decision to publish a podcast card to the federation.
|
||||
///
|
||||
/// Last-writer-wins by `(generation, hlc_ms, op_id)`, with `generation` the
|
||||
/// per-podcast Lamport counter shared with [`SubscriptionRecord`] (see
|
||||
/// [`next_podcast_generation`]). `shared = false` revokes the publication.
|
||||
/// It is privacy-protecting register state, **not** a tombstone: like an
|
||||
/// unsubscribe it is never compacted, otherwise a stale `shared = true`
|
||||
/// delivered again would republish the podcast. Unsubscribing must also write
|
||||
/// `shared = false`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ShareRecord {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Whether the user chose to share the podcast.
|
||||
pub shared: bool,
|
||||
/// Lamport generation shared with the subscription register.
|
||||
#[serde(default)]
|
||||
pub generation: u64,
|
||||
/// Logical write time.
|
||||
pub hlc_ms: i64,
|
||||
/// Writing operation id.
|
||||
pub op_id: String,
|
||||
}
|
||||
|
||||
impl ShareRecord {
|
||||
/// Register ordering key: `(generation, hlc_ms, op_id)`.
|
||||
pub fn order_key(&self) -> (u64, i64, &str) {
|
||||
(self.generation, self.hlc_ms, self.op_id.as_str())
|
||||
}
|
||||
|
||||
fn total_cmp(&self, other: &Self) -> Ordering {
|
||||
self.order_key()
|
||||
.cmp(&other.order_key())
|
||||
.then_with(|| self.shared.cmp(&other.shared))
|
||||
}
|
||||
|
||||
/// Returns whether this record wins over `other`.
|
||||
pub fn supersedes(&self, other: &Self) -> bool {
|
||||
self.total_cmp(other) == Ordering::Greater
|
||||
}
|
||||
|
||||
/// Validates contract bounds.
|
||||
pub fn validate(&self) -> PodcastResult<()> {
|
||||
check_token("op_id", &self.op_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges two share records of the same podcast.
|
||||
pub fn merge_share(current: &ShareRecord, incoming: &ShareRecord) -> PodcastResult<ShareRecord> {
|
||||
if current.podcast_id != incoming.podcast_id {
|
||||
return Err(PodcastError::Mismatch("podcasts"));
|
||||
}
|
||||
Ok(if incoming.supersedes(current) {
|
||||
incoming.clone()
|
||||
} else {
|
||||
current.clone()
|
||||
})
|
||||
}
|
||||
|
||||
/// Generation for a new write of the subscription or share register.
|
||||
///
|
||||
/// One more than the greatest generation of either register of the podcast
|
||||
/// (`None` when it has none), so a write made after observing another
|
||||
/// device's subscription, unsubscription or share decision orders after it
|
||||
/// even when that device's clock runs ahead. Concurrent writes on devices
|
||||
/// that did not see each other receive the same generation and are ordered
|
||||
/// by `hlc_ms`, then `op_id`.
|
||||
pub fn next_podcast_generation(
|
||||
subscription: Option<&SubscriptionRecord>,
|
||||
share: Option<&ShareRecord>,
|
||||
) -> u64 {
|
||||
subscription
|
||||
.map(|record| record.generation)
|
||||
.into_iter()
|
||||
.chain(share.map(|record| record.generation))
|
||||
.max()
|
||||
.map_or(1, |generation| generation.saturating_add(1))
|
||||
}
|
||||
|
||||
/// Whether a podcast must currently be published.
|
||||
///
|
||||
/// True only when the user is subscribed, chose to share, and the share
|
||||
/// decision orders after the latest subscription write (by
|
||||
/// `(generation, hlc_ms, op_id)`). A new subscription (including
|
||||
/// re-subscribing, or subscribing from someone else's public card) is
|
||||
/// therefore always private until the user shares it again, and a stale
|
||||
/// device cannot republish a podcast through an old share decision. A share
|
||||
/// decision taken after observing the subscription always applies, because it
|
||||
/// carries a greater generation.
|
||||
pub fn effective_share(
|
||||
subscription: Option<&SubscriptionRecord>,
|
||||
share: Option<&ShareRecord>,
|
||||
) -> bool {
|
||||
match (subscription, share) {
|
||||
(Some(subscription), Some(share)) => {
|
||||
subscription.subscribed
|
||||
&& share.shared
|
||||
&& subscription.podcast_id == share.podcast_id
|
||||
&& share.order_key() > subscription.order_key()
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// One listening session of an episode on one device.
|
||||
///
|
||||
/// Sessions form a grow-only set keyed by `session_id`, generated when
|
||||
/// playback of the episode starts on a device. A device may re-send its
|
||||
/// session with more listening time (for example after a crash); versions of
|
||||
/// one session merge by [`merge_listen_session`].
|
||||
///
|
||||
/// * A session enters history when it completed or lasted at least
|
||||
/// [`MIN_RECORDED_EPISODE_LISTEN_MS`] ([`ListenSession::should_record`]).
|
||||
/// * `completed` is set when the session reached [`reaches_completion`] or the
|
||||
/// natural end of the stream. Each session counts at most once, so the play
|
||||
/// count of an episode is the number of distinct completed sessions
|
||||
/// ([`completed_listen_count`]); duplicate delivery never double counts.
|
||||
/// * "Mark played" changes [`ProgressRecord::status`] only. It does not create
|
||||
/// a session and does not invent listening time.
|
||||
/// * Sessions listened concurrently on offline devices are all kept.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ListenSession {
|
||||
/// Unique session id, normally a UUID.
|
||||
pub session_id: String,
|
||||
/// Episode.
|
||||
pub episode_id: EpisodeId,
|
||||
/// Owning podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Device that played the session.
|
||||
pub device_id: String,
|
||||
/// Session start in unix milliseconds.
|
||||
pub started_at_ms: i64,
|
||||
/// Latest update of the session in unix milliseconds.
|
||||
pub ended_at_ms: i64,
|
||||
/// Position when the session started.
|
||||
pub start_position_ms: i64,
|
||||
/// Position when the session ended (or was last reported).
|
||||
pub end_position_ms: i64,
|
||||
/// Actual accumulated listening time, excluding pauses and seeks.
|
||||
pub listened_ms: i64,
|
||||
/// Whether the session completed the episode.
|
||||
pub completed: bool,
|
||||
}
|
||||
|
||||
impl ListenSession {
|
||||
fn total_cmp(&self, other: &Self) -> Ordering {
|
||||
(
|
||||
self.completed,
|
||||
self.listened_ms,
|
||||
self.ended_at_ms,
|
||||
self.end_position_ms,
|
||||
self.started_at_ms,
|
||||
self.start_position_ms,
|
||||
)
|
||||
.cmp(&(
|
||||
other.completed,
|
||||
other.listened_ms,
|
||||
other.ended_at_ms,
|
||||
other.end_position_ms,
|
||||
other.started_at_ms,
|
||||
other.start_position_ms,
|
||||
))
|
||||
}
|
||||
|
||||
/// Whether the session belongs in listening history.
|
||||
pub fn should_record(&self) -> bool {
|
||||
self.completed || self.listened_ms >= MIN_RECORDED_EPISODE_LISTEN_MS
|
||||
}
|
||||
|
||||
/// Validates contract bounds.
|
||||
pub fn validate(&self) -> PodcastResult<()> {
|
||||
check_token("session_id", &self.session_id)?;
|
||||
check_token("device_id", &self.device_id)?;
|
||||
if self.started_at_ms < 0 || self.ended_at_ms < self.started_at_ms {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field: "ended_at_ms",
|
||||
reason: "must not precede started_at_ms",
|
||||
});
|
||||
}
|
||||
if self.start_position_ms < 0 || self.end_position_ms < 0 || self.listened_ms < 0 {
|
||||
return Err(PodcastError::InvalidField {
|
||||
field: "listened_ms",
|
||||
reason: "positions and listening time must not be negative",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges two versions of the same listening session.
|
||||
///
|
||||
/// The version with the greatest `(completed, listened_ms, ended_at_ms, ...)`
|
||||
/// wins, so a session can only grow.
|
||||
pub fn merge_listen_session(
|
||||
current: &ListenSession,
|
||||
incoming: &ListenSession,
|
||||
) -> PodcastResult<ListenSession> {
|
||||
if current.session_id != incoming.session_id {
|
||||
return Err(PodcastError::Mismatch("sessions"));
|
||||
}
|
||||
if current.episode_id != incoming.episode_id
|
||||
|| current.podcast_id != incoming.podcast_id
|
||||
|| current.device_id != incoming.device_id
|
||||
{
|
||||
return Err(PodcastError::Mismatch("session identities"));
|
||||
}
|
||||
Ok(if incoming.total_cmp(current) == Ordering::Greater {
|
||||
incoming.clone()
|
||||
} else {
|
||||
current.clone()
|
||||
})
|
||||
}
|
||||
|
||||
/// Number of distinct sessions that completed an episode.
|
||||
pub fn completed_listen_count<'a>(sessions: impl IntoIterator<Item = &'a ListenSession>) -> usize {
|
||||
sessions
|
||||
.into_iter()
|
||||
.filter(|session| session.completed)
|
||||
.map(|session| session.session_id.as_str())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "state_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,699 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::*;
|
||||
use crate::podcast::{EpisodeIdentityHints, EpisodeMeta, EpisodeType, resolve_episode_ids};
|
||||
|
||||
fn permutations<T: Clone>(items: &[T]) -> Vec<Vec<T>> {
|
||||
if items.len() <= 1 {
|
||||
return vec![items.to_vec()];
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for index in 0..items.len() {
|
||||
let mut rest = items.to_vec();
|
||||
let head = rest.remove(index);
|
||||
for mut tail in permutations(&rest) {
|
||||
tail.insert(0, head.clone());
|
||||
out.push(tail);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn feed_url(path: &str) -> FeedUrl {
|
||||
FeedUrl::parse(&format!("https://example.test/{path}")).unwrap()
|
||||
}
|
||||
|
||||
fn podcast_id() -> PodcastId {
|
||||
PodcastId::from_feed_url(&feed_url("feed"))
|
||||
}
|
||||
|
||||
fn episode_id(guid: &str) -> EpisodeId {
|
||||
EpisodeId::from_guid(&podcast_id(), guid).unwrap()
|
||||
}
|
||||
|
||||
fn progress(
|
||||
generation: u64,
|
||||
hlc_ms: i64,
|
||||
device: &str,
|
||||
op: &str,
|
||||
position_ms: i64,
|
||||
status: ProgressStatus,
|
||||
session: &str,
|
||||
) -> ProgressRecord {
|
||||
ProgressRecord {
|
||||
episode_id: episode_id("e1"),
|
||||
podcast_id: podcast_id(),
|
||||
position_ms,
|
||||
status,
|
||||
generation,
|
||||
session_id: session.to_string(),
|
||||
hlc_ms,
|
||||
device_id: device.to_string(),
|
||||
op_id: op.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn fold<T: Clone>(items: &[T], merge: impl Fn(&T, &T) -> PodcastResult<T>) -> T {
|
||||
let mut iter = items.iter();
|
||||
let first = iter.next().expect("non-empty").clone();
|
||||
iter.fold(first, |acc, item| merge(&acc, item).unwrap())
|
||||
}
|
||||
|
||||
fn assert_converges<T: Clone + PartialEq + std::fmt::Debug>(
|
||||
items: &[T],
|
||||
merge: impl Fn(&T, &T) -> PodcastResult<T> + Copy,
|
||||
) -> T {
|
||||
let reference = fold(items, merge);
|
||||
for order in permutations(items) {
|
||||
assert_eq!(fold(&order, merge), reference, "order: {order:?}");
|
||||
}
|
||||
// Merge laws on every pair and triple.
|
||||
for a in items {
|
||||
assert_eq!(
|
||||
merge(&merge(a, a).unwrap(), a).unwrap(),
|
||||
merge(a, a).unwrap()
|
||||
);
|
||||
for b in items {
|
||||
assert_eq!(merge(a, b).unwrap(), merge(b, a).unwrap());
|
||||
for c in items {
|
||||
assert_eq!(
|
||||
merge(&merge(a, b).unwrap(), c).unwrap(),
|
||||
merge(a, &merge(b, c).unwrap()).unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
reference
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_converges_for_every_permutation() {
|
||||
use ProgressStatus::*;
|
||||
let ops = [
|
||||
progress(1, 100, "dev-a", "dev-a:1", 10_000, InProgress, "s1"),
|
||||
progress(1, 160, "dev-a", "dev-a:2", 70_000, InProgress, "s1"),
|
||||
progress(2, 150, "dev-b", "dev-b:1", 0, InProgress, "s2"),
|
||||
progress(2, 150, "dev-c", "dev-c:1", 5_000, InProgress, "s3"),
|
||||
progress(2, 140, "dev-b", "dev-b:2", 0, Completed, "s2"),
|
||||
// Duplicate delivery of the same operation.
|
||||
progress(1, 160, "dev-a", "dev-a:2", 70_000, InProgress, "s1"),
|
||||
];
|
||||
let winner = assert_converges(&ops, merge_progress);
|
||||
assert_eq!(winner.op_id, "dev-c:1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn late_old_generation_checkpoint_loses() {
|
||||
use ProgressStatus::*;
|
||||
// Device A keeps checkpointing session s1 (generation 3) with a clock far
|
||||
// ahead; device B explicitly seeks back (generation 4) with a slow clock.
|
||||
let seek_back = progress(4, 1_000, "dev-b", "dev-b:9", 60_000, InProgress, "s2");
|
||||
let late = progress(
|
||||
3, 9_000_000, "dev-a", "dev-a:50", 3_000_000, InProgress, "s1",
|
||||
);
|
||||
for order in [[&seek_back, &late], [&late, &seek_back]] {
|
||||
let merged = merge_progress(order[0], order[1]).unwrap();
|
||||
assert_eq!(merged, seek_back);
|
||||
}
|
||||
assert!(seek_back.supersedes(&late));
|
||||
assert!(!late.supersedes(&seek_back));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restart_beats_later_checkpoint_of_the_old_session() {
|
||||
use ProgressStatus::*;
|
||||
let checkpoint = progress(2, 100, "dev-a", "dev-a:1", 2_400_000, InProgress, "s1");
|
||||
let generation = ProgressRecord::explicit_generation(Some(checkpoint.generation));
|
||||
assert_eq!(generation, 3);
|
||||
let restart = progress(generation, 110, "dev-a", "dev-a:2", 0, InProgress, "s2");
|
||||
// A checkpoint of s1 that was queued before the restart and delivered
|
||||
// later, even stamped after the restart.
|
||||
let late = progress(2, 120, "dev-a", "dev-a:3", 2_460_000, InProgress, "s1");
|
||||
let merged = assert_converges(&[checkpoint, restart.clone(), late], merge_progress);
|
||||
assert_eq!(merged, restart);
|
||||
assert_eq!(merged.resume_position_ms(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_is_not_the_maximum_position() {
|
||||
use ProgressStatus::*;
|
||||
let ahead = progress(1, 100, "dev-a", "dev-a:1", 3_000_000, InProgress, "s1");
|
||||
let back = progress(2, 101, "dev-a", "dev-a:2", 60_000, InProgress, "s1");
|
||||
assert_eq!(merge_progress(&ahead, &back).unwrap().position_ms, 60_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_offline_sessions_converge_and_keep_both_sessions() {
|
||||
use ProgressStatus::*;
|
||||
// Both devices last saw generation 5 and start listening offline.
|
||||
let generation = ProgressRecord::explicit_generation(Some(5));
|
||||
let a_start = progress(
|
||||
generation, 200, "dev-a", "dev-a:1", 300_000, InProgress, "sa",
|
||||
);
|
||||
let a_cp = progress(
|
||||
generation, 300, "dev-a", "dev-a:2", 900_000, InProgress, "sa",
|
||||
);
|
||||
let b_start = progress(
|
||||
generation, 210, "dev-b", "dev-b:1", 300_000, InProgress, "sb",
|
||||
);
|
||||
let b_cp = progress(
|
||||
generation, 280, "dev-b", "dev-b:2", 1_500_000, InProgress, "sb",
|
||||
);
|
||||
let merged = assert_converges(&[a_start, a_cp.clone(), b_start, b_cp], merge_progress);
|
||||
// Deterministic: the latest write of the shared generation wins.
|
||||
assert_eq!(merged, a_cp);
|
||||
|
||||
let session = |id: &str, device: &str, listened: i64, completed: bool| ListenSession {
|
||||
session_id: id.to_string(),
|
||||
episode_id: episode_id("e1"),
|
||||
podcast_id: podcast_id(),
|
||||
device_id: device.to_string(),
|
||||
started_at_ms: 200,
|
||||
ended_at_ms: 300,
|
||||
start_position_ms: 300_000,
|
||||
end_position_ms: 900_000,
|
||||
listened_ms: listened,
|
||||
completed,
|
||||
};
|
||||
let sessions = [
|
||||
session("sa", "dev-a", 600_000, false),
|
||||
session("sb", "dev-b", 1_200_000, true),
|
||||
session("sb", "dev-b", 1_200_000, true),
|
||||
];
|
||||
let mut set: BTreeMap<String, ListenSession> = BTreeMap::new();
|
||||
for order in permutations(&sessions) {
|
||||
set.clear();
|
||||
for item in order {
|
||||
let merged = match set.get(&item.session_id) {
|
||||
Some(current) => merge_listen_session(current, &item).unwrap(),
|
||||
None => item.clone(),
|
||||
};
|
||||
set.insert(item.session_id.clone(), merged);
|
||||
}
|
||||
assert_eq!(set.len(), 2);
|
||||
assert_eq!(completed_listen_count(set.values()), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_session_versions_only_grow() {
|
||||
let base = ListenSession {
|
||||
session_id: "s".to_string(),
|
||||
episode_id: episode_id("e1"),
|
||||
podcast_id: podcast_id(),
|
||||
device_id: "dev-a".to_string(),
|
||||
started_at_ms: 1_000,
|
||||
ended_at_ms: 2_000,
|
||||
start_position_ms: 0,
|
||||
end_position_ms: 60_000,
|
||||
listened_ms: 60_000,
|
||||
completed: false,
|
||||
};
|
||||
let longer = ListenSession {
|
||||
ended_at_ms: 3_000,
|
||||
end_position_ms: 120_000,
|
||||
listened_ms: 120_000,
|
||||
..base.clone()
|
||||
};
|
||||
let finished = ListenSession {
|
||||
ended_at_ms: 3_500,
|
||||
end_position_ms: 3_600_000,
|
||||
listened_ms: 125_000,
|
||||
completed: true,
|
||||
..base.clone()
|
||||
};
|
||||
let merged = assert_converges(
|
||||
&[base.clone(), longer, finished.clone(), base.clone()],
|
||||
merge_listen_session,
|
||||
);
|
||||
assert_eq!(merged, finished);
|
||||
assert!(merged.should_record());
|
||||
assert_eq!(completed_listen_count([&merged, &merged, &finished]), 1);
|
||||
|
||||
let mut other_episode = base.clone();
|
||||
other_episode.episode_id = episode_id("e2");
|
||||
assert!(merge_listen_session(&base, &other_episode).is_err());
|
||||
let short = ListenSession {
|
||||
listened_ms: MIN_RECORDED_EPISODE_LISTEN_MS - 1,
|
||||
..base.clone()
|
||||
};
|
||||
assert!(!short.should_record());
|
||||
assert!(
|
||||
ListenSession {
|
||||
listened_ms: MIN_RECORDED_EPISODE_LISTEN_MS,
|
||||
..base
|
||||
}
|
||||
.should_record()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_criterion() {
|
||||
// 60 min episode: tail is 30 s.
|
||||
assert!(!reaches_completion(3_569_999, Some(3_600_000)));
|
||||
assert!(reaches_completion(3_570_000, Some(3_600_000)));
|
||||
// 5 min episode: tail is 5 % = 15 s.
|
||||
assert!(!reaches_completion(284_999, Some(300_000)));
|
||||
assert!(reaches_completion(285_000, Some(300_000)));
|
||||
assert!(!reaches_completion(1_000_000, None));
|
||||
assert!(!reaches_completion(1_000_000, Some(0)));
|
||||
}
|
||||
|
||||
fn podcast_meta(title: &str, url: &str) -> PodcastMeta {
|
||||
PodcastMeta {
|
||||
feed_url: feed_url(url),
|
||||
title: title.to_string(),
|
||||
author: None,
|
||||
description: None,
|
||||
link: None,
|
||||
image_url: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn subscription(
|
||||
subscribed: bool,
|
||||
hlc_ms: i64,
|
||||
op: &str,
|
||||
title: &str,
|
||||
url: &str,
|
||||
) -> SubscriptionRecord {
|
||||
let mut record = SubscriptionRecord {
|
||||
podcast_id: podcast_id(),
|
||||
subscribed,
|
||||
podcast: podcast_meta(title, url),
|
||||
feed_aliases: Vec::new(),
|
||||
generation: 1,
|
||||
hlc_ms,
|
||||
op_id: op.to_string(),
|
||||
};
|
||||
record.normalize();
|
||||
record
|
||||
}
|
||||
|
||||
fn with_generation<T: Clone>(record: &T, set: impl Fn(&mut T)) -> T {
|
||||
let mut record = record.clone();
|
||||
set(&mut record);
|
||||
record
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subscriptions_converge_for_every_permutation() {
|
||||
let ops = [
|
||||
// Same podcast added on two devices.
|
||||
subscription(true, 100, "dev-a:1", "Show", "feed"),
|
||||
subscription(true, 101, "dev-b:1", "Show (B)", "feed"),
|
||||
// Unsubscribed on A while B was offline.
|
||||
subscription(false, 200, "dev-a:2", "Show", "feed"),
|
||||
// B comes back and re-delivers its old state after the feed moved.
|
||||
subscription(true, 150, "dev-b:2", "Show", "moved"),
|
||||
subscription(true, 101, "dev-b:1", "Show (B)", "feed"),
|
||||
];
|
||||
let merged = assert_converges(&ops, merge_subscription);
|
||||
assert!(
|
||||
!merged.subscribed,
|
||||
"unsubscribe survives the offline device"
|
||||
);
|
||||
assert_eq!(merged.op_id, "dev-a:2");
|
||||
assert!(merged.feed_aliases.contains(&feed_url("moved")));
|
||||
assert!(merged.feed_aliases.contains(&feed_url("feed")));
|
||||
merged.validate().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resubscribing_keeps_progress_and_history() {
|
||||
use ProgressStatus::*;
|
||||
let mut progress_register = BTreeMap::new();
|
||||
let saved = progress(4, 500, "dev-a", "dev-a:5", 1_234_000, InProgress, "s1");
|
||||
progress_register.insert(saved.episode_id.clone(), saved.clone());
|
||||
let subscribed = subscription(true, 100, "dev-a:1", "Show", "feed");
|
||||
let unsubscribed = subscription(false, 600, "dev-a:6", "Show", "feed");
|
||||
let again = subscription(true, 700, "dev-b:1", "Show", "feed");
|
||||
let merged = fold(&[subscribed, unsubscribed, again], merge_subscription);
|
||||
assert!(merged.subscribed);
|
||||
assert_eq!(progress_register.get(&saved.episode_id), Some(&saved));
|
||||
}
|
||||
|
||||
fn share(shared: bool, hlc_ms: i64, op: &str) -> ShareRecord {
|
||||
ShareRecord {
|
||||
podcast_id: podcast_id(),
|
||||
shared,
|
||||
generation: 1,
|
||||
hlc_ms,
|
||||
op_id: op.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shares_converge_and_false_is_never_beaten_by_a_stale_true() {
|
||||
let ops = [
|
||||
share(true, 100, "dev-a:1"),
|
||||
share(false, 200, "dev-a:2"),
|
||||
share(true, 150, "dev-b:1"),
|
||||
share(false, 200, "dev-a:2"),
|
||||
];
|
||||
let merged = assert_converges(&ops, merge_share);
|
||||
assert!(!merged.shared, "late stale share cannot republish");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_share_rules() {
|
||||
let subscribed = subscription(true, 100, "dev-a:1", "Show", "feed");
|
||||
let shared = share(true, 150, "dev-a:2");
|
||||
assert!(effective_share(Some(&subscribed), Some(&shared)));
|
||||
assert!(!effective_share(Some(&subscribed), None));
|
||||
assert!(!effective_share(None, Some(&shared)));
|
||||
// New subscriptions are private: a share decision older than the
|
||||
// latest subscription write does not apply.
|
||||
let resubscribed = subscription(true, 300, "dev-a:4", "Show", "feed");
|
||||
assert!(!effective_share(Some(&resubscribed), Some(&shared)));
|
||||
// Unsubscribing revokes publication even if a stale share wins.
|
||||
let unsubscribed = subscription(false, 200, "dev-a:3", "Show", "feed");
|
||||
let stale = share(true, 250, "dev-b:1");
|
||||
assert!(!effective_share(Some(&unsubscribed), Some(&stale)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clock_skew_cannot_undo_an_observed_unsubscribe() {
|
||||
// Device A's clock runs ten minutes ahead. B sees A's subscription, then
|
||||
// the user unsubscribes on B.
|
||||
let sub_a = subscription(true, 1_000_600_000, "dev-a:1", "Show", "feed");
|
||||
assert_eq!(next_podcast_generation(None, None), 1);
|
||||
let generation = next_podcast_generation(Some(&sub_a), None);
|
||||
assert_eq!(generation, 2);
|
||||
let unsub_b = with_generation(
|
||||
&subscription(false, 1_000_060_000, "dev-b:1", "Show", "feed"),
|
||||
|r| r.generation = generation,
|
||||
);
|
||||
let merged = assert_converges(&[sub_a.clone(), unsub_b.clone()], merge_subscription);
|
||||
assert!(!merged.subscribed, "the later unsubscribe wins");
|
||||
// Concurrent writes (same generation) still fall back to the clock.
|
||||
let concurrent = with_generation(&unsub_b, |r| r.generation = 1);
|
||||
assert!(merge_subscription(&sub_a, &concurrent).unwrap().subscribed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn share_after_subscribe_applies_despite_clock_skew() {
|
||||
let sub_a = subscription(true, 1_000_600_000, "dev-a:1", "Show", "feed");
|
||||
let share_b = with_generation(&share(true, 1_000_060_000, "dev-b:1"), |r| {
|
||||
r.generation = next_podcast_generation(Some(&sub_a), None)
|
||||
});
|
||||
assert!(effective_share(Some(&sub_a), Some(&share_b)));
|
||||
// Unsubscribe written after the share (higher generation) revokes it,
|
||||
// and so does a later re-subscription until the user shares again.
|
||||
let unsub = with_generation(&subscription(false, 1, "dev-a:2", "Show", "feed"), |r| {
|
||||
r.generation = next_podcast_generation(Some(&sub_a), Some(&share_b))
|
||||
});
|
||||
assert!(!effective_share(Some(&unsub), Some(&share_b)));
|
||||
let resub = with_generation(&subscription(true, 2, "dev-a:3", "Show", "feed"), |r| {
|
||||
r.generation = next_podcast_generation(Some(&unsub), Some(&share_b))
|
||||
});
|
||||
assert!(!effective_share(Some(&resub), Some(&share_b)));
|
||||
let reshare = with_generation(&share(true, 3, "dev-a:4"), |r| {
|
||||
r.generation = next_podcast_generation(Some(&resub), Some(&share_b))
|
||||
});
|
||||
assert!(effective_share(Some(&resub), Some(&reshare)));
|
||||
// Revoking (shared = false) with a higher generation wins over every
|
||||
// stale true, in any order.
|
||||
let revoke = with_generation(&share(false, 0, "dev-b:2"), |r| {
|
||||
r.generation = next_podcast_generation(Some(&resub), Some(&reshare))
|
||||
});
|
||||
let merged = assert_converges(&[share_b, reshare, revoke.clone()], merge_share);
|
||||
assert_eq!(merged, revoke);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_feed_url_stays_resolvable_beyond_the_alias_bound() {
|
||||
let mut record = subscription(true, 1, "dev-a:1", "Show", "feed");
|
||||
record.add_feed_aliases(
|
||||
&(0..=crate::podcast::MAX_FEED_ALIASES)
|
||||
.map(|i| feed_url(&format!("a{i}")))
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
record.podcast.feed_url = FeedUrl::parse("https://z.test/current").unwrap();
|
||||
record.normalize();
|
||||
assert!(!record.feed_aliases.contains(&record.podcast.feed_url));
|
||||
let urls = record.feed_urls();
|
||||
assert!(urls.contains(&record.podcast.feed_url));
|
||||
assert!(urls.contains(&feed_url("feed")), "origin is kept");
|
||||
let known: Vec<(&PodcastId, &FeedUrl)> =
|
||||
urls.iter().map(|url| (&record.podcast_id, url)).collect();
|
||||
assert_eq!(
|
||||
crate::podcast::resolve_podcast_id(&record.podcast.feed_url, known),
|
||||
podcast_id()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handoff_receiver_continues_with_the_carried_generation() {
|
||||
use ProgressStatus::*;
|
||||
// Sender A plays session s1 at generation 5 with a fast clock; its
|
||||
// progress operations have not reached B when the handoff arrives.
|
||||
let a_checkpoint = progress(5, 9_000_000, "dev-a", "dev-a:9", 600_000, InProgress, "s1");
|
||||
let carried = Some(a_checkpoint.generation);
|
||||
let b_local = Some(2);
|
||||
let generation = ProgressRecord::explicit_generation(b_local.max(carried));
|
||||
assert_eq!(generation, 6);
|
||||
let b_start = progress(
|
||||
generation, 1_000, "dev-b", "dev-b:1", 600_000, InProgress, "s2",
|
||||
);
|
||||
let b_pause = progress(
|
||||
generation, 2_000, "dev-b", "dev-b:2", 660_000, InProgress, "s2",
|
||||
);
|
||||
// Without the carried generation B would write generation 3 and lose
|
||||
// every write to A's older checkpoint.
|
||||
let uninformed = progress(
|
||||
ProgressRecord::explicit_generation(b_local),
|
||||
2_000,
|
||||
"dev-b",
|
||||
"dev-b:2",
|
||||
660_000,
|
||||
InProgress,
|
||||
"s2",
|
||||
);
|
||||
assert!(a_checkpoint.supersedes(&uninformed));
|
||||
let merged = assert_converges(&[a_checkpoint, b_start, b_pause.clone()], merge_progress);
|
||||
assert_eq!(merged, b_pause);
|
||||
assert_eq!(merged.resume_position_ms(), 660_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coalescing_keeps_the_superseding_record_not_the_last_queued() {
|
||||
use ProgressStatus::*;
|
||||
// "Mark played" (explicit, generation 5) during a session, followed in
|
||||
// the outgoing log by a checkpoint of the old session (generation 3).
|
||||
let mark_played = progress(5, 100, "dev-a", "dev-a:7", 0, Completed, "mark");
|
||||
let late_checkpoint = progress(3, 101, "dev-a", "dev-a:8", 30_000, InProgress, "s1");
|
||||
let pending = [mark_played.clone(), late_checkpoint.clone()];
|
||||
// Keep only records that no other pending record supersedes.
|
||||
let kept: Vec<&ProgressRecord> = pending
|
||||
.iter()
|
||||
.filter(|record| !pending.iter().any(|other| other.supersedes(record)))
|
||||
.collect();
|
||||
assert_eq!(kept, vec![&mark_played]);
|
||||
// Dropping by sequence order would have published the wrong state.
|
||||
assert_ne!(pending.last(), Some(&mark_played));
|
||||
assert_eq!(fold(&pending, merge_progress), mark_played);
|
||||
}
|
||||
|
||||
/// Minimal client store used to show that whole operation sets converge and
|
||||
/// that publisher metadata never reaches personal progress.
|
||||
#[derive(Debug, Default, Clone, PartialEq)]
|
||||
struct Store {
|
||||
subscriptions: BTreeMap<PodcastId, SubscriptionRecord>,
|
||||
shares: BTreeMap<PodcastId, ShareRecord>,
|
||||
progress: BTreeMap<EpisodeId, ProgressRecord>,
|
||||
sessions: BTreeMap<String, ListenSession>,
|
||||
catalog: BTreeMap<EpisodeId, EpisodeMeta>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum Op {
|
||||
Subscription(SubscriptionRecord),
|
||||
Share(ShareRecord),
|
||||
Progress(ProgressRecord),
|
||||
Session(ListenSession),
|
||||
/// Local RSS refresh: title and audio URL changed at the publisher.
|
||||
Rss {
|
||||
title: &'static str,
|
||||
audio_url: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
fn upsert<K: Ord + Clone, V: Clone>(
|
||||
map: &mut BTreeMap<K, V>,
|
||||
key: K,
|
||||
value: &V,
|
||||
merge: impl Fn(&V, &V) -> PodcastResult<V>,
|
||||
) {
|
||||
let merged = match map.get(&key) {
|
||||
Some(current) => merge(current, value).unwrap(),
|
||||
None => value.clone(),
|
||||
};
|
||||
map.insert(key, merged);
|
||||
}
|
||||
|
||||
impl Store {
|
||||
fn apply(&mut self, op: &Op) {
|
||||
match op {
|
||||
Op::Subscription(record) => upsert(
|
||||
&mut self.subscriptions,
|
||||
record.podcast_id.clone(),
|
||||
record,
|
||||
merge_subscription,
|
||||
),
|
||||
Op::Share(record) => upsert(
|
||||
&mut self.shares,
|
||||
record.podcast_id.clone(),
|
||||
record,
|
||||
merge_share,
|
||||
),
|
||||
Op::Progress(record) => upsert(
|
||||
&mut self.progress,
|
||||
record.episode_id.clone(),
|
||||
record,
|
||||
merge_progress,
|
||||
),
|
||||
Op::Session(session) => upsert(
|
||||
&mut self.sessions,
|
||||
session.session_id.clone(),
|
||||
session,
|
||||
merge_listen_session,
|
||||
),
|
||||
Op::Rss { title, audio_url } => {
|
||||
let hints = EpisodeIdentityHints {
|
||||
guid: Some("e1".to_string()),
|
||||
enclosure_url: Some(audio_url.to_string()),
|
||||
link: None,
|
||||
published_at_ms: Some(1_000),
|
||||
title: Some(title.to_string()),
|
||||
};
|
||||
let id = resolve_episode_ids(&podcast_id(), None, &[hints], &[])[0]
|
||||
.clone()
|
||||
.unwrap();
|
||||
// Metadata only: last refresh on this device wins locally.
|
||||
let meta = EpisodeMeta {
|
||||
episode_id: id.clone(),
|
||||
podcast_id: podcast_id(),
|
||||
guid: Some("e1".to_string()),
|
||||
title: title.to_string(),
|
||||
description: None,
|
||||
season: None,
|
||||
episode_number: None,
|
||||
episode_type: EpisodeType::Full,
|
||||
published_at_ms: Some(1_000),
|
||||
duration_ms: None,
|
||||
audio_url: audio_url.to_string(),
|
||||
mime: None,
|
||||
byte_len: None,
|
||||
image_url: None,
|
||||
link: None,
|
||||
};
|
||||
self.catalog.insert(id, meta);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn personal(&self) -> Store {
|
||||
Store {
|
||||
catalog: BTreeMap::new(),
|
||||
..self.clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_operation_sets_converge_and_rss_never_touches_progress() {
|
||||
use ProgressStatus::*;
|
||||
let session = ListenSession {
|
||||
session_id: "s2".to_string(),
|
||||
episode_id: episode_id("e1"),
|
||||
podcast_id: podcast_id(),
|
||||
device_id: "dev-b".to_string(),
|
||||
started_at_ms: 10,
|
||||
ended_at_ms: 20,
|
||||
start_position_ms: 0,
|
||||
end_position_ms: 400_000,
|
||||
listened_ms: 400_000,
|
||||
completed: false,
|
||||
};
|
||||
let ops = [
|
||||
Op::Subscription(subscription(true, 100, "dev-a:1", "Show", "feed")),
|
||||
Op::Share(share(true, 120, "dev-a:2")),
|
||||
Op::Progress(progress(
|
||||
1, 130, "dev-a", "dev-a:3", 900_000, InProgress, "s1",
|
||||
)),
|
||||
Op::Progress(progress(
|
||||
2, 125, "dev-b", "dev-b:1", 400_000, InProgress, "s2",
|
||||
)),
|
||||
Op::Session(session),
|
||||
Op::Rss {
|
||||
title: "Old title",
|
||||
audio_url: "https://cdn.test/old.mp3",
|
||||
},
|
||||
Op::Rss {
|
||||
title: "New title",
|
||||
audio_url: "https://cdn.test/new.mp3",
|
||||
},
|
||||
];
|
||||
let mut reference: Option<Store> = None;
|
||||
for order in permutations(&ops) {
|
||||
let mut store = Store::default();
|
||||
for op in &order {
|
||||
store.apply(op);
|
||||
}
|
||||
// One episode id across both metadata versions.
|
||||
assert_eq!(store.catalog.len(), 1);
|
||||
let progress = store.progress.values().next().unwrap();
|
||||
assert_eq!(progress.op_id, "dev-b:1");
|
||||
assert_eq!(progress.position_ms, 400_000);
|
||||
let personal = store.personal();
|
||||
match &reference {
|
||||
Some(reference) => assert_eq!(&personal, reference),
|
||||
None => reference = Some(personal),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validation_bounds() {
|
||||
use ProgressStatus::*;
|
||||
let ok = progress(1, 1, "dev-a", "dev-a:1", 0, NotStarted, "s");
|
||||
ok.validate().unwrap();
|
||||
let mut bad = ok.clone();
|
||||
bad.position_ms = -1;
|
||||
assert!(bad.validate().is_err());
|
||||
let mut bad = ok.clone();
|
||||
bad.session_id = String::new();
|
||||
assert!(bad.validate().is_err());
|
||||
let mut bad = ok;
|
||||
bad.device_id = "d".repeat(crate::podcast::MAX_TOKEN_BYTES + 1);
|
||||
assert!(bad.validate().is_err());
|
||||
|
||||
let mut record = subscription(true, 1, "dev-a:1", "Show", "feed");
|
||||
record.feed_aliases = (0..=crate::podcast::MAX_FEED_ALIASES + 1)
|
||||
.map(|i| feed_url(&format!("alias-{i}")))
|
||||
.collect();
|
||||
assert!(record.validate().is_err());
|
||||
record.normalize();
|
||||
record.validate().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_json_shape() {
|
||||
let record = progress(
|
||||
3,
|
||||
42,
|
||||
"dev-a",
|
||||
"dev-a:7",
|
||||
1_000,
|
||||
ProgressStatus::InProgress,
|
||||
"s",
|
||||
);
|
||||
let json = serde_json::to_value(&record).unwrap();
|
||||
assert_eq!(json["status"], "in_progress");
|
||||
assert_eq!(json["generation"], 3);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ProgressRecord>(json).unwrap(),
|
||||
record
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,899 @@
|
||||
//! Signed public podcast cards, tombstones, DHT keys and their canonical
|
||||
//! signing encoding.
|
||||
//!
|
||||
//! Everything here is pure and deterministic: no clock, no I/O. Callers pass
|
||||
//! the current time to the validation functions.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
|
||||
use federation_net::{EndpointId, NetworkId, Signature};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{MusicDhtError, Result};
|
||||
use crate::normalization::{normalize_name, tokenize};
|
||||
use crate::podcast::{
|
||||
FeedUrl, MAX_URL_BYTES, PodcastId, SubscriptionRecord, bounded_text, is_http_url,
|
||||
};
|
||||
|
||||
use super::privacy::{FeedUrlPrivacyWarning, feed_url_privacy_warning};
|
||||
|
||||
/// Current card, tombstone and overlay wire version.
|
||||
pub const PODCAST_DHT_PROTOCOL_VERSION: u16 = 1;
|
||||
/// Signing domain of a [`PodcastCard`]. Changing it invalidates every card.
|
||||
pub const PODCAST_CARD_SIGNING_DOMAIN: &str = "furumi-podcast-card:v1";
|
||||
/// Signing domain of a [`PodcastTombstone`].
|
||||
pub const PODCAST_TOMBSTONE_SIGNING_DOMAIN: &str = "furumi-podcast-card-tombstone:v1";
|
||||
/// BLAKE3 derivation domain of [`PodcastDhtKey`].
|
||||
pub const PODCAST_DHT_KEY_DOMAIN: &str = "furumi-podcast-dht:key:v1";
|
||||
|
||||
/// Maximum lifetime of a card: the main DHT's active-record TTL.
|
||||
pub const PODCAST_CARD_TTL: Duration = crate::record::ACTIVE_RECORD_TTL;
|
||||
/// Maximum lifetime of a tombstone: the main DHT's tombstone TTL. It exceeds
|
||||
/// [`PODCAST_CARD_TTL`], so a card issued before a tombstone always expires
|
||||
/// before the tombstone does.
|
||||
pub const PODCAST_TOMBSTONE_TTL: Duration = crate::record::TOMBSTONE_TTL;
|
||||
/// Tolerated clock skew for `issued_at_ms` in the future.
|
||||
pub const PODCAST_DHT_MAX_CLOCK_SKEW: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
/// Maximum characters of a card title.
|
||||
pub const MAX_CARD_TITLE_CHARS: usize = 300;
|
||||
/// Maximum characters of a card author.
|
||||
pub const MAX_CARD_AUTHOR_CHARS: usize = 300;
|
||||
/// Maximum characters of a card description.
|
||||
pub const MAX_CARD_DESCRIPTION_CHARS: usize = 1_000;
|
||||
/// Maximum bytes of a card language tag (BCP 47 shape: ASCII letters,
|
||||
/// digits and `-`).
|
||||
pub const MAX_CARD_LANGUAGE_BYTES: usize = 35;
|
||||
/// Maximum categories on a card.
|
||||
pub const MAX_CARD_CATEGORIES: usize = 8;
|
||||
/// Maximum characters of one category.
|
||||
pub const MAX_CARD_CATEGORY_CHARS: usize = 64;
|
||||
/// Maximum indexed search tokens of a card (title tokens, then author tokens).
|
||||
pub const MAX_CARD_INDEX_TOKENS: usize = 32;
|
||||
/// Maximum bytes of one indexed or query token. Longer tokens are not indexed.
|
||||
pub const MAX_INDEX_TOKEN_BYTES: usize = 64;
|
||||
/// Maximum DHT keys a record is stored under: the podcast-id key plus one
|
||||
/// key per indexed token.
|
||||
pub const MAX_RECORD_DHT_KEYS: usize = MAX_CARD_INDEX_TOKENS + 1;
|
||||
/// Maximum postcard-encoded size of one signed record.
|
||||
pub const MAX_PODCAST_RECORD_BYTES: usize = 24 * 1024;
|
||||
|
||||
const KEY_KIND_PODCAST: u8 = 1;
|
||||
const KEY_KIND_TOKEN: u8 = 2;
|
||||
|
||||
/// A 256-bit point in the podcast-discovery key space.
|
||||
///
|
||||
/// `BLAKE3("furumi-podcast-dht:key:v1" || 0x00 || network_id || kind || material)`
|
||||
/// with kind `0x01` and the podcast id string for the id key, or kind `0x02`
|
||||
/// and a normalized token for a text key.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub struct PodcastDhtKey([u8; 32]);
|
||||
|
||||
impl PodcastDhtKey {
|
||||
/// Key under which every card and tombstone of `podcast_id` is stored.
|
||||
pub fn podcast(network_id: &NetworkId, podcast_id: &PodcastId) -> Self {
|
||||
Self::derive(network_id, KEY_KIND_PODCAST, podcast_id.as_str())
|
||||
}
|
||||
|
||||
/// Key of one normalized search token (see [`card_index_tokens`]).
|
||||
pub fn token(network_id: &NetworkId, token: &str) -> Self {
|
||||
Self::derive(network_id, KEY_KIND_TOKEN, token)
|
||||
}
|
||||
|
||||
fn derive(network_id: &NetworkId, kind: u8, material: &str) -> Self {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
hasher.update(PODCAST_DHT_KEY_DOMAIN.as_bytes());
|
||||
hasher.update(&[0]);
|
||||
hasher.update(network_id.as_bytes());
|
||||
hasher.update(&[kind]);
|
||||
hasher.update(material.as_bytes());
|
||||
Self(*hasher.finalize().as_bytes())
|
||||
}
|
||||
|
||||
/// Creates a key from raw bytes received on the wire.
|
||||
pub fn from_bytes(bytes: [u8; 32]) -> Self {
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
/// Raw key bytes.
|
||||
pub fn as_bytes(&self) -> &[u8; 32] {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Indexed search tokens of a card: distinct [`normalize_name`] tokens of the
|
||||
/// title, then of the author, in order, skipping tokens longer than
|
||||
/// [`MAX_INDEX_TOKEN_BYTES`], at most [`MAX_CARD_INDEX_TOKENS`].
|
||||
pub fn card_index_tokens(title: &str, author: Option<&str>) -> Vec<String> {
|
||||
let mut seen = HashSet::new();
|
||||
let mut tokens = Vec::new();
|
||||
let sources = std::iter::once(title).chain(author);
|
||||
for source in sources {
|
||||
for token in tokenize(&normalize_name(source)) {
|
||||
if tokens.len() == MAX_CARD_INDEX_TOKENS {
|
||||
return tokens;
|
||||
}
|
||||
if token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone()) {
|
||||
tokens.push(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
tokens
|
||||
}
|
||||
|
||||
/// Unsigned public projection of a podcast prepared by its publisher.
|
||||
///
|
||||
/// It is built from the user's subscription ([`PodcastCardSpec::from_subscription`])
|
||||
/// and contains only publisher metadata: never progress, listening history,
|
||||
/// subscription times, operation ids, device names or episodes.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastCardSpec {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Canonical URL the feed is currently fetched from.
|
||||
pub feed_url: FeedUrl,
|
||||
/// URL `podcast_id` was derived from, when it differs from `feed_url`.
|
||||
pub id_feed_url: Option<FeedUrl>,
|
||||
/// Podcast title.
|
||||
pub title: String,
|
||||
/// Author.
|
||||
pub author: Option<String>,
|
||||
/// Plain-text description.
|
||||
pub description: Option<String>,
|
||||
/// Artwork URL.
|
||||
pub image_url: Option<String>,
|
||||
/// Publisher web page.
|
||||
pub link: Option<String>,
|
||||
/// Feed language tag, for example `en` or `pt-BR`.
|
||||
pub language: Option<String>,
|
||||
/// Feed categories.
|
||||
pub categories: Vec<String>,
|
||||
/// Number of episodes in the feed, when known.
|
||||
pub episode_count: Option<u32>,
|
||||
/// Publication time of the newest episode in unix milliseconds.
|
||||
pub latest_episode_at_ms: Option<i64>,
|
||||
}
|
||||
|
||||
impl PodcastCardSpec {
|
||||
/// Builds the bounded public projection of a subscription.
|
||||
///
|
||||
/// Only `podcast_id`, the feed URLs and the channel metadata are read.
|
||||
/// `id_feed_url` is the alias that hashes to `podcast_id`
|
||||
/// ([`crate::podcast::merge_feed_aliases`] always keeps it). Fails when no
|
||||
/// such URL is known. Callers may then set `language`, `categories`,
|
||||
/// `episode_count` and `latest_episode_at_ms` from their feed cache.
|
||||
pub fn from_subscription(subscription: &SubscriptionRecord) -> Result<Self> {
|
||||
let origin = subscription
|
||||
.feed_urls()
|
||||
.into_iter()
|
||||
.find(|url| PodcastId::from_feed_url(url) == subscription.podcast_id)
|
||||
.ok_or_else(|| {
|
||||
publication_error("the feed URL the podcast id was derived from is unknown")
|
||||
})?;
|
||||
let feed_url = subscription.podcast.feed_url.clone();
|
||||
let meta = &subscription.podcast;
|
||||
Ok(Self {
|
||||
podcast_id: subscription.podcast_id.clone(),
|
||||
id_feed_url: (origin != feed_url).then_some(origin),
|
||||
feed_url,
|
||||
title: meta.title.clone(),
|
||||
author: meta.author.clone(),
|
||||
description: meta.description.clone(),
|
||||
image_url: meta.image_url.clone(),
|
||||
link: meta.link.clone(),
|
||||
language: None,
|
||||
categories: Vec::new(),
|
||||
episode_count: None,
|
||||
latest_episode_at_ms: None,
|
||||
}
|
||||
.bounded())
|
||||
}
|
||||
|
||||
/// Returns a copy that fits the card bounds: text is truncated with
|
||||
/// [`bounded_text`], control characters are removed, invalid URLs and
|
||||
/// language tags are dropped, and categories are trimmed, deduplicated and
|
||||
/// bounded. An empty title becomes `"Untitled podcast"`.
|
||||
pub fn bounded(&self) -> Self {
|
||||
let title = clean_line(&self.title);
|
||||
let mut seen = HashSet::new();
|
||||
let categories = self
|
||||
.categories
|
||||
.iter()
|
||||
.map(|category| bounded_text(&clean_line(category), MAX_CARD_CATEGORY_CHARS))
|
||||
.filter(|category| !category.is_empty() && seen.insert(category.to_lowercase()))
|
||||
.take(MAX_CARD_CATEGORIES)
|
||||
.collect();
|
||||
Self {
|
||||
podcast_id: self.podcast_id.clone(),
|
||||
feed_url: self.feed_url.clone(),
|
||||
id_feed_url: self
|
||||
.id_feed_url
|
||||
.clone()
|
||||
.filter(|origin| *origin != self.feed_url),
|
||||
title: if title.is_empty() {
|
||||
"Untitled podcast".to_string()
|
||||
} else {
|
||||
bounded_text(&title, MAX_CARD_TITLE_CHARS)
|
||||
},
|
||||
author: bounded_line(self.author.as_deref(), MAX_CARD_AUTHOR_CHARS),
|
||||
description: self
|
||||
.description
|
||||
.as_deref()
|
||||
.map(|text| bounded_text(clean_block(text).trim(), MAX_CARD_DESCRIPTION_CHARS))
|
||||
.filter(|text| !text.is_empty()),
|
||||
image_url: self.image_url.clone().filter(|url| is_http_url(url)),
|
||||
link: self.link.clone().filter(|url| is_http_url(url)),
|
||||
language: self
|
||||
.language
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|language| valid_language(language))
|
||||
.map(str::to_string),
|
||||
categories,
|
||||
episode_count: self.episode_count,
|
||||
latest_episode_at_ms: self.latest_episode_at_ms.filter(|at| *at >= 0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the card bounds without bounding anything.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
let id_url = self.id_feed_url.as_ref().unwrap_or(&self.feed_url);
|
||||
if PodcastId::from_feed_url(id_url) != self.podcast_id {
|
||||
return Err(invalid("podcast id does not match its feed URL"));
|
||||
}
|
||||
if self.id_feed_url.as_ref() == Some(&self.feed_url) {
|
||||
return Err(invalid("id_feed_url must differ from feed_url"));
|
||||
}
|
||||
check_line("title", &self.title, MAX_CARD_TITLE_CHARS, true)?;
|
||||
if let Some(author) = &self.author {
|
||||
check_line("author", author, MAX_CARD_AUTHOR_CHARS, false)?;
|
||||
}
|
||||
if let Some(description) = &self.description {
|
||||
check_chars("description", description, MAX_CARD_DESCRIPTION_CHARS)?;
|
||||
if description
|
||||
.chars()
|
||||
.any(|c| c.is_control() && c != '\n' && c != '\t')
|
||||
{
|
||||
return Err(invalid("description contains control characters"));
|
||||
}
|
||||
}
|
||||
for url in [&self.image_url, &self.link].into_iter().flatten() {
|
||||
if !is_http_url(url) {
|
||||
return Err(invalid("card URLs must be absolute http(s) URLs"));
|
||||
}
|
||||
}
|
||||
if self
|
||||
.language
|
||||
.as_deref()
|
||||
.is_some_and(|language| !valid_language(language))
|
||||
{
|
||||
return Err(invalid("invalid language tag"));
|
||||
}
|
||||
if self.categories.len() > MAX_CARD_CATEGORIES {
|
||||
return Err(invalid("too many categories"));
|
||||
}
|
||||
for category in &self.categories {
|
||||
check_line("category", category, MAX_CARD_CATEGORY_CHARS, true)?;
|
||||
}
|
||||
if self.latest_episode_at_ms.is_some_and(|at| at < 0) {
|
||||
return Err(invalid("negative latest_episode_at_ms"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Advisory privacy warnings for both feed URLs of the card (see
|
||||
/// [`feed_url_privacy_warning`]). The UI must ask before publishing a
|
||||
/// card with warnings.
|
||||
pub fn privacy_warnings(&self) -> Vec<FeedUrlPrivacyWarning> {
|
||||
std::iter::once(&self.feed_url)
|
||||
.chain(self.id_feed_url.as_ref())
|
||||
.filter_map(feed_url_privacy_warning)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Indexed search tokens of this card.
|
||||
pub fn index_tokens(&self) -> Vec<String> {
|
||||
card_index_tokens(&self.title, self.author.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed public podcast card: one publisher device's projection of one
|
||||
/// podcast. See `PODCASTS.md`, "Public discovery (podcast_dht)".
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastCard {
|
||||
/// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||
pub version: u16,
|
||||
/// Network scope; prevents replay into another federation.
|
||||
pub network_id: NetworkId,
|
||||
/// Publisher device (its persistent transport identity); signs the card.
|
||||
pub publisher: EndpointId,
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Canonical URL the feed is currently fetched from.
|
||||
pub feed_url: FeedUrl,
|
||||
/// URL `podcast_id` was derived from, when it differs from `feed_url`.
|
||||
pub id_feed_url: Option<FeedUrl>,
|
||||
/// Podcast title.
|
||||
pub title: String,
|
||||
/// Author.
|
||||
pub author: Option<String>,
|
||||
/// Plain-text description.
|
||||
pub description: Option<String>,
|
||||
/// Artwork URL.
|
||||
pub image_url: Option<String>,
|
||||
/// Publisher web page.
|
||||
pub link: Option<String>,
|
||||
/// Feed language tag.
|
||||
pub language: Option<String>,
|
||||
/// Feed categories.
|
||||
pub categories: Vec<String>,
|
||||
/// Number of episodes in the feed, when known.
|
||||
pub episode_count: Option<u32>,
|
||||
/// Publication time of the newest episode in unix milliseconds.
|
||||
pub latest_episode_at_ms: Option<i64>,
|
||||
/// Monotonic per `(podcast_id, publisher)` revision.
|
||||
pub revision: u64,
|
||||
/// Unix time at which this version was signed.
|
||||
pub issued_at_ms: u64,
|
||||
/// Unix time after which the card must be discarded.
|
||||
pub expires_at_ms: u64,
|
||||
}
|
||||
|
||||
impl PodcastCard {
|
||||
/// Builds a card from a spec. `expires_at_ms` is
|
||||
/// `issued_at_ms + PODCAST_CARD_TTL`.
|
||||
pub fn from_spec(
|
||||
spec: PodcastCardSpec,
|
||||
network_id: NetworkId,
|
||||
publisher: EndpointId,
|
||||
revision: u64,
|
||||
issued_at_ms: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
network_id,
|
||||
publisher,
|
||||
podcast_id: spec.podcast_id,
|
||||
feed_url: spec.feed_url,
|
||||
id_feed_url: spec.id_feed_url,
|
||||
title: spec.title,
|
||||
author: spec.author,
|
||||
description: spec.description,
|
||||
image_url: spec.image_url,
|
||||
link: spec.link,
|
||||
language: spec.language,
|
||||
categories: spec.categories,
|
||||
episode_count: spec.episode_count,
|
||||
latest_episode_at_ms: spec.latest_episode_at_ms,
|
||||
revision,
|
||||
issued_at_ms,
|
||||
expires_at_ms: issued_at_ms.saturating_add(PODCAST_CARD_TTL.as_millis() as u64),
|
||||
}
|
||||
}
|
||||
|
||||
/// The publisher-provided part of the card.
|
||||
pub fn spec(&self) -> PodcastCardSpec {
|
||||
PodcastCardSpec {
|
||||
podcast_id: self.podcast_id.clone(),
|
||||
feed_url: self.feed_url.clone(),
|
||||
id_feed_url: self.id_feed_url.clone(),
|
||||
title: self.title.clone(),
|
||||
author: self.author.clone(),
|
||||
description: self.description.clone(),
|
||||
image_url: self.image_url.clone(),
|
||||
link: self.link.clone(),
|
||||
language: self.language.clone(),
|
||||
categories: self.categories.clone(),
|
||||
episode_count: self.episode_count,
|
||||
latest_episode_at_ms: self.latest_episode_at_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// Indexed search tokens.
|
||||
pub fn index_tokens(&self) -> Vec<String> {
|
||||
card_index_tokens(&self.title, self.author.as_deref())
|
||||
}
|
||||
|
||||
/// The URL `podcast_id` is derived from (`id_feed_url`, or `feed_url`
|
||||
/// when the feed has not moved). A valid card's id hashes from it, so it
|
||||
/// is the only URL of the card bound to the podcast.
|
||||
pub fn origin_feed_url(&self) -> &FeedUrl {
|
||||
self.id_feed_url.as_ref().unwrap_or(&self.feed_url)
|
||||
}
|
||||
|
||||
/// Whether `feed_url` is the URL the id is derived from. Otherwise
|
||||
/// `feed_url` is an unverified claim of the publisher that the feed moved.
|
||||
pub fn feed_url_is_origin(&self) -> bool {
|
||||
self.id_feed_url.is_none()
|
||||
}
|
||||
|
||||
/// Canonical signing input (see `PODCASTS.md`).
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
let mut out = Canonical::new(PODCAST_CARD_SIGNING_DOMAIN);
|
||||
out.u16(self.version);
|
||||
out.raw(self.network_id.as_bytes());
|
||||
out.raw(self.publisher.as_bytes());
|
||||
out.str(self.podcast_id.as_str());
|
||||
out.str(self.feed_url.as_str());
|
||||
out.opt_str(self.id_feed_url.as_ref().map(FeedUrl::as_str));
|
||||
out.str(&self.title);
|
||||
out.opt_str(self.author.as_deref());
|
||||
out.opt_str(self.description.as_deref());
|
||||
out.opt_str(self.image_url.as_deref());
|
||||
out.opt_str(self.link.as_deref());
|
||||
out.opt_str(self.language.as_deref());
|
||||
out.u32(self.categories.len() as u32);
|
||||
for category in &self.categories {
|
||||
out.str(category);
|
||||
}
|
||||
match self.episode_count {
|
||||
Some(count) => {
|
||||
out.u8(1);
|
||||
out.u32(count);
|
||||
}
|
||||
None => out.u8(0),
|
||||
}
|
||||
match self.latest_episode_at_ms {
|
||||
Some(at) => {
|
||||
out.u8(1);
|
||||
out.i64(at);
|
||||
}
|
||||
None => out.u8(0),
|
||||
}
|
||||
out.u64(self.revision);
|
||||
out.u64(self.issued_at_ms);
|
||||
out.u64(self.expires_at_ms);
|
||||
out.finish()
|
||||
}
|
||||
|
||||
fn validate(&self, expected_network: &NetworkId) -> Result<()> {
|
||||
check_header(self.version, &self.network_id, expected_network)?;
|
||||
self.spec().validate()?;
|
||||
check_lifetime(
|
||||
self.revision,
|
||||
self.issued_at_ms,
|
||||
self.expires_at_ms,
|
||||
PODCAST_CARD_TTL,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A signed retraction of one publisher's card of one podcast.
|
||||
///
|
||||
/// It supersedes every card of the same `(podcast_id, publisher)` with a
|
||||
/// revision lower than or equal to its own and is retained until it expires.
|
||||
/// `index_keys` lists the text keys of the retracted cards so the tombstone
|
||||
/// can be stored next to them; it carries hashes, not the tokens.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastTombstone {
|
||||
/// Record format version ([`PODCAST_DHT_PROTOCOL_VERSION`]).
|
||||
pub version: u16,
|
||||
/// Network scope.
|
||||
pub network_id: NetworkId,
|
||||
/// Publisher device; signs the tombstone.
|
||||
pub publisher: EndpointId,
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// Revision; greater than that of every retracted card.
|
||||
pub revision: u64,
|
||||
/// Unix time at which the tombstone was signed.
|
||||
pub issued_at_ms: u64,
|
||||
/// Unix time after which the tombstone may be discarded.
|
||||
pub expires_at_ms: u64,
|
||||
/// Sorted, distinct text keys besides the podcast-id key, at most
|
||||
/// [`MAX_CARD_INDEX_TOKENS`].
|
||||
pub index_keys: Vec<PodcastDhtKey>,
|
||||
}
|
||||
|
||||
impl PodcastTombstone {
|
||||
/// Builds a tombstone. `expires_at_ms` is `issued_at_ms + PODCAST_TOMBSTONE_TTL`;
|
||||
/// `index_keys` is sorted, deduplicated and stripped of the podcast-id key.
|
||||
pub fn new(
|
||||
network_id: NetworkId,
|
||||
publisher: EndpointId,
|
||||
podcast_id: PodcastId,
|
||||
revision: u64,
|
||||
issued_at_ms: u64,
|
||||
index_keys: impl IntoIterator<Item = PodcastDhtKey>,
|
||||
) -> Self {
|
||||
let id_key = PodcastDhtKey::podcast(&network_id, &podcast_id);
|
||||
let mut index_keys: Vec<_> = index_keys
|
||||
.into_iter()
|
||||
.filter(|key| *key != id_key)
|
||||
.collect();
|
||||
index_keys.sort();
|
||||
index_keys.dedup();
|
||||
index_keys.truncate(MAX_CARD_INDEX_TOKENS);
|
||||
Self {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
network_id,
|
||||
publisher,
|
||||
podcast_id,
|
||||
revision,
|
||||
issued_at_ms,
|
||||
expires_at_ms: issued_at_ms.saturating_add(PODCAST_TOMBSTONE_TTL.as_millis() as u64),
|
||||
index_keys,
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical signing input (see `PODCASTS.md`).
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
let mut out = Canonical::new(PODCAST_TOMBSTONE_SIGNING_DOMAIN);
|
||||
out.u16(self.version);
|
||||
out.raw(self.network_id.as_bytes());
|
||||
out.raw(self.publisher.as_bytes());
|
||||
out.str(self.podcast_id.as_str());
|
||||
out.u64(self.revision);
|
||||
out.u64(self.issued_at_ms);
|
||||
out.u64(self.expires_at_ms);
|
||||
out.u32(self.index_keys.len() as u32);
|
||||
for key in &self.index_keys {
|
||||
out.raw(key.as_bytes());
|
||||
}
|
||||
out.finish()
|
||||
}
|
||||
|
||||
fn validate(&self, expected_network: &NetworkId) -> Result<()> {
|
||||
check_header(self.version, &self.network_id, expected_network)?;
|
||||
if self.index_keys.len() > MAX_CARD_INDEX_TOKENS {
|
||||
return Err(invalid("tombstone has too many keys"));
|
||||
}
|
||||
let id_key = PodcastDhtKey::podcast(&self.network_id, &self.podcast_id);
|
||||
if self.index_keys.contains(&id_key)
|
||||
|| self.index_keys.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
{
|
||||
return Err(invalid("tombstone keys are not canonical"));
|
||||
}
|
||||
check_lifetime(
|
||||
self.revision,
|
||||
self.issued_at_ms,
|
||||
self.expires_at_ms,
|
||||
PODCAST_TOMBSTONE_TTL,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Body of a signed podcast-discovery record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum PodcastRecordBody {
|
||||
/// An active public card.
|
||||
Card(Box<PodcastCard>),
|
||||
/// A retraction.
|
||||
Tombstone(PodcastTombstone),
|
||||
}
|
||||
|
||||
/// End-to-end signed podcast-discovery record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SignedPodcastRecord {
|
||||
/// Signed body.
|
||||
pub body: PodcastRecordBody,
|
||||
/// Ed25519 signature by `publisher` over [`Self::signing_bytes`].
|
||||
pub signature: Signature,
|
||||
}
|
||||
|
||||
impl SignedPodcastRecord {
|
||||
/// Validates and signs a card.
|
||||
pub fn sign_card(card: PodcastCard, signer: impl FnOnce(&[u8]) -> Signature) -> Result<Self> {
|
||||
card.validate(&card.network_id)?;
|
||||
let signature = signer(&card.signing_bytes());
|
||||
let record = Self {
|
||||
body: PodcastRecordBody::Card(Box::new(card)),
|
||||
signature,
|
||||
};
|
||||
record.check_size()?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// Validates and signs a tombstone.
|
||||
pub fn sign_tombstone(
|
||||
tombstone: PodcastTombstone,
|
||||
signer: impl FnOnce(&[u8]) -> Signature,
|
||||
) -> Result<Self> {
|
||||
tombstone.validate(&tombstone.network_id)?;
|
||||
let signature = signer(&tombstone.signing_bytes());
|
||||
let record = Self {
|
||||
body: PodcastRecordBody::Tombstone(tombstone),
|
||||
signature,
|
||||
};
|
||||
record.check_size()?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// Canonical signing input of the body.
|
||||
pub fn signing_bytes(&self) -> Vec<u8> {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.signing_bytes(),
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.signing_bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates bounds, network, lifetime shape, encoded size and the
|
||||
/// publisher's signature. Does not look at the clock; see
|
||||
/// [`Self::verify_at`].
|
||||
pub fn verify(&self, expected_network: &NetworkId) -> Result<()> {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.validate(expected_network)?,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.validate(expected_network)?,
|
||||
}
|
||||
self.check_size()?;
|
||||
self.publisher()
|
||||
.verify(&self.signing_bytes(), &self.signature)
|
||||
.map_err(|_| invalid("invalid podcast record signature"))
|
||||
}
|
||||
|
||||
/// [`Self::verify`] plus time checks: not issued more than
|
||||
/// [`PODCAST_DHT_MAX_CLOCK_SKEW`] in the future and not expired at `now_ms`.
|
||||
pub fn verify_at(&self, expected_network: &NetworkId, now_ms: u64) -> Result<()> {
|
||||
self.verify(expected_network)?;
|
||||
let future_limit = now_ms.saturating_add(PODCAST_DHT_MAX_CLOCK_SKEW.as_millis() as u64);
|
||||
if self.issued_at_ms() > future_limit || self.expires_at_ms() <= now_ms {
|
||||
return Err(invalid("podcast record is expired or from the future"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The card, when this is not a tombstone.
|
||||
pub fn card(&self) -> Option<&PodcastCard> {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => Some(card.as_ref()),
|
||||
PodcastRecordBody::Tombstone(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this is a tombstone.
|
||||
pub fn is_tombstone(&self) -> bool {
|
||||
matches!(self.body, PodcastRecordBody::Tombstone(_))
|
||||
}
|
||||
|
||||
/// Podcast.
|
||||
pub fn podcast_id(&self) -> &PodcastId {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => &card.podcast_id,
|
||||
PodcastRecordBody::Tombstone(tombstone) => &tombstone.podcast_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Publisher device.
|
||||
pub fn publisher(&self) -> EndpointId {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.publisher,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.publisher,
|
||||
}
|
||||
}
|
||||
|
||||
/// Network scope.
|
||||
pub fn network_id(&self) -> NetworkId {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.network_id,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.network_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Revision.
|
||||
pub fn revision(&self) -> u64 {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.revision,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.revision,
|
||||
}
|
||||
}
|
||||
|
||||
/// Signing time.
|
||||
pub fn issued_at_ms(&self) -> u64 {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.issued_at_ms,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.issued_at_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// Expiry time.
|
||||
pub fn expires_at_ms(&self) -> u64 {
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => card.expires_at_ms,
|
||||
PodcastRecordBody::Tombstone(tombstone) => tombstone.expires_at_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every DHT key this record belongs under: the podcast-id key, then the
|
||||
/// text keys (card tokens or the tombstone's `index_keys`).
|
||||
pub fn dht_keys(&self) -> Vec<PodcastDhtKey> {
|
||||
let network_id = self.network_id();
|
||||
let mut keys = vec![PodcastDhtKey::podcast(&network_id, self.podcast_id())];
|
||||
match &self.body {
|
||||
PodcastRecordBody::Card(card) => keys.extend(
|
||||
card.index_tokens()
|
||||
.iter()
|
||||
.map(|token| PodcastDhtKey::token(&network_id, token)),
|
||||
),
|
||||
PodcastRecordBody::Tombstone(tombstone) => {
|
||||
keys.extend(tombstone.index_keys.iter().copied());
|
||||
}
|
||||
}
|
||||
keys
|
||||
}
|
||||
|
||||
/// Whether the record may be stored under `key`.
|
||||
pub fn belongs_under(&self, key: &PodcastDhtKey) -> bool {
|
||||
self.dht_keys().contains(key)
|
||||
}
|
||||
|
||||
/// postcard-encoded size.
|
||||
pub fn encoded_len(&self) -> usize {
|
||||
postcard::to_stdvec(self).map_or(usize::MAX, |bytes| bytes.len())
|
||||
}
|
||||
|
||||
fn check_size(&self) -> Result<()> {
|
||||
if self.encoded_len() > MAX_PODCAST_RECORD_BYTES {
|
||||
Err(invalid("podcast record is too large"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Total order of two records of the same `(podcast_id, publisher)`.
|
||||
///
|
||||
/// Greater wins: higher `revision`; at an equal revision a tombstone beats a
|
||||
/// card; then later `issued_at_ms` (a republish); then later
|
||||
/// `expires_at_ms`; then the lexicographically greater canonical signing
|
||||
/// input. Every node therefore keeps the same winner whatever the delivery
|
||||
/// order.
|
||||
pub fn compare_records(left: &SignedPodcastRecord, right: &SignedPodcastRecord) -> Ordering {
|
||||
left.revision()
|
||||
.cmp(&right.revision())
|
||||
.then_with(|| left.is_tombstone().cmp(&right.is_tombstone()))
|
||||
.then_with(|| left.issued_at_ms().cmp(&right.issued_at_ms()))
|
||||
.then_with(|| left.expires_at_ms().cmp(&right.expires_at_ms()))
|
||||
.then_with(|| left.signing_bytes().cmp(&right.signing_bytes()))
|
||||
}
|
||||
|
||||
struct Canonical(Vec<u8>);
|
||||
|
||||
impl Canonical {
|
||||
fn new(domain: &str) -> Self {
|
||||
let mut bytes = Vec::with_capacity(512);
|
||||
bytes.extend_from_slice(domain.as_bytes());
|
||||
bytes.push(0);
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
fn raw(&mut self, bytes: &[u8]) {
|
||||
self.0.extend_from_slice(bytes);
|
||||
}
|
||||
|
||||
fn u8(&mut self, value: u8) {
|
||||
self.0.push(value);
|
||||
}
|
||||
|
||||
fn u16(&mut self, value: u16) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn u32(&mut self, value: u32) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn u64(&mut self, value: u64) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn i64(&mut self, value: i64) {
|
||||
self.raw(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
fn str(&mut self, value: &str) {
|
||||
self.u32(value.len() as u32);
|
||||
self.raw(value.as_bytes());
|
||||
}
|
||||
|
||||
fn opt_str(&mut self, value: Option<&str>) {
|
||||
match value {
|
||||
Some(value) => {
|
||||
self.u8(1);
|
||||
self.str(value);
|
||||
}
|
||||
None => self.u8(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(self) -> Vec<u8> {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
fn check_header(version: u16, network_id: &NetworkId, expected: &NetworkId) -> Result<()> {
|
||||
if version != PODCAST_DHT_PROTOCOL_VERSION {
|
||||
return Err(invalid("unsupported podcast record version"));
|
||||
}
|
||||
if network_id != expected {
|
||||
return Err(invalid("podcast record belongs to another network"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_lifetime(
|
||||
revision: u64,
|
||||
issued_at_ms: u64,
|
||||
expires_at_ms: u64,
|
||||
ttl: Duration,
|
||||
) -> Result<()> {
|
||||
if revision == 0
|
||||
|| issued_at_ms == 0
|
||||
|| expires_at_ms <= issued_at_ms
|
||||
|| expires_at_ms - issued_at_ms > ttl.as_millis() as u64
|
||||
{
|
||||
return Err(invalid("invalid podcast record revision or lifetime"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_chars(field: &'static str, value: &str, max_chars: usize) -> Result<()> {
|
||||
if value.len() > max_chars * 4 || value.chars().count() > max_chars {
|
||||
return Err(MusicDhtError::Protocol(format!("card {field} is too long")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_line(field: &'static str, value: &str, max_chars: usize, required: bool) -> Result<()> {
|
||||
if required && value.trim().is_empty() {
|
||||
return Err(MusicDhtError::Protocol(format!("card {field} is empty")));
|
||||
}
|
||||
check_chars(field, value, max_chars)?;
|
||||
if value.chars().any(char::is_control) {
|
||||
return Err(MusicDhtError::Protocol(format!(
|
||||
"card {field} contains control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn valid_language(language: &str) -> bool {
|
||||
!language.is_empty()
|
||||
&& language.len() <= MAX_CARD_LANGUAGE_BYTES
|
||||
&& language
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
}
|
||||
|
||||
fn clean_line(text: &str) -> String {
|
||||
text.split(char::is_control)
|
||||
.filter(|part| !part.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn clean_block(text: &str) -> String {
|
||||
text.chars()
|
||||
.filter(|c| !c.is_control() || *c == '\n' || *c == '\t')
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bounded_line(text: Option<&str>, max_chars: usize) -> Option<String> {
|
||||
text.map(clean_line)
|
||||
.filter(|text| !text.is_empty())
|
||||
.map(|text| bounded_text(&text, max_chars))
|
||||
}
|
||||
|
||||
fn invalid(reason: &str) -> MusicDhtError {
|
||||
MusicDhtError::Protocol(reason.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn publication_error(reason: &str) -> MusicDhtError {
|
||||
MusicDhtError::PodcastPublication(reason.to_string())
|
||||
}
|
||||
|
||||
// Statically bound the worst-case card so `MAX_PODCAST_RECORD_BYTES` is a
|
||||
// defensive limit, not the one users hit first.
|
||||
const _: () = assert!(
|
||||
MAX_CARD_TITLE_CHARS * 4
|
||||
+ MAX_CARD_AUTHOR_CHARS * 4
|
||||
+ MAX_CARD_DESCRIPTION_CHARS * 4
|
||||
+ MAX_CARD_CATEGORIES * MAX_CARD_CATEGORY_CHARS * 4
|
||||
+ MAX_CARD_LANGUAGE_BYTES
|
||||
+ 4 * MAX_URL_BYTES
|
||||
+ 1024
|
||||
<= MAX_PODCAST_RECORD_BYTES
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,339 @@
|
||||
//! Advisory detection of feed URLs that look private.
|
||||
//!
|
||||
//! Premium and member feeds usually carry a personal secret in the URL. A
|
||||
//! card publishes its feed URL verbatim, so the UI must ask the user before
|
||||
//! publishing a URL for which [`feed_url_privacy_warning`] returns a warning.
|
||||
//! The heuristics are deliberately broad and can produce false positives
|
||||
//! (and miss secrets); they never block publication by themselves.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::podcast::FeedUrl;
|
||||
|
||||
/// Query-parameter name words that usually carry a credential. A name is
|
||||
/// split into lowercase words at non-alphanumeric characters and camelCase
|
||||
/// boundaries; any listed word triggers a warning.
|
||||
pub const SECRET_QUERY_WORDS: &[&str] = &[
|
||||
"token",
|
||||
"key",
|
||||
"auth",
|
||||
"secret",
|
||||
"sig",
|
||||
"signature",
|
||||
"uid",
|
||||
"user",
|
||||
"pass",
|
||||
"password",
|
||||
"passwd",
|
||||
"pwd",
|
||||
"session",
|
||||
"sess",
|
||||
"sid",
|
||||
"access",
|
||||
"private",
|
||||
"credential",
|
||||
"credentials",
|
||||
"apikey",
|
||||
"jwt",
|
||||
"hmac",
|
||||
];
|
||||
|
||||
/// Substrings that trigger a warning anywhere in a query-parameter name,
|
||||
/// for names such as `apitoken` or `privatefeed` that do not split.
|
||||
const SECRET_QUERY_SUBSTRINGS: &[&str] = &[
|
||||
"token",
|
||||
"secret",
|
||||
"signature",
|
||||
"password",
|
||||
"passwd",
|
||||
"session",
|
||||
"private",
|
||||
"credential",
|
||||
"apikey",
|
||||
];
|
||||
|
||||
/// Hosts (and their subdomains) known to serve personal member feeds.
|
||||
pub const PRIVATE_FEED_HOSTS: &[&str] = &[
|
||||
"patreon.com",
|
||||
"supercast.com",
|
||||
"supportingcast.fm",
|
||||
"glow.fm",
|
||||
];
|
||||
|
||||
/// Minimum length of a query value considered a possible secret.
|
||||
const MIN_SECRET_VALUE_CHARS: usize = 16;
|
||||
/// Minimum length of a path segment considered a possible secret.
|
||||
const MIN_SECRET_SEGMENT_CHARS: usize = 32;
|
||||
/// Minimum Shannon entropy, in bits per character, of a possible secret.
|
||||
const MIN_SECRET_ENTROPY_BITS: f64 = 3.0;
|
||||
|
||||
/// Why a feed URL may contain a personal secret.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "reason", rename_all = "snake_case")]
|
||||
#[non_exhaustive]
|
||||
pub enum FeedUrlPrivacyWarning {
|
||||
/// A query parameter whose name suggests a credential.
|
||||
SecretQueryParameter {
|
||||
/// Parameter name as it appears in the URL.
|
||||
name: String,
|
||||
},
|
||||
/// The host is known to serve personal member feeds.
|
||||
PrivateFeedHost {
|
||||
/// Lowercase host.
|
||||
host: String,
|
||||
},
|
||||
/// A long, random-looking query value.
|
||||
HighEntropyQueryValue {
|
||||
/// Name of the parameter carrying the value.
|
||||
name: String,
|
||||
},
|
||||
/// A long, random-looking path segment.
|
||||
HighEntropyPathSegment {
|
||||
/// Zero-based index of the segment after the host.
|
||||
index: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl fmt::Display for FeedUrlPrivacyWarning {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::SecretQueryParameter { name } => write!(
|
||||
f,
|
||||
"the feed URL has a query parameter \"{name}\" that looks like a credential"
|
||||
),
|
||||
Self::PrivateFeedHost { host } => {
|
||||
write!(f, "{host} usually serves personal member feeds")
|
||||
}
|
||||
Self::HighEntropyQueryValue { name } => write!(
|
||||
f,
|
||||
"the value of the query parameter \"{name}\" looks like a secret token"
|
||||
),
|
||||
Self::HighEntropyPathSegment { index } => write!(
|
||||
f,
|
||||
"path segment {} of the feed URL looks like a secret token",
|
||||
index + 1
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns why `url` may contain a personal secret, or `None`.
|
||||
///
|
||||
/// Advisory only (see the module documentation). Checks, in order: query
|
||||
/// parameter names ([`SECRET_QUERY_WORDS`]), known member-feed hosts
|
||||
/// ([`PRIVATE_FEED_HOSTS`]), long high-entropy query values, and long
|
||||
/// high-entropy path segments.
|
||||
pub fn feed_url_privacy_warning(url: &FeedUrl) -> Option<FeedUrlPrivacyWarning> {
|
||||
let value = url.as_str();
|
||||
let rest = value.split_once("://").map_or(value, |(_, rest)| rest);
|
||||
let (authority, path_and_query) = match rest.find('/') {
|
||||
Some(index) => rest.split_at(index),
|
||||
None => (rest, "/"),
|
||||
};
|
||||
let host = if authority.starts_with('[') {
|
||||
authority
|
||||
.find(']')
|
||||
.map_or(authority, |end| &authority[..=end])
|
||||
} else {
|
||||
authority.split(':').next().unwrap_or(authority)
|
||||
};
|
||||
let (path, query) = path_and_query
|
||||
.split_once('?')
|
||||
.unwrap_or((path_and_query, ""));
|
||||
let parameters: Vec<(&str, &str)> = query
|
||||
.split('&')
|
||||
.filter(|pair| !pair.is_empty())
|
||||
.map(|pair| pair.split_once('=').unwrap_or((pair, "")))
|
||||
.collect();
|
||||
|
||||
if let Some((name, _)) = parameters
|
||||
.iter()
|
||||
.find(|(name, _)| is_secret_parameter_name(name))
|
||||
{
|
||||
return Some(FeedUrlPrivacyWarning::SecretQueryParameter {
|
||||
name: (*name).to_string(),
|
||||
});
|
||||
}
|
||||
if PRIVATE_FEED_HOSTS
|
||||
.iter()
|
||||
.any(|known| host == *known || host.ends_with(&format!(".{known}")))
|
||||
{
|
||||
return Some(FeedUrlPrivacyWarning::PrivateFeedHost {
|
||||
host: host.to_string(),
|
||||
});
|
||||
}
|
||||
if let Some((name, _)) = parameters
|
||||
.iter()
|
||||
.find(|(_, value)| looks_random(value, MIN_SECRET_VALUE_CHARS))
|
||||
{
|
||||
return Some(FeedUrlPrivacyWarning::HighEntropyQueryValue {
|
||||
name: (*name).to_string(),
|
||||
});
|
||||
}
|
||||
path.split('/')
|
||||
.filter(|segment| !segment.is_empty())
|
||||
.position(|segment| {
|
||||
let stem = segment.rsplit_once('.').map_or(segment, |(stem, _)| stem);
|
||||
looks_random(stem, MIN_SECRET_SEGMENT_CHARS)
|
||||
})
|
||||
.map(|index| FeedUrlPrivacyWarning::HighEntropyPathSegment { index })
|
||||
}
|
||||
|
||||
fn is_secret_parameter_name(name: &str) -> bool {
|
||||
let words = split_words(name);
|
||||
if words
|
||||
.iter()
|
||||
.any(|word| SECRET_QUERY_WORDS.contains(&word.as_str()))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let compact: String = name
|
||||
.chars()
|
||||
.filter(char::is_ascii_alphanumeric)
|
||||
.collect::<String>()
|
||||
.to_ascii_lowercase();
|
||||
SECRET_QUERY_SUBSTRINGS
|
||||
.iter()
|
||||
.any(|needle| compact.contains(needle))
|
||||
}
|
||||
|
||||
/// Splits at non-alphanumeric characters and lower-to-upper case changes.
|
||||
fn split_words(name: &str) -> Vec<String> {
|
||||
let mut words = Vec::new();
|
||||
let mut current = String::new();
|
||||
let mut previous_lower = false;
|
||||
for c in name.chars() {
|
||||
if !c.is_ascii_alphanumeric() {
|
||||
if !current.is_empty() {
|
||||
words.push(std::mem::take(&mut current));
|
||||
}
|
||||
previous_lower = false;
|
||||
continue;
|
||||
}
|
||||
if c.is_ascii_uppercase() && previous_lower && !current.is_empty() {
|
||||
words.push(std::mem::take(&mut current));
|
||||
}
|
||||
previous_lower = c.is_ascii_lowercase() || c.is_ascii_digit();
|
||||
current.push(c.to_ascii_lowercase());
|
||||
}
|
||||
if !current.is_empty() {
|
||||
words.push(current);
|
||||
}
|
||||
words
|
||||
}
|
||||
|
||||
/// A long token-alphabet string mixing letters and digits (or both letter
|
||||
/// cases) with high per-character entropy.
|
||||
fn looks_random(value: &str, min_chars: usize) -> bool {
|
||||
if value.chars().count() < min_chars
|
||||
|| !value
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || "-_.~%=+".contains(c))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let has_digit = value.chars().any(|c| c.is_ascii_digit());
|
||||
let has_lower = value.chars().any(|c| c.is_ascii_lowercase());
|
||||
let has_upper = value.chars().any(|c| c.is_ascii_uppercase());
|
||||
let classes = [has_digit, has_lower, has_upper]
|
||||
.into_iter()
|
||||
.filter(|present| *present)
|
||||
.count();
|
||||
classes >= 2 && shannon_entropy(value) >= MIN_SECRET_ENTROPY_BITS
|
||||
}
|
||||
|
||||
fn shannon_entropy(value: &str) -> f64 {
|
||||
let mut counts: HashMap<char, usize> = HashMap::new();
|
||||
for c in value.chars() {
|
||||
*counts.entry(c).or_default() += 1;
|
||||
}
|
||||
let total = value.chars().count() as f64;
|
||||
counts
|
||||
.values()
|
||||
.map(|count| {
|
||||
let p = *count as f64 / total;
|
||||
-p * p.log2()
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn warn(url: &str) -> Option<FeedUrlPrivacyWarning> {
|
||||
feed_url_privacy_warning(&FeedUrl::parse(url).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_feeds_have_no_warning() {
|
||||
for url in [
|
||||
"https://feeds.example.com/podcast.xml",
|
||||
"https://example.com/feed?format=rss&page=2",
|
||||
"https://anchor.example/s/abc123de/podcast/rss",
|
||||
"https://feeds.example.com/12345/rss",
|
||||
"https://example.com/feed?author=jane",
|
||||
"https://example.com:8443/shows/my-great-show/feed.xml",
|
||||
] {
|
||||
assert_eq!(warn(url), None, "{url}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_query_names_warn() {
|
||||
for (url, name) in [
|
||||
("https://example.com/feed?token=abc", "token"),
|
||||
("https://example.com/feed?auth_token=abc", "auth_token"),
|
||||
(
|
||||
"https://example.com/feed?X-Amz-Signature=abc",
|
||||
"X-Amz-Signature",
|
||||
),
|
||||
("https://example.com/feed?accessKey=abc", "accessKey"),
|
||||
("https://example.com/feed?a=1&uid=42", "uid"),
|
||||
("https://example.com/feed?apitoken=x", "apitoken"),
|
||||
("https://example.com/feed?PRIVATE=1", "PRIVATE"),
|
||||
("https://example.com/feed?session_id=1", "session_id"),
|
||||
] {
|
||||
assert_eq!(
|
||||
warn(url),
|
||||
Some(FeedUrlPrivacyWarning::SecretQueryParameter { name: name.into() }),
|
||||
"{url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_member_hosts_warn() {
|
||||
assert_eq!(
|
||||
warn("https://www.patreon.com/rss/show"),
|
||||
Some(FeedUrlPrivacyWarning::PrivateFeedHost {
|
||||
host: "www.patreon.com".into()
|
||||
})
|
||||
);
|
||||
assert_eq!(warn("https://notpatreon.com/rss"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn high_entropy_values_and_segments_warn() {
|
||||
assert_eq!(
|
||||
warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb"),
|
||||
Some(FeedUrlPrivacyWarning::HighEntropyQueryValue { name: "c".into() })
|
||||
);
|
||||
assert_eq!(warn("https://example.com/feed?c=aaaaaaaaaaaaaaaa1"), None);
|
||||
assert_eq!(
|
||||
warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB4sYc1HdWe9GuJa3F.rss"),
|
||||
Some(FeedUrlPrivacyWarning::HighEntropyPathSegment { index: 1 })
|
||||
);
|
||||
// Below the segment length: an ordinary short id.
|
||||
assert_eq!(warn("https://example.com/feeds/q8Zr2LmK0pXv7TnB.rss"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warnings_render_without_the_secret() {
|
||||
let warning = warn("https://example.com/feed?c=Zx81kQ2mP0aLr7Vb").unwrap();
|
||||
assert!(!warning.to_string().contains("Zx81"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
//! Podcast search hits and their merge with music search results.
|
||||
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::fmt;
|
||||
|
||||
use federation_net::EndpointId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::normalization::{normalize_name, tokenize};
|
||||
use crate::podcast::{FeedUrl, PodcastId};
|
||||
use crate::record::{ItemKind, LibraryItem};
|
||||
|
||||
use super::card::{MAX_INDEX_TOKEN_BYTES, PodcastCard, SignedPodcastRecord, compare_records};
|
||||
|
||||
/// Maximum distinct query tokens used by a podcast text search.
|
||||
pub const MAX_PODCAST_QUERY_TOKENS: usize = 8;
|
||||
/// Maximum publishers reported per hit and per claimed feed URL.
|
||||
pub const MAX_HIT_PUBLISHERS: usize = 64;
|
||||
/// Maximum claimed moved feed URLs reported per hit.
|
||||
pub const MAX_HIT_FEED_URL_CLAIMS: usize = 8;
|
||||
|
||||
/// Kind of a federated search hit.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum SearchHitKind {
|
||||
/// A musical artist.
|
||||
Artist,
|
||||
/// A release.
|
||||
Release,
|
||||
/// A track.
|
||||
Track,
|
||||
/// A public podcast card.
|
||||
Podcast,
|
||||
}
|
||||
|
||||
impl SearchHitKind {
|
||||
/// Stable lowercase name.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Artist => "artist",
|
||||
Self::Release => "release",
|
||||
Self::Track => "track",
|
||||
Self::Podcast => "podcast",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for SearchHitKind {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ItemKind> for SearchHitKind {
|
||||
fn from(kind: ItemKind) -> Self {
|
||||
match kind {
|
||||
ItemKind::Artist => Self::Artist,
|
||||
ItemKind::Release => Self::Release,
|
||||
ItemKind::Track => Self::Track,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One public podcast found in the discovery overlay.
|
||||
///
|
||||
/// Cards of the same podcast published by several devices are merged by
|
||||
/// `podcast_id`. The hit carries no personal state of any publisher.
|
||||
///
|
||||
/// Only [`Self::feed_url`] is bound to the podcast: the id is its hash. Every
|
||||
/// other field is a claim of some publisher device, and device identities are
|
||||
/// free, so neither the metadata nor the number of publishers is a trust
|
||||
/// signal.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PodcastSearchHit {
|
||||
/// Podcast.
|
||||
pub podcast_id: PodcastId,
|
||||
/// The URL `podcast_id` is derived from
|
||||
/// (`PodcastId::from_feed_url(feed_url) == podcast_id`). Subscribe with
|
||||
/// this URL; follow the feed's own HTTP redirects and
|
||||
/// `itunes:new-feed-url` from there.
|
||||
pub feed_url: FeedUrl,
|
||||
/// Representative card for display: a card whose `feed_url` is the
|
||||
/// origin URL when one exists, then the title most publishers agree on,
|
||||
/// then the smallest publisher key. Its metadata is a publisher's claim.
|
||||
pub card: PodcastCard,
|
||||
/// Sorted distinct publisher devices with a current valid card, at most
|
||||
/// [`MAX_HIT_PUBLISHERS`]. These are devices, not users or subscribers:
|
||||
/// one user may publish from several devices, and anyone can mint
|
||||
/// devices.
|
||||
pub publishers: Vec<EndpointId>,
|
||||
/// Feed URLs other than [`Self::feed_url`] that publishers claim the
|
||||
/// podcast moved to, most claimed first, at most
|
||||
/// [`MAX_HIT_FEED_URL_CLAIMS`]. Unverified: never fetch one as this
|
||||
/// podcast or record it as an alias unless the feed at
|
||||
/// [`Self::feed_url`] itself redirects there.
|
||||
pub claimed_feed_urls: Vec<ClaimedFeedUrl>,
|
||||
}
|
||||
|
||||
impl PodcastSearchHit {
|
||||
/// Always [`SearchHitKind::Podcast`].
|
||||
pub fn kind(&self) -> SearchHitKind {
|
||||
SearchHitKind::Podcast
|
||||
}
|
||||
}
|
||||
|
||||
/// An unverified moved-feed URL claimed by some publishers of a podcast.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ClaimedFeedUrl {
|
||||
/// Claimed current feed URL.
|
||||
pub feed_url: FeedUrl,
|
||||
/// Sorted distinct publisher devices claiming it, at most
|
||||
/// [`MAX_HIT_PUBLISHERS`].
|
||||
pub publishers: Vec<EndpointId>,
|
||||
}
|
||||
|
||||
/// A music or podcast result of a federated search.
|
||||
///
|
||||
/// Serialized as `{"type": "music" | "podcast", "hit": {...}}`; use
|
||||
/// [`FederatedSearchHit::kind`] for the fine-grained kind.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", content = "hit", rename_all = "snake_case")]
|
||||
pub enum FederatedSearchHit {
|
||||
/// A music-DHT record (artist, release or track).
|
||||
Music(LibraryItem),
|
||||
/// A public podcast.
|
||||
Podcast(PodcastSearchHit),
|
||||
}
|
||||
|
||||
impl FederatedSearchHit {
|
||||
/// Kind of the hit: `artist`, `release`, `track` or `podcast`.
|
||||
pub fn kind(&self) -> SearchHitKind {
|
||||
match self {
|
||||
Self::Music(item) => item.kind.into(),
|
||||
Self::Podcast(_) => SearchHitKind::Podcast,
|
||||
}
|
||||
}
|
||||
|
||||
/// Display name: the item name or the podcast title.
|
||||
pub fn title(&self) -> &str {
|
||||
match self {
|
||||
Self::Music(item) => &item.name,
|
||||
Self::Podcast(hit) => &hit.card.title,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges music results (as returned by `MusicDhtService::search_network`)
|
||||
/// with podcast hits for the same query.
|
||||
///
|
||||
/// Results whose normalized name equals the normalized query come first;
|
||||
/// within each tier music keeps its order and precedes podcasts, which keep
|
||||
/// theirs. Existing music consumers that only read `SearchOutcome` are
|
||||
/// unaffected.
|
||||
pub fn merge_search_results(
|
||||
query: &str,
|
||||
music: Vec<LibraryItem>,
|
||||
podcasts: Vec<PodcastSearchHit>,
|
||||
) -> Vec<FederatedSearchHit> {
|
||||
let normalized = normalize_name(query);
|
||||
let mut exact = Vec::new();
|
||||
let mut rest = Vec::new();
|
||||
let hits = music
|
||||
.into_iter()
|
||||
.map(FederatedSearchHit::Music)
|
||||
.chain(podcasts.into_iter().map(FederatedSearchHit::Podcast));
|
||||
for hit in hits {
|
||||
let is_exact = !normalized.is_empty() && normalize_name(hit.title()) == normalized;
|
||||
if is_exact {
|
||||
exact.push(hit);
|
||||
} else {
|
||||
rest.push(hit);
|
||||
}
|
||||
}
|
||||
exact.extend(rest);
|
||||
exact
|
||||
}
|
||||
|
||||
/// Distinct normalized query tokens, at most [`MAX_PODCAST_QUERY_TOKENS`].
|
||||
pub fn podcast_query_tokens(query: &str) -> Vec<String> {
|
||||
let mut seen = HashSet::new();
|
||||
tokenize(&normalize_name(query))
|
||||
.into_iter()
|
||||
.filter(|token| token.len() <= MAX_INDEX_TOKEN_BYTES && seen.insert(token.clone()))
|
||||
.take(MAX_PODCAST_QUERY_TOKENS)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Keeps the winning record per `(podcast_id, publisher)` and groups the
|
||||
/// cards that are still active by podcast. Records must be verified.
|
||||
pub(crate) fn aggregate_hits(
|
||||
records: impl IntoIterator<Item = SignedPodcastRecord>,
|
||||
) -> Vec<PodcastSearchHit> {
|
||||
let mut winners: HashMap<(PodcastId, EndpointId), SignedPodcastRecord> = HashMap::new();
|
||||
for record in records {
|
||||
let owner = (record.podcast_id().clone(), record.publisher());
|
||||
match winners.get(&owner) {
|
||||
Some(current) if compare_records(current, &record).is_ge() => {}
|
||||
_ => {
|
||||
winners.insert(owner, record);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut by_podcast: BTreeMap<PodcastId, Vec<PodcastCard>> = BTreeMap::new();
|
||||
for record in winners.into_values() {
|
||||
if let Some(card) = record.card() {
|
||||
by_podcast
|
||||
.entry(card.podcast_id.clone())
|
||||
.or_default()
|
||||
.push(card.clone());
|
||||
}
|
||||
}
|
||||
by_podcast
|
||||
.into_iter()
|
||||
.map(|(podcast_id, cards)| {
|
||||
let mut title_votes: HashMap<String, usize> = HashMap::new();
|
||||
for card in &cards {
|
||||
*title_votes.entry(normalize_name(&card.title)).or_default() += 1;
|
||||
}
|
||||
// Publisher-controlled times and revisions never break ties.
|
||||
let card = cards
|
||||
.iter()
|
||||
.max_by_key(|card| {
|
||||
(
|
||||
card.feed_url_is_origin(),
|
||||
title_votes[&normalize_name(&card.title)],
|
||||
Reverse(*card.publisher.as_bytes()),
|
||||
)
|
||||
})
|
||||
.cloned()
|
||||
.expect("a group has at least one card");
|
||||
let mut claims: BTreeMap<&FeedUrl, Vec<EndpointId>> = BTreeMap::new();
|
||||
for card in cards.iter().filter(|card| !card.feed_url_is_origin()) {
|
||||
claims
|
||||
.entry(&card.feed_url)
|
||||
.or_default()
|
||||
.push(card.publisher);
|
||||
}
|
||||
let mut claimed_feed_urls: Vec<ClaimedFeedUrl> = claims
|
||||
.into_iter()
|
||||
.map(|(feed_url, publishers)| ClaimedFeedUrl {
|
||||
feed_url: feed_url.clone(),
|
||||
publishers: sorted_publishers(publishers),
|
||||
})
|
||||
.collect();
|
||||
claimed_feed_urls.sort_by(|left, right| {
|
||||
right
|
||||
.publishers
|
||||
.len()
|
||||
.cmp(&left.publishers.len())
|
||||
.then_with(|| left.feed_url.cmp(&right.feed_url))
|
||||
});
|
||||
claimed_feed_urls.truncate(MAX_HIT_FEED_URL_CLAIMS);
|
||||
PodcastSearchHit {
|
||||
podcast_id,
|
||||
feed_url: card.origin_feed_url().clone(),
|
||||
publishers: sorted_publishers(cards.iter().map(|card| card.publisher).collect()),
|
||||
card,
|
||||
claimed_feed_urls,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn sorted_publishers(mut publishers: Vec<EndpointId>) -> Vec<EndpointId> {
|
||||
publishers.sort_by_key(|publisher| *publisher.as_bytes());
|
||||
publishers.dedup();
|
||||
publishers.truncate(MAX_HIT_PUBLISHERS);
|
||||
publishers
|
||||
}
|
||||
|
||||
/// Keeps hits whose card is indexed under every query token and ranks
|
||||
/// them: exact normalized title first, then hits with a card published under
|
||||
/// the origin feed URL, then title and id. The publisher count is not used:
|
||||
/// anyone can inflate it with new device identities.
|
||||
pub(crate) fn rank_hits(
|
||||
mut hits: Vec<PodcastSearchHit>,
|
||||
query: &str,
|
||||
tokens: &[String],
|
||||
) -> Vec<PodcastSearchHit> {
|
||||
let normalized_query = normalize_name(query);
|
||||
hits.retain(|hit| {
|
||||
let indexed = hit.card.index_tokens();
|
||||
tokens.iter().all(|token| indexed.contains(token))
|
||||
});
|
||||
hits.sort_by_cached_key(|hit| {
|
||||
let title = normalize_name(&hit.card.title);
|
||||
(
|
||||
title != normalized_query,
|
||||
!hit.card.feed_url_is_origin(),
|
||||
title,
|
||||
hit.podcast_id.clone(),
|
||||
)
|
||||
});
|
||||
hits
|
||||
}
|
||||
@@ -0,0 +1,891 @@
|
||||
//! SQLite cache of the podcast-discovery overlay: replicas, routing contacts
|
||||
//! and this device's own publications.
|
||||
//!
|
||||
//! The cache is replaceable federation state, kept in its own database file
|
||||
//! (like the similarity-routing cache). Every record read from it is decoded
|
||||
//! and verified again before it is used or served.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::str::FromStr;
|
||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||
|
||||
use federation_net::{EndpointId, NetworkId};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
use crate::error::{MusicDhtError, Result};
|
||||
use crate::podcast::PodcastId;
|
||||
use crate::routing::{NodeContact, NodeId};
|
||||
|
||||
use super::MAX_PODCAST_DHT_RECORDS;
|
||||
use super::card::{PodcastCardSpec, PodcastDhtKey, SignedPodcastRecord, compare_records};
|
||||
|
||||
/// `PRAGMA user_version` of the podcast-discovery cache schema. Version 1
|
||||
/// (pre-release) stored one payload per key; its replica table is dropped on
|
||||
/// open because the cache is replaceable. Own publications are kept.
|
||||
pub const PODCAST_DHT_CACHE_SCHEMA_VERSION: i32 = 2;
|
||||
/// Maximum records (cards and tombstones) indexed under one key. It equals
|
||||
/// [`MAX_PODCAST_DHT_RECORDS`], so one `FindValue` response carries every
|
||||
/// record of a key: nothing admitted is unreachable.
|
||||
///
|
||||
/// A full key admits a newcomer by evicting a random placement from its
|
||||
/// younger half (by first indexing time). The older half is never evicted
|
||||
/// for newcomers, so cards that were there before a flood stay, and the
|
||||
/// younger half stays contestable, so nobody who stored first holds the key
|
||||
/// for good.
|
||||
pub const MAX_PODCAST_RECORDS_PER_KEY: usize = MAX_PODCAST_DHT_RECORDS;
|
||||
/// Maximum tombstones indexed under one key, including tombstones that
|
||||
/// replaced their owner's card in place. Over budget, a random tombstone of
|
||||
/// another owner from the younger half of the key's tombstones is evicted.
|
||||
/// It equals [`MAX_PODCAST_RESPONSE_TOMBSTONES`], so every tombstone of a key
|
||||
/// is served.
|
||||
pub const MAX_PODCAST_TOMBSTONES_PER_KEY: usize = MAX_PODCAST_RESPONSE_TOMBSTONES;
|
||||
/// Maximum tombstones in one `FindValue` response; cards fill the rest.
|
||||
pub const MAX_PODCAST_RESPONSE_TOMBSTONES: usize = 32;
|
||||
/// Maximum `(podcast_id, publisher)` records one publisher device keeps in a
|
||||
/// replica. A publisher over it is refused (nothing is evicted for it).
|
||||
pub const MAX_PODCAST_RECORDS_PER_PUBLISHER: usize = 256;
|
||||
/// Maximum `(podcast_id, publisher)` records in a replica cache. A full cache
|
||||
/// admits a new record by evicting random records from the younger half of
|
||||
/// the cache (by first indexing time), like a full key.
|
||||
pub const MAX_PODCAST_CACHE_RECORDS: usize = 4_096;
|
||||
/// Maximum total encoded size of the records in a replica cache, enforced
|
||||
/// like [`MAX_PODCAST_CACHE_RECORDS`].
|
||||
pub const MAX_PODCAST_CACHE_BYTES: usize = 32 * 1024 * 1024;
|
||||
/// Maximum persisted overlay contacts; the least recently seen are dropped.
|
||||
pub const MAX_PERSISTED_PODCAST_DHT_PEERS: usize = 1_024;
|
||||
/// Maximum records evicted from the cache to admit one record; when that is
|
||||
/// not enough room, the record is refused and nothing is evicted.
|
||||
const MAX_EVICTIONS_PER_RECORD: usize = 16;
|
||||
|
||||
const _: () = assert!(
|
||||
MAX_PODCAST_RECORDS_PER_KEY <= MAX_PODCAST_DHT_RECORDS
|
||||
&& MAX_PODCAST_TOMBSTONES_PER_KEY <= MAX_PODCAST_RESPONSE_TOMBSTONES
|
||||
&& MAX_PODCAST_RESPONSE_TOMBSTONES <= MAX_PODCAST_DHT_RECORDS,
|
||||
"every record admitted under a key must fit one FindValue response"
|
||||
);
|
||||
|
||||
const SCHEMA: &str = "
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_records (
|
||||
podcast_id TEXT NOT NULL,
|
||||
publisher TEXT NOT NULL,
|
||||
tombstone INTEGER NOT NULL,
|
||||
expires_at_ms INTEGER NOT NULL,
|
||||
payload_bytes INTEGER NOT NULL,
|
||||
payload BLOB NOT NULL,
|
||||
PRIMARY KEY (podcast_id, publisher)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_expiry
|
||||
ON podcast_dht_records(expires_at_ms);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_records_publisher
|
||||
ON podcast_dht_records(publisher);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_keys (
|
||||
dht_key BLOB NOT NULL,
|
||||
podcast_id TEXT NOT NULL,
|
||||
publisher TEXT NOT NULL,
|
||||
tombstone INTEGER NOT NULL,
|
||||
expires_at_ms INTEGER NOT NULL,
|
||||
first_stored_at_ms INTEGER NOT NULL,
|
||||
PRIMARY KEY (dht_key, podcast_id, publisher)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_owner
|
||||
ON podcast_dht_keys(podcast_id, publisher);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_dht_keys_expiry
|
||||
ON podcast_dht_keys(expires_at_ms);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_peers (
|
||||
peer_id TEXT PRIMARY KEY,
|
||||
node_id BLOB NOT NULL,
|
||||
ticket TEXT NOT NULL,
|
||||
last_seen_ms INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS podcast_dht_local (
|
||||
podcast_id TEXT PRIMARY KEY,
|
||||
revision INTEGER NOT NULL,
|
||||
spec BLOB,
|
||||
record BLOB NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
";
|
||||
|
||||
/// Admission limits of a replica cache.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct StoreLimits {
|
||||
pub per_key: usize,
|
||||
pub tombstones_per_key: usize,
|
||||
pub per_publisher: usize,
|
||||
pub records: usize,
|
||||
pub bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for StoreLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
per_key: MAX_PODCAST_RECORDS_PER_KEY,
|
||||
tombstones_per_key: MAX_PODCAST_TOMBSTONES_PER_KEY,
|
||||
per_publisher: MAX_PODCAST_RECORDS_PER_PUBLISHER,
|
||||
records: MAX_PODCAST_CACHE_RECORDS,
|
||||
bytes: MAX_PODCAST_CACHE_BYTES,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This device's latest publication of one podcast.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct LocalPublication {
|
||||
pub podcast_id: PodcastId,
|
||||
/// Greatest revision ever used for this podcast by this device.
|
||||
pub revision: u64,
|
||||
/// Published card content; `None` once unpublished.
|
||||
pub spec: Option<PodcastCardSpec>,
|
||||
/// Latest signed record (card or tombstone).
|
||||
pub record: SignedPodcastRecord,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct PodcastDhtDatabase {
|
||||
conn: Arc<Mutex<Connection>>,
|
||||
network_id: NetworkId,
|
||||
limits: StoreLimits,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PodcastDhtDatabase {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("PodcastDhtDatabase")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl PodcastDhtDatabase {
|
||||
pub async fn open(path: &Path, network_id: NetworkId) -> Result<Self> {
|
||||
Self::open_with_limits(path, network_id, StoreLimits::default()).await
|
||||
}
|
||||
|
||||
pub async fn open_with_limits(
|
||||
path: &Path,
|
||||
network_id: NetworkId,
|
||||
limits: StoreLimits,
|
||||
) -> Result<Self> {
|
||||
let path = path.to_path_buf();
|
||||
let conn = tokio::task::spawn_blocking(move || -> Result<Connection> {
|
||||
let conn = Connection::open(&path).map_err(|error| {
|
||||
MusicDhtError::Database(format!("failed to open {}: {error}", path.display()))
|
||||
})?;
|
||||
let version: i32 = conn
|
||||
.query_row("PRAGMA user_version", [], |row| row.get(0))
|
||||
.map_err(database_error)?;
|
||||
if version > PODCAST_DHT_CACHE_SCHEMA_VERSION {
|
||||
return Err(MusicDhtError::Database(format!(
|
||||
"podcast DHT cache schema {version} is newer than supported {PODCAST_DHT_CACHE_SCHEMA_VERSION}"
|
||||
)));
|
||||
}
|
||||
if version == 1 {
|
||||
// Replicas are replaceable; the old layout is dropped.
|
||||
conn.execute_batch("DROP TABLE IF EXISTS podcast_dht_records;")
|
||||
.map_err(database_error)?;
|
||||
}
|
||||
conn.execute_batch(SCHEMA).map_err(|error| {
|
||||
MusicDhtError::Database(format!("failed to initialize podcast DHT cache: {error}"))
|
||||
})?;
|
||||
conn.pragma_update(None, "user_version", PODCAST_DHT_CACHE_SCHEMA_VERSION)
|
||||
.map_err(database_error)?;
|
||||
Ok(conn)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))??;
|
||||
Ok(Self {
|
||||
conn: Arc::new(Mutex::new(conn)),
|
||||
network_id,
|
||||
limits,
|
||||
})
|
||||
}
|
||||
|
||||
async fn call<F, T>(&self, operation: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce(&Connection, &NetworkId, &StoreLimits) -> rusqlite::Result<T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let conn = Arc::clone(&self.conn);
|
||||
let network_id = self.network_id;
|
||||
let limits = self.limits;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let guard = lock(&conn);
|
||||
operation(&guard, &network_id, &limits).map_err(database_error)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| MusicDhtError::Database(format!("database task panicked: {error}")))?
|
||||
}
|
||||
|
||||
/// Stores records under the given keys. The caller has verified every
|
||||
/// record, checked that each key belongs to it and that the sender is its
|
||||
/// publisher. Returns the number of `(key, record)` placements written.
|
||||
pub async fn store(
|
||||
&self,
|
||||
entries: Vec<(Vec<PodcastDhtKey>, SignedPodcastRecord)>,
|
||||
now_ms: u64,
|
||||
) -> Result<usize> {
|
||||
self.call(move |conn, network_id, limits| {
|
||||
let transaction = conn.unchecked_transaction()?;
|
||||
let mut written = 0;
|
||||
for (keys, record) in entries {
|
||||
written += store_one(&transaction, network_id, limits, &keys, &record, now_ms)?;
|
||||
}
|
||||
transaction.commit()?;
|
||||
Ok(written)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Valid, unexpired records under `key`: at most
|
||||
/// [`MAX_PODCAST_RESPONSE_TOMBSTONES`] tombstones (always returned so stale
|
||||
/// replicas elsewhere can be overruled) and cards containing every token
|
||||
/// in `filter_tokens` up to `limit` records in all. Records indexed longest
|
||||
/// come first. Cards precede tombstones. With the default limits and
|
||||
/// `limit` = [`MAX_PODCAST_DHT_RECORDS`], every record of the key fits.
|
||||
pub async fn records_by_key(
|
||||
&self,
|
||||
key: PodcastDhtKey,
|
||||
filter_tokens: Vec<String>,
|
||||
now_ms: u64,
|
||||
limit: usize,
|
||||
) -> Result<Vec<SignedPodcastRecord>> {
|
||||
self.call(move |conn, network_id, _| {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT r.payload FROM podcast_dht_keys k
|
||||
JOIN podcast_dht_records r
|
||||
ON r.podcast_id = k.podcast_id AND r.publisher = k.publisher
|
||||
WHERE k.dht_key = ?1 AND k.tombstone = ?2 AND k.expires_at_ms > ?3
|
||||
ORDER BY k.first_stored_at_ms, k.podcast_id, k.publisher",
|
||||
)?;
|
||||
let mut select = |tombstone: bool, limit: usize| -> rusqlite::Result<Vec<_>> {
|
||||
let mut records = Vec::new();
|
||||
if limit == 0 {
|
||||
return Ok(records);
|
||||
}
|
||||
let rows = statement.query_map(
|
||||
params![key.as_bytes().as_slice(), tombstone, clamp_i64(now_ms)],
|
||||
|row| row.get::<_, Vec<u8>>(0),
|
||||
)?;
|
||||
for row in rows {
|
||||
let Some(record) = decode_valid(&row?, network_id, now_ms) else {
|
||||
continue;
|
||||
};
|
||||
if record.is_tombstone() != tombstone || !record.belongs_under(&key) {
|
||||
continue;
|
||||
}
|
||||
let matches = record.card().is_none_or(|card| {
|
||||
let tokens = card.index_tokens();
|
||||
filter_tokens.iter().all(|token| tokens.contains(token))
|
||||
});
|
||||
if matches {
|
||||
records.push(record);
|
||||
if records.len() == limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(records)
|
||||
};
|
||||
let tombstones = select(true, MAX_PODCAST_RESPONSE_TOMBSTONES.min(limit))?;
|
||||
let mut records = select(false, limit - tombstones.len())?;
|
||||
records.extend(tombstones);
|
||||
Ok(records)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Removes expired records and their key placements. Returns the number
|
||||
/// of records removed.
|
||||
pub async fn delete_expired(&self, now_ms: u64) -> Result<usize> {
|
||||
self.call(move |conn, _, _| {
|
||||
let now = clamp_i64(now_ms);
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys WHERE expires_at_ms <= ?1",
|
||||
[now],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records WHERE expires_at_ms <= ?1",
|
||||
[now],
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn upsert_peer(&self, contact: NodeContact) -> Result<()> {
|
||||
self.call(move |conn, _, _| {
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO podcast_dht_peers
|
||||
(peer_id, node_id, ticket, last_seen_ms)
|
||||
VALUES (?1, ?2, ?3, ?4)",
|
||||
params![
|
||||
contact.peer_id.to_string(),
|
||||
contact.node_id.as_bytes().as_slice(),
|
||||
contact.ticket,
|
||||
clamp_i64(contact.last_seen_ms),
|
||||
],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_peers WHERE peer_id IN (
|
||||
SELECT peer_id FROM podcast_dht_peers
|
||||
ORDER BY last_seen_ms DESC, peer_id
|
||||
LIMIT -1 OFFSET ?1)",
|
||||
[MAX_PERSISTED_PODCAST_DHT_PEERS as i64],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn remove_peer(&self, peer: EndpointId) -> Result<()> {
|
||||
self.call(move |conn, _, _| {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_peers WHERE peer_id = ?1",
|
||||
[peer.to_string()],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn load_peers(&self) -> Result<Vec<NodeContact>> {
|
||||
self.call(move |conn, _, _| {
|
||||
let mut statement =
|
||||
conn.prepare("SELECT peer_id, ticket, last_seen_ms FROM podcast_dht_peers")?;
|
||||
let rows = statement.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, i64>(2)?,
|
||||
))
|
||||
})?;
|
||||
let mut peers = Vec::new();
|
||||
for row in rows {
|
||||
let (peer, ticket, last_seen_ms) = row?;
|
||||
let Ok(peer_id) = EndpointId::from_str(&peer) else {
|
||||
continue;
|
||||
};
|
||||
peers.push(NodeContact {
|
||||
node_id: NodeId::from_endpoint(&peer_id),
|
||||
peer_id,
|
||||
ticket,
|
||||
last_seen_ms: last_seen_ms.max(0) as u64,
|
||||
});
|
||||
}
|
||||
Ok(peers)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn local_publication(
|
||||
&self,
|
||||
podcast_id: PodcastId,
|
||||
) -> Result<Option<LocalPublication>> {
|
||||
self.call(move |conn, _, _| {
|
||||
conn.query_row(
|
||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||
WHERE podcast_id = ?1",
|
||||
[podcast_id.as_str()],
|
||||
local_from_row,
|
||||
)
|
||||
.optional()
|
||||
.map(Option::flatten)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn local_publications(&self) -> Result<Vec<LocalPublication>> {
|
||||
self.call(move |conn, _, _| {
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id, revision, spec, record FROM podcast_dht_local
|
||||
ORDER BY podcast_id",
|
||||
)?;
|
||||
let rows = statement.query_map([], local_from_row)?;
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
if let Some(publication) = row? {
|
||||
out.push(publication);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn put_local_publication(
|
||||
&self,
|
||||
publication: LocalPublication,
|
||||
now_ms: u64,
|
||||
) -> Result<()> {
|
||||
self.call(move |conn, _, _| {
|
||||
let spec = publication
|
||||
.spec
|
||||
.as_ref()
|
||||
.map(postcard::to_stdvec)
|
||||
.transpose()
|
||||
.map_err(to_sql_error)?;
|
||||
let record = postcard::to_stdvec(&publication.record).map_err(to_sql_error)?;
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO podcast_dht_local
|
||||
(podcast_id, revision, spec, record, updated_at_ms)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
params![
|
||||
publication.podcast_id.as_str(),
|
||||
publication.revision as i64,
|
||||
spec,
|
||||
record,
|
||||
clamp_i64(now_ms),
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn store_one(
|
||||
conn: &Connection,
|
||||
network_id: &NetworkId,
|
||||
limits: &StoreLimits,
|
||||
keys: &[PodcastDhtKey],
|
||||
record: &SignedPodcastRecord,
|
||||
now_ms: u64,
|
||||
) -> rusqlite::Result<usize> {
|
||||
let podcast_id = record.podcast_id().as_str().to_string();
|
||||
let publisher = record.publisher().to_string();
|
||||
let now = clamp_i64(now_ms);
|
||||
|
||||
// The cached record of this (podcast, publisher), re-verified.
|
||||
let cached: Option<(Vec<u8>, i64)> = conn
|
||||
.query_row(
|
||||
"SELECT payload, payload_bytes FROM podcast_dht_records
|
||||
WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![podcast_id, publisher],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()?;
|
||||
let (existing, existing_bytes) = match cached {
|
||||
Some((payload, bytes)) => match decode_valid(&payload, network_id, now_ms) {
|
||||
Some(existing) => (Some(existing), bytes.max(0) as usize),
|
||||
None => {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
(None, 0)
|
||||
}
|
||||
},
|
||||
None => {
|
||||
// Stray placements without a record.
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
(None, 0)
|
||||
}
|
||||
};
|
||||
|
||||
// Higher revision, tombstone at an equal revision, later republish...
|
||||
let replace = match &existing {
|
||||
Some(existing) => match compare_records(existing, record) {
|
||||
std::cmp::Ordering::Greater => return Ok(0),
|
||||
std::cmp::Ordering::Equal => false,
|
||||
std::cmp::Ordering::Less => true,
|
||||
},
|
||||
None => true,
|
||||
};
|
||||
let payload = postcard::to_stdvec(record).map_err(to_sql_error)?;
|
||||
if replace
|
||||
&& !make_room_for_record(
|
||||
conn,
|
||||
limits,
|
||||
(&podcast_id, &publisher),
|
||||
existing.is_none(),
|
||||
existing_bytes,
|
||||
payload.len(),
|
||||
now,
|
||||
)?
|
||||
{
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let mut indexed: HashSet<[u8; 32]> = HashSet::new();
|
||||
{
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT dht_key FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
)?;
|
||||
let rows = statement.query_map(params![podcast_id, publisher], |row| {
|
||||
row.get::<_, Vec<u8>>(0)
|
||||
})?;
|
||||
for row in rows {
|
||||
if let Ok(key) = <[u8; 32]>::try_from(row?.as_slice()) {
|
||||
indexed.insert(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut written = 0;
|
||||
if replace {
|
||||
// Placements the new version no longer belongs under are dropped (a
|
||||
// renamed card, or a tombstone that does not list the key); the others
|
||||
// now serve the new version.
|
||||
let belongs: HashSet<[u8; 32]> = record
|
||||
.dht_keys()
|
||||
.iter()
|
||||
.map(|key| *key.as_bytes())
|
||||
.collect();
|
||||
let stale: Vec<[u8; 32]> = indexed
|
||||
.iter()
|
||||
.filter(|key| !belongs.contains(*key))
|
||||
.copied()
|
||||
.collect();
|
||||
for key in stale {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key.as_slice(), podcast_id, publisher],
|
||||
)?;
|
||||
indexed.remove(&key);
|
||||
}
|
||||
written += conn.execute(
|
||||
"UPDATE podcast_dht_keys SET tombstone = ?3, expires_at_ms = ?4
|
||||
WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
],
|
||||
)?;
|
||||
// A card converted to a tombstone in place counts against the
|
||||
// tombstone budget of each key it stays under, like a new one.
|
||||
if record.is_tombstone() {
|
||||
for key in &indexed {
|
||||
let key = PodcastDhtKey::from_bytes(*key);
|
||||
make_room_under_key(conn, limits, &key, true, (&podcast_id, &publisher), now, 0)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// New placements. A tombstone only takes a new place under the podcast-id
|
||||
// key: under a text key it may only replace a record of its owner, so
|
||||
// nobody can index tombstones under someone else's search terms.
|
||||
let id_key = PodcastDhtKey::podcast(network_id, record.podcast_id());
|
||||
for key in keys {
|
||||
if indexed.contains(key.as_bytes()) {
|
||||
continue;
|
||||
}
|
||||
if record.is_tombstone() && *key != id_key {
|
||||
continue;
|
||||
}
|
||||
if !make_room_under_key(
|
||||
conn,
|
||||
limits,
|
||||
key,
|
||||
record.is_tombstone(),
|
||||
(&podcast_id, &publisher),
|
||||
now,
|
||||
1,
|
||||
)? {
|
||||
continue;
|
||||
}
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_dht_keys
|
||||
(dht_key, podcast_id, publisher, tombstone, expires_at_ms, first_stored_at_ms)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
params![
|
||||
key.as_bytes().as_slice(),
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
now,
|
||||
],
|
||||
)?;
|
||||
indexed.insert(*key.as_bytes());
|
||||
written += 1;
|
||||
}
|
||||
|
||||
if indexed.is_empty() {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
return Ok(0);
|
||||
}
|
||||
if replace {
|
||||
conn.execute(
|
||||
"INSERT INTO podcast_dht_records
|
||||
(podcast_id, publisher, tombstone, expires_at_ms, payload_bytes, payload)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(podcast_id, publisher) DO UPDATE SET
|
||||
tombstone = excluded.tombstone,
|
||||
expires_at_ms = excluded.expires_at_ms,
|
||||
payload_bytes = excluded.payload_bytes,
|
||||
payload = excluded.payload",
|
||||
params![
|
||||
podcast_id,
|
||||
publisher,
|
||||
record.is_tombstone(),
|
||||
clamp_i64(record.expires_at_ms()),
|
||||
payload.len() as i64,
|
||||
payload,
|
||||
],
|
||||
)?;
|
||||
}
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
/// Makes room in the cache for the new version of a record of `owner`.
|
||||
///
|
||||
/// * A publisher at [`StoreLimits::per_publisher`] gets no new record.
|
||||
/// * When the record count (for a new owner) or the byte budget would be
|
||||
/// exceeded, random records of other owners from the younger half of the
|
||||
/// cache (by first indexing time) are evicted, at most
|
||||
/// [`MAX_EVICTIONS_PER_RECORD`]. If that is not enough, nothing is evicted
|
||||
/// and the record is refused.
|
||||
fn make_room_for_record(
|
||||
conn: &Connection,
|
||||
limits: &StoreLimits,
|
||||
owner: (&str, &str),
|
||||
new_owner: bool,
|
||||
previous_bytes: usize,
|
||||
new_bytes: usize,
|
||||
now: i64,
|
||||
) -> rusqlite::Result<bool> {
|
||||
if new_bytes > limits.bytes {
|
||||
return Ok(false);
|
||||
}
|
||||
if new_owner {
|
||||
let own: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM podcast_dht_records WHERE publisher = ?1 AND expires_at_ms > ?2",
|
||||
params![owner.1, now],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if own.max(0) as usize >= limits.per_publisher {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
let (records, bytes): (i64, i64) = conn.query_row(
|
||||
"SELECT COUNT(*), COALESCE(SUM(payload_bytes), 0) FROM podcast_dht_records
|
||||
WHERE expires_at_ms > ?1",
|
||||
[now],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)?;
|
||||
let mut records = records.max(0) as usize;
|
||||
let mut bytes = (bytes.max(0) as usize).saturating_sub(previous_bytes);
|
||||
let fits = |records: usize, bytes: usize| {
|
||||
bytes.saturating_add(new_bytes) <= limits.bytes && (!new_owner || records < limits.records)
|
||||
};
|
||||
if fits(records, bytes) {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
// Candidates: the younger half of the other owners' records, shuffled.
|
||||
let others = records.saturating_sub(usize::from(!new_owner));
|
||||
let mut statement = conn.prepare(
|
||||
"SELECT podcast_id, publisher, payload_bytes FROM (
|
||||
SELECT r.podcast_id, r.publisher, r.payload_bytes,
|
||||
(SELECT MIN(k.first_stored_at_ms) FROM podcast_dht_keys k
|
||||
WHERE k.podcast_id = r.podcast_id AND k.publisher = r.publisher) AS since
|
||||
FROM podcast_dht_records r
|
||||
WHERE r.expires_at_ms > ?1 AND NOT (r.podcast_id = ?2 AND r.publisher = ?3)
|
||||
ORDER BY since DESC, random()
|
||||
LIMIT ?4)
|
||||
ORDER BY random() LIMIT ?5",
|
||||
)?;
|
||||
let candidates = statement
|
||||
.query_map(
|
||||
params![
|
||||
now,
|
||||
owner.0,
|
||||
owner.1,
|
||||
others.div_ceil(2) as i64,
|
||||
MAX_EVICTIONS_PER_RECORD as i64
|
||||
],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, i64>(2)?,
|
||||
))
|
||||
},
|
||||
)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let mut evict = Vec::new();
|
||||
for (podcast_id, publisher, payload_bytes) in candidates {
|
||||
if fits(records, bytes) {
|
||||
break;
|
||||
}
|
||||
records = records.saturating_sub(1);
|
||||
bytes = bytes.saturating_sub(payload_bytes.max(0) as usize);
|
||||
evict.push((podcast_id, publisher));
|
||||
}
|
||||
if !fits(records, bytes) {
|
||||
return Ok(false);
|
||||
}
|
||||
for (podcast_id, publisher) in evict {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Makes room under `key` for `incoming` more placements (0 when an existing
|
||||
/// placement of `owner` was converted to a tombstone in place) and returns
|
||||
/// whether the key has room.
|
||||
///
|
||||
/// Over [`StoreLimits::per_key`], or over [`StoreLimits::tombstones_per_key`]
|
||||
/// for a tombstone, a random placement of another owner from the younger
|
||||
/// half of the key (by first indexing time; only tombstones for the
|
||||
/// tombstone budget) is evicted. The older half is never evicted for a
|
||||
/// newcomer.
|
||||
fn make_room_under_key(
|
||||
conn: &Connection,
|
||||
limits: &StoreLimits,
|
||||
key: &PodcastDhtKey,
|
||||
tombstone: bool,
|
||||
owner: (&str, &str),
|
||||
now: i64,
|
||||
incoming: usize,
|
||||
) -> rusqlite::Result<bool> {
|
||||
for _ in 0..=MAX_EVICTIONS_PER_RECORD {
|
||||
let (total, tombstones): (i64, i64) = conn.query_row(
|
||||
"SELECT COUNT(*), COALESCE(SUM(tombstone), 0) FROM podcast_dht_keys
|
||||
WHERE dht_key = ?1 AND expires_at_ms > ?2",
|
||||
params![key.as_bytes().as_slice(), now],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)?;
|
||||
let over_tombstones =
|
||||
tombstone && tombstones.max(0) as usize + incoming > limits.tombstones_per_key;
|
||||
let over_total = total.max(0) as usize + incoming > limits.per_key;
|
||||
if !over_tombstones && !over_total {
|
||||
return Ok(true);
|
||||
}
|
||||
let Some((podcast_id, publisher)) =
|
||||
young_placement(conn, key, over_tombstones, owner, now)?
|
||||
else {
|
||||
return Ok(false);
|
||||
};
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys
|
||||
WHERE dht_key = ?1 AND podcast_id = ?2 AND publisher = ?3",
|
||||
params![key.as_bytes().as_slice(), podcast_id, publisher],
|
||||
)?;
|
||||
let placed: bool = conn.query_row(
|
||||
"SELECT EXISTS (SELECT 1 FROM podcast_dht_keys
|
||||
WHERE podcast_id = ?1 AND publisher = ?2)",
|
||||
params![podcast_id, publisher],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if !placed {
|
||||
delete_owner(conn, &podcast_id, &publisher)?;
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// A random unexpired placement of another owner than `owner` from the
|
||||
/// younger half of `key` (only tombstones with `tombstones_only`).
|
||||
fn young_placement(
|
||||
conn: &Connection,
|
||||
key: &PodcastDhtKey,
|
||||
tombstones_only: bool,
|
||||
owner: (&str, &str),
|
||||
now: i64,
|
||||
) -> rusqlite::Result<Option<(String, String)>> {
|
||||
let filter = "dht_key = ?1 AND expires_at_ms > ?2 AND (?3 = 0 OR tombstone = 1)
|
||||
AND NOT (podcast_id = ?4 AND publisher = ?5)";
|
||||
let count: i64 = conn.query_row(
|
||||
&format!("SELECT COUNT(*) FROM podcast_dht_keys WHERE {filter}"),
|
||||
params![
|
||||
key.as_bytes().as_slice(),
|
||||
now,
|
||||
tombstones_only,
|
||||
owner.0,
|
||||
owner.1
|
||||
],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if count <= 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
conn.query_row(
|
||||
&format!(
|
||||
"SELECT podcast_id, publisher FROM (
|
||||
SELECT podcast_id, publisher FROM podcast_dht_keys WHERE {filter}
|
||||
ORDER BY first_stored_at_ms DESC, random() LIMIT ?6)
|
||||
ORDER BY random() LIMIT 1"
|
||||
),
|
||||
params![
|
||||
key.as_bytes().as_slice(),
|
||||
now,
|
||||
tombstones_only,
|
||||
owner.0,
|
||||
owner.1,
|
||||
(count as usize).div_ceil(2) as i64
|
||||
],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()
|
||||
}
|
||||
|
||||
fn delete_owner(conn: &Connection, podcast_id: &str, publisher: &str) -> rusqlite::Result<()> {
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_keys WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![podcast_id, publisher],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM podcast_dht_records WHERE podcast_id = ?1 AND publisher = ?2",
|
||||
params![podcast_id, publisher],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_valid(
|
||||
payload: &[u8],
|
||||
network_id: &NetworkId,
|
||||
now_ms: u64,
|
||||
) -> Option<SignedPodcastRecord> {
|
||||
let record: SignedPodcastRecord = postcard::from_bytes(payload).ok()?;
|
||||
record.verify_at(network_id, now_ms).ok()?;
|
||||
Some(record)
|
||||
}
|
||||
|
||||
fn local_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Option<LocalPublication>> {
|
||||
let podcast_id: String = row.get(0)?;
|
||||
let revision: i64 = row.get(1)?;
|
||||
let spec: Option<Vec<u8>> = row.get(2)?;
|
||||
let record: Vec<u8> = row.get(3)?;
|
||||
let Ok(podcast_id) = PodcastId::parse(&podcast_id) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let spec = match spec {
|
||||
Some(bytes) => match postcard::from_bytes::<PodcastCardSpec>(&bytes) {
|
||||
Ok(spec) => Some(spec),
|
||||
Err(_) => return Ok(None),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
let Ok(record) = postcard::from_bytes::<SignedPodcastRecord>(&record) else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(LocalPublication {
|
||||
podcast_id,
|
||||
revision: revision as u64,
|
||||
spec,
|
||||
record,
|
||||
}))
|
||||
}
|
||||
|
||||
fn clamp_i64(value: u64) -> i64 {
|
||||
value.min(i64::MAX as u64) as i64
|
||||
}
|
||||
|
||||
fn to_sql_error(error: impl std::error::Error + Send + Sync + 'static) -> rusqlite::Error {
|
||||
rusqlite::Error::ToSqlConversionFailure(Box::new(error))
|
||||
}
|
||||
|
||||
fn database_error(error: rusqlite::Error) -> MusicDhtError {
|
||||
MusicDhtError::Database(error.to_string())
|
||||
}
|
||||
|
||||
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
|
||||
mutex.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,763 @@
|
||||
//! Integration test: public podcast discovery on three real loopback iroh
|
||||
//! nodes.
|
||||
//!
|
||||
//! * A publishes a podcast its user shared. B is another device of the same
|
||||
//! user: it applies the same merged registers (as device sync would deliver
|
||||
//! them) and publishes the same podcast under its own key. C is an
|
||||
//! unrelated peer outside the user's trusted-device group, with no
|
||||
//! device-sync pairing.
|
||||
//! * C finds the podcast by text and by id, merged over publishers {A, B};
|
||||
//! once B unshares and goes offline, C still finds A's card, so B is
|
||||
//! irrelevant to discovery. A podcast the user never shared (held by both
|
||||
//! A and B) is never discoverable; after A unpublishes, the tombstone hides
|
||||
//! the card from C.
|
||||
//! * Forged, foreign, tampered, replayed, malformed, oversized and
|
||||
//! wrong-version requests are rejected by a real peer.
|
||||
//! * D, a music-DHT peer without the overlay, is dialed by bootstrap but
|
||||
//! never answers on the overlay ALPN, so it never becomes an overlay contact
|
||||
//! and is never asked for records or to store them.
|
||||
//! * `sync_shares` stays fail-safe for revocation and holds back URLs that
|
||||
//! look private (single node).
|
||||
//! * Contacts a peer returns are never trusted: fabricated, unreachable
|
||||
//! contacts do not enter the routing table (two nodes).
|
||||
//! * After a restart, own cards persisted by the previous run are neither
|
||||
//! republished nor served until the first `sync_shares`, so a share
|
||||
//! revoked on another device meanwhile is never re-sent (two nodes).
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use music_dht::capabilities::{CapabilityManifest, PODCAST_DHT_ID};
|
||||
use music_dht::podcast::{FeedUrl, PodcastId, PodcastMeta, ShareRecord, SubscriptionRecord};
|
||||
use music_dht::podcast_dht::{
|
||||
MAX_PODCAST_DHT_REQUEST_BYTES, PODCAST_DHT_ALPN, PODCAST_DHT_PROTOCOL_VERSION, PodcastCard,
|
||||
PodcastCardSpec, PodcastDht, PodcastDhtOperation, PodcastDhtRequest, PodcastDhtResponse,
|
||||
PodcastRecordBody, PodcastSearchHit, PodcastShareState, PodcastStoreEntry, SignedPodcastRecord,
|
||||
};
|
||||
use music_dht::{MusicDhtConfig, MusicDhtError, MusicDhtService, NetworkId, NodeContact, NodeId};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const TEST_TIMEOUT: Duration = Duration::from_secs(240);
|
||||
|
||||
struct Node {
|
||||
service: Arc<MusicDhtService>,
|
||||
dht: Arc<PodcastDht>,
|
||||
tasks: Vec<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl Node {
|
||||
async fn start(directory: &Path, network: NetworkId) -> Self {
|
||||
let (service, events) = start_service(directory, network, true).await;
|
||||
let acceptor = service.stream_acceptor(PODCAST_DHT_ALPN).unwrap();
|
||||
let dht = PodcastDht::open(Arc::clone(&service), directory.join("podcast-dht.sqlite3"))
|
||||
.await
|
||||
.unwrap();
|
||||
let tasks = vec![events, tokio::spawn(Arc::clone(&dht).serve(acceptor))];
|
||||
Self {
|
||||
service,
|
||||
dht,
|
||||
tasks,
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop(self) {
|
||||
for task in &self.tasks {
|
||||
task.abort();
|
||||
}
|
||||
self.service.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
async fn contact(&self) -> NodeContact {
|
||||
let peer_id = self.service.endpoint_id();
|
||||
NodeContact {
|
||||
node_id: NodeId::from_endpoint(&peer_id),
|
||||
peer_id,
|
||||
ticket: self.service.ticket().await.unwrap().to_string(),
|
||||
last_seen_ms: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn start_service(
|
||||
directory: &Path,
|
||||
network: NetworkId,
|
||||
overlay: bool,
|
||||
) -> (Arc<MusicDhtService>, tokio::task::JoinHandle<()>) {
|
||||
std::fs::create_dir_all(directory).unwrap();
|
||||
let mut config = MusicDhtConfig::builder()
|
||||
.data_dir(directory)
|
||||
.network_id(network)
|
||||
.request_timeout(Duration::from_secs(2))
|
||||
.lookup_timeout(Duration::from_secs(5))
|
||||
.transport_timeout(Duration::from_secs(5))
|
||||
.dial_timeout(Duration::from_secs(2));
|
||||
if overlay {
|
||||
config = config.schema_independent_stream_protocol(PODCAST_DHT_ALPN);
|
||||
}
|
||||
let (service, mut events) = MusicDhtService::start(config.build().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let events = tokio::spawn(async move { while events.recv().await.is_some() {} });
|
||||
(Arc::new(service), events)
|
||||
}
|
||||
|
||||
fn feed(path: &str) -> FeedUrl {
|
||||
FeedUrl::parse(&format!("https://feeds.example.test/{path}")).unwrap()
|
||||
}
|
||||
|
||||
/// A subscription of A's user (generation 1).
|
||||
fn subscription(path: &str, title: &str) -> SubscriptionRecord {
|
||||
let feed_url = feed(path);
|
||||
SubscriptionRecord {
|
||||
podcast_id: PodcastId::from_feed_url(&feed_url),
|
||||
subscribed: true,
|
||||
podcast: PodcastMeta {
|
||||
feed_url,
|
||||
title: title.to_string(),
|
||||
author: Some("Ferris Crab".to_string()),
|
||||
description: Some("Weekly news.".to_string()),
|
||||
link: None,
|
||||
image_url: None,
|
||||
},
|
||||
feed_aliases: Vec::new(),
|
||||
generation: 1,
|
||||
hlc_ms: 1_000,
|
||||
op_id: "device-a-op-1".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn share(subscription: &SubscriptionRecord, shared: bool, generation: u64) -> ShareRecord {
|
||||
ShareRecord {
|
||||
podcast_id: subscription.podcast_id.clone(),
|
||||
shared,
|
||||
generation,
|
||||
hlc_ms: 2_000 + generation as i64,
|
||||
op_id: format!("device-a-share-{generation}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for<T>(what: &str, mut check: impl AsyncFnMut() -> Option<T>) -> T {
|
||||
let started = std::time::Instant::now();
|
||||
loop {
|
||||
if let Some(value) = check().await {
|
||||
return value;
|
||||
}
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(60),
|
||||
"timed out waiting for {what}"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends raw bytes as one overlay request and decodes the response.
|
||||
async fn raw_request(from: &Node, to: &Node, payload: &[u8]) -> PodcastDhtResponse {
|
||||
let ticket = to.service.ticket().await.unwrap();
|
||||
let mut stream = from
|
||||
.service
|
||||
.open_stream_to(&ticket, PODCAST_DHT_ALPN)
|
||||
.await
|
||||
.unwrap();
|
||||
stream.send.write_all(payload).await.unwrap();
|
||||
stream.send.finish().unwrap();
|
||||
let mut bytes = Vec::new();
|
||||
AsyncReadExt::read_to_end(&mut stream.recv, &mut bytes)
|
||||
.await
|
||||
.unwrap();
|
||||
postcard::from_bytes(&bytes).unwrap()
|
||||
}
|
||||
|
||||
async fn store_request(from: &Node, to: &Node, record: SignedPodcastRecord) -> PodcastDhtResponse {
|
||||
let request = PodcastDhtRequest {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
requester: from.contact().await,
|
||||
operation: PodcastDhtOperation::StoreBatch {
|
||||
records: vec![PodcastStoreEntry {
|
||||
keys: record.dht_keys(),
|
||||
record,
|
||||
}],
|
||||
},
|
||||
};
|
||||
raw_request(from, to, &postcard::to_stdvec(&request).unwrap()).await
|
||||
}
|
||||
|
||||
async fn search(node: &Node, query: &str) -> Vec<PodcastSearchHit> {
|
||||
node.dht.search(query, 10).await.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn shared_podcast_is_discoverable_by_strangers_and_retractable() {
|
||||
tokio::time::timeout(TEST_TIMEOUT, scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn scenario() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let network = NetworkId::from_name("podcast-dht-integration");
|
||||
let a = Node::start(&temp.path().join("a"), network).await;
|
||||
let b = Node::start(&temp.path().join("b"), network).await;
|
||||
let c = Node::start(&temp.path().join("c"), network).await;
|
||||
// D runs the music DHT only.
|
||||
let (d, d_events) = start_service(&temp.path().join("d"), network, false).await;
|
||||
|
||||
// B and C reach A through the ordinary music-DHT transport; C has no
|
||||
// device-sync pairing with A or B.
|
||||
b.service
|
||||
.connect(a.service.ticket().await.unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
c.service
|
||||
.connect(a.service.ticket().await.unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
d.connect(a.service.ticket().await.unwrap()).await.unwrap();
|
||||
wait_for("music DHT contacts", async || {
|
||||
(a.service.known_peers().len() >= 3
|
||||
&& !b.service.known_peers().is_empty()
|
||||
&& !c.service.known_peers().is_empty())
|
||||
.then_some(())
|
||||
})
|
||||
.await;
|
||||
for node in [&a, &b, &c] {
|
||||
node.dht.bootstrap().await.unwrap();
|
||||
}
|
||||
wait_for("podcast overlay contacts", async || {
|
||||
(a.dht.known_peers() >= 2 && c.dht.known_peers() >= 1).then_some(())
|
||||
})
|
||||
.await;
|
||||
// D is dialed by bootstrap but never answers on the overlay ALPN, so it
|
||||
// never becomes an overlay contact: it is never asked for records or to
|
||||
// store them.
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(a.dht.known_peers(), 2);
|
||||
|
||||
// A capability manifest without podcast_dht evicts a peer; one with it
|
||||
// lets the peer be learned again.
|
||||
let manifest = CapabilityManifest::frid("furumi", "test").with_podcast_dht();
|
||||
a.dht
|
||||
.note_peer_capabilities(
|
||||
c.service.endpoint_id(),
|
||||
&manifest.clone().without_protocol(PODCAST_DHT_ID),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(a.dht.known_peers(), 1);
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(
|
||||
a.dht.known_peers(),
|
||||
1,
|
||||
"the evicted peer is not bootstrapped"
|
||||
);
|
||||
a.dht
|
||||
.note_peer_capabilities(c.service.endpoint_id(), &manifest)
|
||||
.await
|
||||
.unwrap();
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(a.dht.known_peers(), 2);
|
||||
|
||||
// A request with another overlay version is refused.
|
||||
let request = PodcastDhtRequest {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION + 1,
|
||||
requester: c.contact().await,
|
||||
operation: PodcastDhtOperation::FindNode { target: [0; 32] },
|
||||
};
|
||||
let response = raw_request(&c, &a, &postcard::to_stdvec(&request).unwrap()).await;
|
||||
assert!(!response.ok);
|
||||
|
||||
// A's user shares "Rust Radio Weekly" but never shares "Secret Garden".
|
||||
let shared = subscription("rust-radio.xml", "Rust Radio Weekly");
|
||||
let private = subscription("secret-garden.xml", "Secret Garden Diaries");
|
||||
let states = vec![
|
||||
PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, true, 2)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
PodcastShareState {
|
||||
subscription: private.clone(),
|
||||
share: None,
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
];
|
||||
let stats = a.dht.sync_shares(states.clone()).await.unwrap();
|
||||
assert_eq!(
|
||||
(stats.published, stats.unchanged, stats.unpublished),
|
||||
(1, 0, 0)
|
||||
);
|
||||
assert!(stats.push.local_replica);
|
||||
assert!(stats.push.remote_nodes >= 1);
|
||||
let error = a
|
||||
.dht
|
||||
.publish(
|
||||
&private,
|
||||
&share(&private, false, 2),
|
||||
PodcastCardSpec::from_subscription(&private).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, MusicDhtError::PodcastPublication(_)));
|
||||
// Idempotent: nothing new is published.
|
||||
let again = a.dht.sync_shares(states).await.unwrap();
|
||||
assert_eq!(
|
||||
(again.published, again.unchanged, again.unpublished),
|
||||
(0, 1, 0)
|
||||
);
|
||||
let published = a.dht.published().await.unwrap();
|
||||
assert_eq!(published.len(), 1);
|
||||
let first_card = published[0].record.clone();
|
||||
|
||||
// B, the same user's other device, holds the same merged registers and
|
||||
// publishes the same podcast under its own key; the private podcast
|
||||
// stays private there too.
|
||||
let b_states = vec![
|
||||
PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, true, 2)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
PodcastShareState {
|
||||
subscription: private.clone(),
|
||||
share: None,
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
},
|
||||
];
|
||||
let stats = b.dht.sync_shares(b_states).await.unwrap();
|
||||
assert_eq!((stats.published, stats.unpublished), (1, 0));
|
||||
let mut both = vec![a.service.endpoint_id(), b.service.endpoint_id()];
|
||||
both.sort_by_key(|key| *key.as_bytes());
|
||||
let hits = wait_for("C sees both publishers", async || {
|
||||
let hits = search(&c, "rust radio").await;
|
||||
(hits.len() == 1 && hits[0].publishers == both).then_some(hits)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
|
||||
assert!(hits[0].claimed_feed_urls.is_empty());
|
||||
|
||||
// B unshares (for example a stale register on B) and goes offline: B is
|
||||
// not needed for discovery, A's card is still found.
|
||||
let stats = b
|
||||
.dht
|
||||
.sync_shares(vec![PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, false, 3)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
}])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.unpublished, 1);
|
||||
let a_only = vec![a.service.endpoint_id()];
|
||||
wait_for("B's tombstone reaches C's lookup", async || {
|
||||
let hits = search(&c, "rust radio").await;
|
||||
(hits.len() == 1 && hits[0].publishers == a_only).then_some(())
|
||||
})
|
||||
.await;
|
||||
b.stop().await;
|
||||
|
||||
let hits = wait_for("C finds the card by text", async || {
|
||||
let hits = search(&c, "rust radio").await;
|
||||
(!hits.is_empty()).then_some(hits)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(hits[0].podcast_id, shared.podcast_id);
|
||||
assert_eq!(hits[0].publishers, vec![a.service.endpoint_id()]);
|
||||
assert_eq!(hits[0].feed_url, shared.podcast.feed_url);
|
||||
assert_eq!(hits[0].card.feed_url, shared.podcast.feed_url);
|
||||
assert_eq!(hits[0].card.title, "Rust Radio Weekly");
|
||||
assert_eq!(search(&c, "ferris weekly").await.len(), 1, "author tokens");
|
||||
assert!(
|
||||
search(&c, "rust jazz").await.is_empty(),
|
||||
"every token must match"
|
||||
);
|
||||
let by_id = c.dht.lookup_podcast(&shared.podcast_id).await.unwrap();
|
||||
assert_eq!(by_id.unwrap().card.podcast_id, shared.podcast_id);
|
||||
|
||||
// The never-shared podcast leaves no trace anywhere.
|
||||
assert!(search(&c, "secret garden").await.is_empty());
|
||||
assert!(search(&a, "secret garden").await.is_empty());
|
||||
assert!(
|
||||
c.dht
|
||||
.lookup_podcast(&private.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
a.dht
|
||||
.lookup_podcast(&private.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
// Forged: C signs a card that claims A as publisher.
|
||||
let forged_spec =
|
||||
PodcastCardSpec::from_subscription(&subscription("forged.xml", "Forged Show")).unwrap();
|
||||
let forged_card = PodcastCard::from_spec(
|
||||
forged_spec.clone(),
|
||||
network,
|
||||
a.service.endpoint_id(),
|
||||
u64::MAX - 1,
|
||||
first_card.issued_at_ms(),
|
||||
);
|
||||
let forged = SignedPodcastRecord {
|
||||
signature: c.service.sign_identity(&forged_card.signing_bytes()),
|
||||
body: PodcastRecordBody::Card(Box::new(forged_card)),
|
||||
};
|
||||
let response = store_request(&c, &a, forged).await;
|
||||
assert!(!response.ok && response.stored == 0, "{response:?}");
|
||||
|
||||
// Foreign: A's genuine card stored by C. Only publishers store records.
|
||||
let response = store_request(&c, &a, first_card.clone()).await;
|
||||
assert!(!response.ok && response.stored == 0, "{response:?}");
|
||||
|
||||
// Tampered: A's genuine card with a different description.
|
||||
let mut tampered = first_card.clone();
|
||||
if let PodcastRecordBody::Card(card) = &mut tampered.body {
|
||||
card.description = Some("Now with malware".into());
|
||||
}
|
||||
let response = store_request(&c, &a, tampered).await;
|
||||
assert!(!response.ok, "{response:?}");
|
||||
|
||||
// Malformed and oversized requests.
|
||||
let response = raw_request(&c, &a, b"definitely not postcard").await;
|
||||
assert!(!response.ok);
|
||||
let response = raw_request(&c, &a, &vec![0u8; MAX_PODCAST_DHT_REQUEST_BYTES + 1]).await;
|
||||
assert!(!response.ok);
|
||||
assert!(response.error.unwrap().contains("size"));
|
||||
|
||||
// A valid card signed by C is accepted: the overlay is open to anyone
|
||||
// who signs their own cards.
|
||||
let own_card = PodcastCard::from_spec(
|
||||
forged_spec,
|
||||
network,
|
||||
c.service.endpoint_id(),
|
||||
1,
|
||||
first_card.issued_at_ms(),
|
||||
);
|
||||
let own =
|
||||
SignedPodcastRecord::sign_card(own_card, |bytes| c.service.sign_identity(bytes)).unwrap();
|
||||
let response = store_request(&c, &a, own).await;
|
||||
assert!(response.ok && response.stored > 0, "{response:?}");
|
||||
let forged_hits = search(&a, "forged show").await;
|
||||
assert_eq!(forged_hits.len(), 1);
|
||||
assert_eq!(forged_hits[0].publishers, vec![c.service.endpoint_id()]);
|
||||
|
||||
// A's user stops sharing: a tombstone with the next revision replaces
|
||||
// the card and C stops seeing it.
|
||||
let unshare = vec![PodcastShareState {
|
||||
subscription: shared.clone(),
|
||||
share: Some(share(&shared, false, 3)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
}];
|
||||
let stats = a.dht.sync_shares(unshare).await.unwrap();
|
||||
assert_eq!(stats.unpublished, 1);
|
||||
let published = a.dht.published().await.unwrap();
|
||||
assert!(!published[0].active);
|
||||
assert!(published[0].record.is_tombstone());
|
||||
assert_eq!(published[0].revision, first_card.revision() + 1);
|
||||
wait_for("tombstone hides the card from C", async || {
|
||||
search(&c, "rust radio").await.is_empty().then_some(())
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
c.dht
|
||||
.lookup_podcast(&shared.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
|
||||
// A replaying its own old card to a replica that stores the tombstone
|
||||
// does not bring it back; nobody else may replay it at all.
|
||||
let response = store_request(&a, &c, first_card.clone()).await;
|
||||
assert!(response.ok && response.stored == 0, "{response:?}");
|
||||
let response = store_request(&c, &a, first_card).await;
|
||||
assert!(!response.ok, "{response:?}");
|
||||
assert!(search(&c, "rust radio").await.is_empty());
|
||||
assert!(search(&a, "rust radio").await.is_empty());
|
||||
|
||||
a.stop().await;
|
||||
c.stop().await;
|
||||
d_events.abort();
|
||||
d.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn sync_shares_is_fail_safe_and_holds_back_private_urls() {
|
||||
tokio::time::timeout(TEST_TIMEOUT, single_node_scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn single_node_scenario() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let network = NetworkId::from_name("podcast-dht-sync-shares");
|
||||
let a = Node::start(&temp.path().join("a"), network).await;
|
||||
let state =
|
||||
|subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState {
|
||||
subscription: subscription.clone(),
|
||||
share: Some(share(subscription, shared, generation)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
};
|
||||
let first = subscription("first.xml", "First Show");
|
||||
let second = subscription("second.xml", "Second Show");
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&first, true, 2), state(&second, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.published, 2);
|
||||
assert!(stats.failed.is_empty() && stats.held_back.is_empty());
|
||||
|
||||
// In one call the user unshares the first podcast while the second
|
||||
// one's card cannot be built (the URL its id derives from is unknown).
|
||||
let mut broken = second.clone();
|
||||
broken.podcast.feed_url = feed("elsewhere.xml");
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&first, false, 3), state(&broken, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.failed.len(), 1);
|
||||
assert_eq!(stats.failed[0].podcast_id, second.podcast_id);
|
||||
assert_eq!(stats.unpublished, 2, "the unshared and the broken podcast");
|
||||
let published = a.dht.published().await.unwrap();
|
||||
assert!(published.iter().all(|publication| !publication.active));
|
||||
assert!(
|
||||
published
|
||||
.iter()
|
||||
.all(|publication| publication.record.is_tombstone())
|
||||
);
|
||||
// Republishing re-sends tombstones only; nothing comes back.
|
||||
a.dht.republish().await.unwrap();
|
||||
assert!(search(&a, "first show").await.is_empty());
|
||||
assert!(search(&a, "second show").await.is_empty());
|
||||
|
||||
// A feed URL that looks private is held back until the user confirms.
|
||||
let mut member = subscription("member.xml?token=abc", "Member Show");
|
||||
member.podcast_id = PodcastId::from_feed_url(&member.podcast.feed_url);
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&member, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.published, 0);
|
||||
assert_eq!(stats.held_back.len(), 1);
|
||||
assert_eq!(stats.held_back[0].podcast_id, member.podcast_id);
|
||||
assert!(!stats.held_back[0].warnings.is_empty());
|
||||
assert!(search(&a, "member show").await.is_empty());
|
||||
let mut confirmed = state(&member, true, 2);
|
||||
confirmed.privacy_confirmed = true;
|
||||
let stats = a.dht.sync_shares(vec![confirmed]).await.unwrap();
|
||||
assert_eq!(stats.published, 1);
|
||||
assert_eq!(search(&a, "member show").await.len(), 1);
|
||||
// Once published with the user's consent, the same URLs are kept.
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&member, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((stats.unchanged, stats.held_back.len()), (1, 0));
|
||||
|
||||
a.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn fabricated_contacts_never_join_the_overlay() {
|
||||
tokio::time::timeout(TEST_TIMEOUT, fabricated_contacts_scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// K contacts whose node ids are genuine hashes of their endpoint ids but
|
||||
/// whose tickets carry no address, as a malicious peer could return them.
|
||||
fn fabricated_contacts(network: NetworkId) -> Vec<NodeContact> {
|
||||
(0..music_dht::K)
|
||||
.map(|index| {
|
||||
let peer = federation_net::SecretKey::from_bytes(&[index as u8 + 101; 32]).public();
|
||||
let ticket = federation_net::PeerTicket {
|
||||
ticket_version: federation_net::TICKET_VERSION,
|
||||
protocol_version: 1,
|
||||
network_id: network,
|
||||
schema_id: federation_net::SchemaId::from_bytes([0; 32]),
|
||||
endpoint_addr: federation_net::EndpointAddr::new(peer),
|
||||
};
|
||||
NodeContact {
|
||||
node_id: NodeId::from_endpoint(&peer),
|
||||
peer_id: peer,
|
||||
ticket: ticket.to_string(),
|
||||
last_seen_ms: u64::MAX,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn fabricated_contacts_scenario() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let network = NetworkId::from_name("podcast-dht-fabricated-contacts");
|
||||
let a = Node::start(&temp.path().join("a"), network).await;
|
||||
// M speaks the overlay ALPN but answers every request with fabricated
|
||||
// contacts.
|
||||
let (m, m_events) = start_service(&temp.path().join("m"), network, true).await;
|
||||
let mut acceptor = m.stream_acceptor(PODCAST_DHT_ALPN).unwrap();
|
||||
let fabricated = fabricated_contacts(network);
|
||||
let response = PodcastDhtResponse {
|
||||
version: PODCAST_DHT_PROTOCOL_VERSION,
|
||||
ok: true,
|
||||
error: None,
|
||||
nodes: fabricated.clone(),
|
||||
records: Vec::new(),
|
||||
stored: 0,
|
||||
};
|
||||
let payload = postcard::to_stdvec(&response).unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
while let Some(mut stream) = acceptor.accept().await {
|
||||
let mut request = Vec::new();
|
||||
let _ = AsyncReadExt::read_to_end(
|
||||
&mut (&mut stream.recv).take(MAX_PODCAST_DHT_REQUEST_BYTES as u64),
|
||||
&mut request,
|
||||
)
|
||||
.await;
|
||||
let _ = stream.send.write_all(&payload).await;
|
||||
let _ = stream.send.finish();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(2), stream.send.stopped()).await;
|
||||
}
|
||||
});
|
||||
m.connect(a.service.ticket().await.unwrap()).await.unwrap();
|
||||
wait_for("music DHT contact", async || {
|
||||
(!a.service.known_peers().is_empty()).then_some(())
|
||||
})
|
||||
.await;
|
||||
|
||||
// Bootstrap learns M, which answered, and none of the contacts it made
|
||||
// up: they are tried once and never answer.
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(a.dht.known_peers(), 1);
|
||||
// A lookup treats them as candidates only.
|
||||
let podcast_id = PodcastId::from_feed_url(&feed("nothing.xml"));
|
||||
assert!(a.dht.lookup_podcast(&podcast_id).await.unwrap().is_none());
|
||||
assert_eq!(a.dht.known_peers(), 1);
|
||||
a.dht.bootstrap().await.unwrap();
|
||||
assert_eq!(
|
||||
a.dht.known_peers(),
|
||||
1,
|
||||
"fabricated contacts never count for replica responsibility"
|
||||
);
|
||||
|
||||
server.abort();
|
||||
a.stop().await;
|
||||
m_events.abort();
|
||||
m.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn restarted_device_waits_for_sync_shares_before_resending_cards() {
|
||||
tokio::time::timeout(TEST_TIMEOUT, restart_scenario())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn restart_scenario() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let network = NetworkId::from_name("podcast-dht-restart");
|
||||
let state =
|
||||
|subscription: &SubscriptionRecord, shared: bool, generation: u64| PodcastShareState {
|
||||
subscription: subscription.clone(),
|
||||
share: Some(share(subscription, shared, generation)),
|
||||
card: None,
|
||||
privacy_confirmed: false,
|
||||
};
|
||||
let kept = subscription("kept.xml", "Kept Show");
|
||||
let revoked = subscription("revoked.xml", "Revoked Show");
|
||||
|
||||
// A publishes both podcasts while alone, then goes offline.
|
||||
let a = Node::start(&temp.path().join("a"), network).await;
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&kept, true, 2), state(&revoked, true, 2)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((stats.published, stats.push.remote_nodes), (2, 0));
|
||||
a.stop().await;
|
||||
|
||||
// Meanwhile another trusted device revokes "Revoked Show". A comes back
|
||||
// with its persisted cards and runs maintenance before its client
|
||||
// reconciled: the first bootstrap finds B.
|
||||
let b = Node::start(&temp.path().join("b"), network).await;
|
||||
let mut a = Node::start(&temp.path().join("a"), network).await;
|
||||
b.service
|
||||
.connect(a.service.ticket().await.unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for("music DHT contacts", async || {
|
||||
(!a.service.known_peers().is_empty()).then_some(())
|
||||
})
|
||||
.await;
|
||||
a.tasks.push(tokio::spawn(Arc::clone(&a.dht).maintenance()));
|
||||
wait_for("overlay contacts", async || {
|
||||
(a.dht.known_peers() >= 1 && b.dht.known_peers() >= 1).then_some(())
|
||||
})
|
||||
.await;
|
||||
// Neither maintenance nor an explicit republish sends a stale card, and
|
||||
// A does not serve them to lookups either.
|
||||
let stats = a.dht.republish().await.unwrap();
|
||||
assert_eq!(stats.records, 0, "stale cards are withheld");
|
||||
for podcast in [&kept, &revoked] {
|
||||
assert!(
|
||||
b.dht
|
||||
.lookup_podcast(&podcast.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(search(&b, "show").await.is_empty());
|
||||
|
||||
// The client's first reconciliation revokes the stale share and
|
||||
// re-sends the unchanged card.
|
||||
let stats = a
|
||||
.dht
|
||||
.sync_shares(vec![state(&kept, true, 2), state(&revoked, false, 3)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(stats.published, stats.unchanged, stats.unpublished),
|
||||
(0, 1, 1)
|
||||
);
|
||||
assert_eq!(stats.push.records, 2, "the kept card and the tombstone");
|
||||
assert!(stats.push.remote_nodes >= 1);
|
||||
let a_only = vec![a.service.endpoint_id()];
|
||||
let hit = b
|
||||
.dht
|
||||
.lookup_podcast(&kept.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("the kept card is found");
|
||||
assert_eq!(hit.publishers, a_only);
|
||||
assert!(
|
||||
b.dht
|
||||
.lookup_podcast(&revoked.podcast_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
// From now on republishing covers the kept card and the tombstone.
|
||||
let stats = a.dht.republish().await.unwrap();
|
||||
assert_eq!(stats.records, 2);
|
||||
let hits = search(&b, "show").await;
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(hits[0].podcast_id, kept.podcast_id);
|
||||
|
||||
a.stop().await;
|
||||
b.stop().await;
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Publish missing workspace versions together; Cargo orders their dependencies."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
INDEXES = {
|
||||
"crates-io": "https://index.crates.io/",
|
||||
"gitea": "https://gt.hexor.cy/api/packages/Furumi/cargo/",
|
||||
}
|
||||
|
||||
|
||||
def index_path(name):
|
||||
name = name.lower()
|
||||
if len(name) < 3:
|
||||
return f"{len(name)}/{name}"
|
||||
if len(name) == 3:
|
||||
return f"3/{name[0]}/{name}"
|
||||
return f"{name[:2]}/{name[2:4]}/{name}"
|
||||
|
||||
|
||||
def is_published(name, version, registry="crates-io"):
|
||||
request = urllib.request.Request(
|
||||
INDEXES[registry] + index_path(name),
|
||||
headers={"User-Agent": "frid-release-ci", "Cache-Control": "no-cache"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
versions = [json.loads(line) for line in response.read().splitlines()]
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code == 404:
|
||||
return False
|
||||
raise
|
||||
return any(row["vers"] == version for row in versions)
|
||||
|
||||
|
||||
def unpublished_packages(metadata, registry="crates-io"):
|
||||
members = set(metadata["workspace_members"])
|
||||
packages = []
|
||||
for package in sorted(metadata["packages"], key=lambda item: item["name"]):
|
||||
registries = package["publish"]
|
||||
if package["id"] not in members or (
|
||||
registries is not None and registry not in registries
|
||||
):
|
||||
continue
|
||||
name, version = package["name"], package["version"]
|
||||
if is_published(name, version, registry):
|
||||
print(f"Skip {name} {version}: already published", flush=True)
|
||||
else:
|
||||
print(f"Publish {name} {version}", flush=True)
|
||||
packages.append(name)
|
||||
return packages
|
||||
|
||||
|
||||
def wait_for_mirror_dependencies(metadata, selected):
|
||||
# Mirrored manifests retain their crates.io dependencies. In the parallel
|
||||
# upload job, wait for new workspace dependencies to reach crates.io before
|
||||
# Cargo resolves the mirror's packages (it does not compile them again).
|
||||
members = {p["name"]: p for p in metadata["packages"] if p["id"] in metadata["workspace_members"]}
|
||||
required = {
|
||||
(dep["name"], members[dep["name"]]["version"])
|
||||
for name in selected
|
||||
for dep in members[name]["dependencies"]
|
||||
if dep.get("path") and dep["name"] in members and dep["kind"] != "dev"
|
||||
}
|
||||
deadline = time.monotonic() + 600
|
||||
while required:
|
||||
required = {(name, version) for name, version in required if not is_published(name, version)}
|
||||
if not required:
|
||||
return
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError(f"Workspace dependencies did not reach crates.io: {sorted(required)}")
|
||||
print(f"Waiting for crates.io dependencies before mirroring: {sorted(required)}", flush=True)
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--plan", action="store_true", help="list packages without publishing")
|
||||
parser.add_argument("--dry-run", action="store_true", help="verify packages without uploading")
|
||||
parser.add_argument("--registry", choices=INDEXES, default="crates-io")
|
||||
parser.add_argument("--no-verify", action="store_true", help="publish packages already verified by CI without rebuilding")
|
||||
args = parser.parse_args()
|
||||
token_variable = "CARGO_REGISTRY_TOKEN" if args.registry == "crates-io" else "CARGO_REGISTRIES_GITEA_TOKEN"
|
||||
if not (args.plan or args.dry_run) and not os.environ.get(token_variable):
|
||||
parser.error(f"{token_variable} must be set for publishing")
|
||||
metadata = json.loads(subprocess.check_output(
|
||||
["cargo", "metadata", "--no-deps", "--format-version", "1", "--locked"],
|
||||
text=True,
|
||||
))
|
||||
packages = unpublished_packages(metadata, args.registry)
|
||||
if args.plan or not packages:
|
||||
return
|
||||
if args.registry == "gitea":
|
||||
wait_for_mirror_dependencies(metadata, packages)
|
||||
command = ["cargo", "publish", "--registry", args.registry, "--locked", "--all-features"]
|
||||
for name in packages:
|
||||
command.extend(["--package", name])
|
||||
if args.dry_run:
|
||||
command.append("--dry-run")
|
||||
if args.no_verify:
|
||||
command.append("--no-verify")
|
||||
# Multi-package publishing verifies against a temporary local registry,
|
||||
# then uploads and waits for dependencies to appear before their users.
|
||||
subprocess.run(command, check=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,82 @@
|
||||
import io
|
||||
import json
|
||||
import unittest
|
||||
import urllib.error
|
||||
from contextlib import redirect_stdout
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import publish_crates
|
||||
|
||||
|
||||
class PublishCratesTests(unittest.TestCase):
|
||||
def test_publish_job_does_not_rebuild_verified_packages(self):
|
||||
with patch("sys.argv", ["publish_crates.py", "--no-verify"]):
|
||||
with patch.dict(publish_crates.os.environ, {"CARGO_REGISTRY_TOKEN": "test-only"}):
|
||||
with patch.object(publish_crates.subprocess, "check_output", return_value=json.dumps({})):
|
||||
with patch.object(publish_crates, "unpublished_packages", return_value=["music-dht", "furumi-library"]):
|
||||
with patch.object(publish_crates.subprocess, "run") as run:
|
||||
publish_crates.main()
|
||||
run.assert_called_once_with([
|
||||
"cargo", "publish", "--registry", "crates-io", "--locked", "--all-features",
|
||||
"--package", "music-dht", "--package", "furumi-library", "--no-verify",
|
||||
], check=True)
|
||||
|
||||
def test_existing_and_private_versions_are_excluded_on_retry(self):
|
||||
packages = [
|
||||
{"id": "net", "name": "federation-net", "version": "0.3.1", "publish": None},
|
||||
{"id": "dht", "name": "music-dht", "version": "0.6.0", "publish": None},
|
||||
{"id": "lib", "name": "furumi-library", "version": "0.3.0", "publish": None},
|
||||
{"id": "demo", "name": "demo", "version": "0.1.0", "publish": []},
|
||||
{"id": "internal", "name": "internal", "version": "0.1.0", "publish": ["private"]},
|
||||
{"id": "external", "name": "external", "version": "0.1.0", "publish": None},
|
||||
]
|
||||
metadata = {"packages": packages, "workspace_members": [p["id"] for p in packages[:-1]]}
|
||||
for published, expected in [
|
||||
({"federation-net"}, ["furumi-library", "music-dht"]),
|
||||
({"federation-net", "music-dht"}, ["furumi-library"]),
|
||||
]:
|
||||
with self.subTest(published=published), redirect_stdout(io.StringIO()):
|
||||
with patch.object(publish_crates, "is_published", side_effect=lambda name, *args: name in published):
|
||||
self.assertEqual(publish_crates.unpublished_packages(metadata), expected)
|
||||
|
||||
def test_mirror_waits_for_workspace_dependencies_without_building(self):
|
||||
metadata = {"workspace_members": ["lib", "dht"], "packages": [
|
||||
{"id": "lib", "name": "furumi-library", "version": "0.3.0", "dependencies": [
|
||||
{"name": "music-dht", "path": "/workspace/music-dht", "kind": None},
|
||||
]},
|
||||
{"id": "dht", "name": "music-dht", "version": "0.6.0", "dependencies": []},
|
||||
]}
|
||||
with patch.object(publish_crates, "is_published", side_effect=[False, True]) as published:
|
||||
with patch.object(publish_crates.time, "sleep") as sleep, redirect_stdout(io.StringIO()):
|
||||
publish_crates.wait_for_mirror_dependencies(metadata, ["furumi-library"])
|
||||
self.assertEqual(published.call_count, 2)
|
||||
sleep.assert_called_once_with(10)
|
||||
|
||||
def test_gitea_uses_its_own_index(self):
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = b'{"vers":"0.3.1"}\n'
|
||||
with patch.object(publish_crates.urllib.request, "urlopen", return_value=response) as fetch:
|
||||
self.assertTrue(publish_crates.is_published("federation-net", "0.3.1", "gitea"))
|
||||
self.assertEqual(fetch.call_args.args[0].full_url,
|
||||
"https://gt.hexor.cy/api/packages/Furumi/cargo/fe/de/federation-net")
|
||||
|
||||
def test_registry_errors_do_not_look_like_unpublished_versions(self):
|
||||
for status in (403, 429, 500):
|
||||
error = urllib.error.HTTPError("https://index.crates.io/test", status, "error", {}, None)
|
||||
with self.subTest(status=status), patch.object(publish_crates.urllib.request, "urlopen", side_effect=error):
|
||||
with self.assertRaises(urllib.error.HTTPError):
|
||||
publish_crates.is_published("music-dht", "0.6.0")
|
||||
|
||||
def test_missing_crate_and_existing_version(self):
|
||||
error = urllib.error.HTTPError("https://index.crates.io/test", 404, "missing", {}, None)
|
||||
with patch.object(publish_crates.urllib.request, "urlopen", side_effect=error):
|
||||
self.assertFalse(publish_crates.is_published("new-crate", "0.1.0"))
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = b'{"vers":"0.6.0","yanked":false}\n'
|
||||
with patch.object(publish_crates.urllib.request, "urlopen", return_value=response):
|
||||
self.assertTrue(publish_crates.is_published("music-dht", "0.6.0"))
|
||||
self.assertFalse(publish_crates.is_published("music-dht", "0.7.0"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user