Compare commits

..
Author SHA1 Message Date
sanapachandClaude Opus 5.5 5d29b61f18 Point frid deps at the fork branch
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:13:07 +02:00
sanapachandClaude Opus 5.5 0e985b4267 Publish podcasts after the first device sync on restart
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 abd100d6e7 Prefer the account's current subscription for a moved feed
An old, unsubscribed podcast of the same feed no longer hides the
podcast the account follows now: the preview marks the card as already
subscribed, and subscribing from the card or by URL no longer
resubscribes the old podcast next to the current one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 c5f3b3cfa4 Never ingest another feed's document into a followed podcast
Subscribing from a URL that maps onto an existing podcast only through
an alias or the feed's itunes:new-feed-url claim no longer writes that
document into the podcast: its episodes, metadata and validators stay,
and an unsubscribed podcast is subscribed again with its stored
metadata. The document is ingested only for a new podcast or when it
came from the podcast's own fetch URL.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 bbdac75967 Keep a published card's representative across restarts
The account that provides a published card stays its representative
ahead of server-fetched metadata, and after a restart it is recognized
by the text of the active card. Server-fetched metadata only counts as
trusted when the catalog URL is the podcast's origin, since an account
can point its synced catalog URL anywhere.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 767134d1dd Keep replica room for the cards a rebalance publishes
Replicas count unexpired tombstones toward the 256 records per
publisher and refuse new keys beyond it. A round now admits at most as
many podcasts that need a new key as that room allows, so a fair-share
rebalance never displaces cards for newcomers the replicas would refuse.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 4b45aa5805 Document fair publication limits, share states and moved feeds
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 12617a6d52 Browse a found podcast by season and cancel abandoned previews
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 2ea3f7b337 Report a failed podcast search without ending the music search
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 678c7b8211 Recognize a found podcast the account follows at its new address
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 3a1c50e78e Tell the account why its shared podcast is not published
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 5b2973f958 Share the publication budget fairly and keep published cards stable
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 14f310f145 Keep revoking other podcasts when one shared row does not decode
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 3fb85a4bd1 Release 0.15.0
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 88af30754f Document finding and subscribing to shared podcasts
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:52 +02:00
sanapachandClaude Opus 5.5 59d94203db Show public podcasts in the federated search and subscribe from a card
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 20bf30eae8 Test podcast search events and subscribing from a card
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 659307b741 Preview and subscribe to podcasts found in the federation
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 4c5ba3ff0d Stream public podcast hits in the federated search
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 d4b4543be1 Rank public podcast hits for the searching account
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 27fc2c5354 Document public podcast discovery in the web server
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 675a63a465 Test podcast sharing across accounts and publication over loopback
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 74ad88d06e Add the share toggle and privacy dialog to the podcast page
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 588ea4e99e Add the podcast share endpoint with privacy confirmation
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 c17cd790a7 Run public podcast discovery on the federation node
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 0400a08718 Decide which shared podcasts the server publishes
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 e0f39f503f Store local podcast share confirmations
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:51 +02:00
sanapachandClaude Opus 5.5 4fef5c2bd4 Format the refresh runner
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 7aa2253e00 Document the proxy head check and the playback handoff details
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 7214c57624 Save the failed session's position before preparing the episode again
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 056c53a808 Record the listening after a handoff that did not go through
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 f87d70c593 Bound the episodes the browser can add outside the account's feeds
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 0aead73135 Let one account use the whole refresh run when others have nothing due
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 62141c0840 Answer blocked host names like unreachable ones
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 402ad0a03a Accept a bare MPEG frame only when its whole header is valid
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 2549e9e50d Sniff the file head separately when the browser probes a short range
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 404d9cc175 Note the podcast review fixes in the changelog
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 9b43a25d8f Resume episodes from the progress register and keep saving after a takeover
A reload or a Play after a pause continues from the register instead of
this page's saved position, a seek during preparation is kept, a player
that keeps listening after being superseded writes again as a new
session, a player that loses the playback role records its listening,
marking the loaded episode closes its session, and Prev no longer
resets an episode to 0.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 4050214912 Report the progress register in podcast playback answers
The player checks it before continuing a paused session and resumes
from it after a reload.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 fc0befc4a3 Pass generic proxy responses through only when they start like audio
The enclosure type of a feed or of a client-sent episode reference no
longer makes an application/octet-stream response acceptable.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 cfebe1898f Bound user-started feed and audio requests per account
Feed downloads and proxy streams have per-account and server-wide
limits, new subscriptions stop at 1000 per account, and scheduled
refreshes take accounts in turns.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 2eb27cfe77 Accept only application/json bodies on podcast endpoints
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 c51a9af9e5 Keep network details and URLs out of podcast fetch errors
Users see the kind of failure only; resolved addresses and transport
details go to the logs, without URLs. Feed and audio requests are
limited to the web ports.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 99b2935b35 Compare podcast alias sets in byte order on any collation
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 047fc9e887 Keep the podcast API's 401 path out of a large Result
Clippy flagged the helper that returned the 401 response as its error
variant (result_large_err); it now returns the account id as an Option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 bf6981948a Refresh podcast counters when an episode changes status
Playing, completing or restarting an episode from the player updated
its row but left the unplayed and in-progress counters of the podcast
page as they were loaded.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 592d61dd58 Release 0.14.0 with podcasts in the web player
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 ff10f4354d Document podcasts in the web player
README and the administration guide describe the Podcasts section,
resume and completion, handoffs, unsubscribing, what stays private and
what the publisher sees, and the security model of the optional audio
proxy. The development guide lists the endpoints, how the player
writes progress and the node tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 9d1a46c9fe Add a podcasts section to the web player
The sidebar gets Podcasts: add an RSS feed (fetched by the server; a
refused or broken feed shows why), the subscriptions with unplayed
counts and the last fetch error, and a podcast page with seasons, an
episode search, played/sort filters, paging, manual refresh and
unsubscribe (the confirmation says progress and history are kept), or
subscribe again. Episodes show the saved position, the time left and a
played mark, with Play/Resume, Start over, Mark played, Mark unplayed
and Add to queue; Jam guests cannot add episodes. The podcast page also
lists the recent listening history.

The queue labels episodes, the podcast name of the current episode
opens its page, like and share are hidden for episodes, and the player
bar shows when an episode is loading or cannot be played or resumed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 3ebfc45abb Play podcast episodes in the web player
Episodes in the queue are no longer skipped. The player asks the
podcast API for the audio source and the resume position, plays the
publisher URL (or the same-origin proxy when the administrator enabled
it and the direct URL fails or is http:// on an https:// page) and
seeks before it starts, so it never reports a resumed episode that
plays from elsewhere: a server that cannot seek to the saved position
shows that and keeps the position until the user plays from the start.

Progress follows the contract: Play starts a listening session as an
explicit action above the handed-off generation, checkpoints go out
every 20 s and on pause, a seek is an explicit action, the end of the
stream completes the episode, and a stop, a page hide (beacon) or a
handoff saves the final position. Writes of a session run one after
another in the background; a superseded session stops writing, and a
device that loses the playback role ends its session without a report.
Reports always carry the position of their own episode.

Jam placeholders stay unplayable and episodes stay private for Jam
rules. A podcasts store holds the list, the podcast page and the
explicit episode actions for the UI of the next commit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 2c551c5b71 Carry the newest progress generation in outgoing playback states
A browser queue item keeps the handoff hint it arrived with, which is
older than the checkpoints this account has written since. Every
podcast episode in a playback state the web sends to a trusted device
now carries max(hint, register generation), so the device taking over
writes its explicit action above everything the web has seen. Which
state reads the resume position from the register is decided on the
converted queue, so the bridge itself stays testable without a
database.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 1778942714 Add the podcast HTTP API for the UI and the web player
/api/podcasts now lists subscriptions with counters and fetch state,
subscribes by feed URL (fetched first; nothing is stored for a URL that
is not a public feed), shows a podcast with its seasons and history,
pages and searches episodes, and runs the explicit progress actions
(start over, played, unplayed) and unsubscribe/subscribe again.

The web player gets two endpoints: playback returns the audio source
and the resume position and, when the browser starts playing, writes a
new listening session as an explicit action above the carried handoff
generation; progress stores checkpoints, pause, stop and handoff at the
session generation, a seek as an explicit action, completes the episode
once at the end or within the completion tail, and records the
listening session. Handlers delegate to podcasts::api, which the new
Postgres test drives directly.

Podcast queue items are no longer marked unavailable in browser JSON
and carry the episode artwork. The temporary dead-code allowance of the
podcast module is gone; helpers used only by tests are cfg(test) and
complete_session gave way to complete_if_current.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 5aaea8a8eb Document podcast feed refresh and the audio proxy
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 2c0c3d21aa Proxy podcast audio from publishers on request
Browsers stream episode audio from the publisher by default. For
publishers they cannot play directly (CORS, mixed content), the
administrator can enable podcast_audio_proxy_enabled (admin Settings,
FURU_PODCAST_AUDIO_PROXY_ENABLED). The endpoints live in a new app at
/api/podcasts:

- GET /episodes/{episode_id}/audio streams an episode of the signed-in
  account. The URL comes from the account's catalog, never from the
  request. Requests go through SafeHttp with one sanitized byte Range and
  no client headers; 200/206/416 pass through, only audio content types
  are served, with nosniff and a sandbox CSP, and responses stop at 1 GiB
  or 4 hours.
- GET /episodes/{episode_id}/source returns the publisher URL and, when
  enabled, the proxy URL.
- POST /{podcast_id}/refresh refreshes one subscribed feed now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 bf9a6abc3f Refresh subscribed podcast feeds in a background job
The podcast_refresh job runs every five minutes and refreshes the podcasts
whose next check is due: conditional requests (ETag, Last-Modified), 32 MiB
and 60 s limits, the shared music-dht parser and PodcastStore::apply_feed.
A chain of permanent redirects moves the feed and becomes an alias.
Healthy feeds are checked hourly with a stable spread; failures back off
from 15 minutes up to a day, honouring Retry-After. Accounts subscribed to
the same feed URL share one request, at most four run at once, and one
run at a time holds an advisory lock. runner::refresh_now is the manual
refresh for one subscribed podcast, at most every 30 seconds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 04ae7d5bd1 Guard podcast HTTP requests against SSRF
Feed and enclosure URLs come from users and third-party feeds. SafeHttp
accepts only http(s) URLs without credentials, resolves names with a
resolver that refuses the whole answer unless every address is public
(loopback, private, CGNAT, link-local, cloud metadata, unique local,
multicast, reserved; IPv4-mapped, NAT64 and 6to4 judged by their IPv4
part), and connects to exactly the checked addresses, so DNS rebinding
cannot swap them. IP literals and every redirect hop are checked; system
proxies, cookies and Referer are off. Loopback is reachable only through a
#[cfg(test)] policy used by the local test server.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 c2627444c5 Document the podcast store and the relay import
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 681ff7af89 Sync podcast state through the store instead of the relay
Received podcast operations and snapshot sections are applied to the
podcast store, this device's snapshots are filled from it, and an
unknown episode position resumes from its progress register. The
relay module is removed; operations, gating and snapshot bounds on the
wire are unchanged. The device-sync Postgres tests now migrate their
database themselves and cover locally emitted podcast operations.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 7f07cb206a Add a PostgreSQL podcast store with relay import
PodcastStore implements the music_dht::podcast contract per account:
catalog, feed aliases, episodes with a derived season view, the
subscription, share and progress registers, listening sessions and the
local-only fetch state. Received operations, snapshot records and local
actions share one apply path (materializers plus merge functions) under
a per-account advisory lock. Local actions append their operation to the
device-sync log with the next stamp of the web device and apply exactly
that operation in the same transaction.

Migration m_0056 creates the tables; m_0057 imports the registers that
0.13 kept in furumusic__fed_podcast_relay and the catalog rows of the
episode references in the operation log. The relay table is kept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:38 +02:00
sanapachandClaude Opus 5.5 cc1705457e Enable the shared podcast feed parser
The podcast store resolves episode ids from parsed feeds with the
identity hints of music_dht::podcast::feed, which are part of the id
contract.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 15:12:37 +02:00
57 changed files with 21772 additions and 968 deletions

No files matched your search

+104
View File
@@ -1,5 +1,109 @@
# Changelog
## v0.15.0 — 2026-11-19
- Find podcasts in the federated search: cards shared by federation members
appear in the search results as **Podcasts**, typed apart from music
(`federation.podcast` events, `"type": "podcast"`; music events are
unchanged and now carry `"type": "music"`), with **Already subscribed**
when you follow the podcast, also under another feed address. A card
opens with its metadata and the seasons and episodes read from the feed,
browsable by season (`POST /api/podcasts/discovery/preview`), and
**Subscribe**
(`POST /api/podcasts/discovery/subscribe`) creates your own private
subscription from the card's original feed address with the card's
podcast id, or your existing podcast when you already follow the feed
under the address it moved to (also for **Add RSS**); the publishing
member is never contacted. A failed podcast search keeps the music
results and says so. Card texts and
links are untrusted: shown as text, links and artwork only as checked
`http(s)` (artwork `https`) addresses.
- Share a podcast in the federation: a **Share in the federation** toggle on
the podcast page publishes a public card of the podcast (title, author,
description, artwork, feed URL, episode count; never progress, history,
devices or the account) through the server's federation node on the
separate `podcast_dht` overlay of `music-dht` 0.6. Sharing is the
account's synchronized decision; turning it off or unsubscribing on any
device revokes it, and a late state of another device cannot publish it
again. After a restart the server publishes only after its first
device-sync round, so sharing turned off on a TUI while the server was
stopped is not announced again. A feed URL that looks private (token or key in the address, a
member-feed host) needs a confirmation of the exact address first; the
confirmation is local to the server (`m_0058`,
`furumusic__podcast_share_confirmation`) and asked again for a new share
decision or a moved feed. Accounts sharing the same podcast share one
card, published while any of them shares it; at most 192 podcasts per
server, shared fairly between the sharing accounts (an account within
its part never loses a published card to other accounts). The podcast
page says whether this server published the card, waits for a
confirmation, holds it back at the limit, or cannot publish because
federation is off or discovery is unavailable; the list marks
**Sharing on**. New endpoint `POST /api/podcasts/feeds/{id}/share`
(`{"shared": bool, "confirm"?: {...}}`, 409 when a confirmation is
needed); `GET /api/podcasts/feeds/{id}` reports the share state.
- The federation node serves the podcast-discovery overlay
(`furumi-fd/podcast-dht/1`, cache `podcast-dht.sqlite3` in the federation
data directory), advertises `podcast_dht` in its capability manifest only
while it runs, and probes peers' manifests so the overlay skips peers
without it. If the overlay cannot start, only discovery is disabled.
## v0.14.0 — 2026-11-19
- Podcasts in the web player: a Podcasts section to add RSS feeds, browse
podcasts with unplayed counts and fetch errors, seasons, episode search,
filters and paging, refresh, unsubscribe (progress and history are kept)
and subscribe again, with Resume, Start over, Mark played/unplayed and the
recent listening history.
- The web player plays podcast episodes from the publisher (or the optional
proxy), resumes them from the synchronized position and saves progress in
the background: every 20 s and on pause, seek, item change, page hide and
handoff, with listening sessions and automatic completion by the shared
rule. A source that cannot seek to the saved position is shown as such
instead of playing from elsewhere. Handoffs to and from other devices keep
the episode, position and pause state; outgoing states carry the newest
progress generation. Play (also after a reload or a pause) continues from
the newest saved position of any device; a player that keeps listening
after another device wrote the episode saves again as a new session, and
one that loses the playback role records what it listened. Previous goes
back 15 s within an episode instead of resetting it.
- New endpoints under `/api/podcasts` (`/feeds`, `/feeds/{id}`,
`/feeds/{id}/episodes`, subscribe/unsubscribe, `/episodes/{id}/playback`,
`/progress`, `/restart`, `/played`, `/unplayed`). Request bodies must be
`application/json`.
- Jam guests still see the host's episodes only as private items and cannot
add episodes from the Podcasts section.
- Store podcast subscriptions, feed aliases, episodes, seasons, progress and
listening sessions of each account in PostgreSQL (`src/podcasts/`), with
the shared `music-dht` podcast rules. Received podcast operations and
snapshot sections are applied to this store, and snapshots are built from
it; the wire behavior towards other devices is unchanged. Migrations
`m_0056` and `m_0057` import the state that 0.13 relayed
(`furumusic__fed_podcast_relay`, kept but unused).
- Refresh subscribed podcast feeds in the background (`podcast_refresh` job,
every five minutes; each feed about hourly) with conditional requests,
permanent redirects learned as feed aliases, size and time limits and an
exponential backoff up to a day on errors. Accounts subscribed to the same
feed share one request; accounts take turns within a run of up to 200
feeds. `POST /api/podcasts/{podcast_id}/refresh` refreshes one feed on
demand. Only metadata is fetched, never audio. An account can have up to
1000 subscriptions and two user-started feed downloads at once.
- Optional podcast audio proxy (`podcast_audio_proxy_enabled`, off by
default; browsers stream from the publisher otherwise):
`GET /api/podcasts/episodes/{episode_id}/audio` streams an episode of the
signed-in account from its publisher with Range pass-through, audio only
(a generic binary type needs an audio file extension or audio content; the
feed's enclosure type is not trusted), size/time limits and four streams
per account. `GET /api/podcasts/episodes/{episode_id}/source`
tells the player which URLs to use.
- Feed and proxy requests only reach public addresses on ports 80, 443, 8080
and 8443: names are resolved and checked once per connection (no DNS
rebinding), IP literals and every redirect hop are checked, and no cookies,
credentials or system proxies are used. Users see only the kind of a
failure; details go to the logs without URLs. A host name that resolves to
a blocked address gets the same answer as one that does not resolve.
- The browser can add at most 10 000 episodes outside the account's feeds
(handed-off queue items) through the playback endpoint (HTTP 422 beyond).
## v0.13.0 — 2026-11-19
- Use the shared `music-dht` 0.6 device-sync types and announce device-sync
Generated
+6 -3
View File
@@ -1843,7 +1843,7 @@ dependencies = [
[[package]]
name = "federation-net"
version = "0.3.1"
source = "git+https://gt.hexor.cy/Sanapach/frid.git?branch=podcasts%2Fcontract#65d6ca2f14a1324dc129b90694da83ceb55c60dc"
source = "git+https://gt.hexor.cy/Sanapach/frid.git?branch=podcasts%2Fdiscovery#9c82f6d91691a66d0a559caba06e13a1293f06be"
dependencies = [
"blake3",
"data-encoding",
@@ -1987,7 +1987,7 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
[[package]]
name = "furumusic"
version = "0.13.0"
version = "0.15.0"
dependencies = [
"anyhow",
"async-stream",
@@ -3900,14 +3900,16 @@ dependencies = [
[[package]]
name = "music-dht"
version = "0.6.0"
source = "git+https://gt.hexor.cy/Sanapach/frid.git?branch=podcasts%2Fcontract#65d6ca2f14a1324dc129b90694da83ceb55c60dc"
source = "git+https://gt.hexor.cy/Sanapach/frid.git?branch=podcasts%2Fdiscovery#9c82f6d91691a66d0a559caba06e13a1293f06be"
dependencies = [
"async-trait",
"blake3",
"data-encoding",
"encoding_rs",
"federation-net",
"futures",
"postcard",
"quick-xml 0.41.0",
"rand 0.9.5",
"rusqlite",
"serde",
@@ -3916,6 +3918,7 @@ dependencies = [
"tokio",
"tracing",
"unicode-normalization",
"url",
]
[[package]]
+6 -4
View File
@@ -1,6 +1,6 @@
[package]
name = "furumusic"
version = "0.13.0"
version = "0.15.0"
edition = "2024"
description = "Reusable web-app boilerplate: auth, OIDC/SSO, admin panel, user management, i18n, PostgreSQL"
@@ -56,8 +56,10 @@ uuid = "1"
librqbit = { version = "8.1.1", features = ["disable-upload"] }
# P2P federation: publishes the library into a shared DHT and serves audio /
# catalogs to furumi peers (TUI clients) over the frid stack.
music-dht = "0.6.0"
# `podcast-feed`: the shared RSS/Atom parser, whose identity hints are part of
# the podcast episode id contract.
music-dht = { version = "0.6.0", features = ["podcast-feed"] }
[patch.crates-io]
# Unpublished frid (MR-F1); remove once music-dht 0.6.0 is on crates.io.
music-dht = { git = "https://gt.hexor.cy/Sanapach/frid.git", branch = "podcasts/contract" }
# Unpublished frid (MR-F1, MR-F3); remove once music-dht 0.6.0 is on crates.io.
music-dht = { git = "https://gt.hexor.cy/Sanapach/frid.git", branch = "podcasts/discovery" }
+6
View File
@@ -20,6 +20,12 @@ the Furumi federated network.
without creating an account.
- **Welcome your people.** Use password login or SSO. Instance admins can approve
new users without needing access to the identity provider.
- **Follow podcasts.** Subscribe to RSS feeds, browse seasons and episodes, and
pick up where you stopped, also on your other Furumi players. Episodes stream
from the publisher and are never imported into the library; subscriptions
and listening progress stay private to your own devices. Share a podcast
in the federation on purpose, and find podcasts others shared in the
federated search.
- **Discover more.** Find similar tracks and connect Last.fm for scrobbling.
- **Connect your players.** Pair trusted Furumi devices to synchronize playlists,
likes, playback state, and playback controls.
+239 -7
View File
@@ -71,13 +71,245 @@ Trusted-device pairing is separate from public catalog discovery. It connects
a user's own Furumi players so likes, playlists, playback state, and control can
move between approved devices.
Queues synchronized from other Furumi players can contain podcast episodes. The
web player keeps them in the queue, in order and with their position, and shows
them as **Podcast · not playable in the web player yet**; local playback skips
them. A device that does not support podcasts yet (device-sync protocol below
4, for example an older terminal player) cannot receive such a queue: selecting
it as the playback device shows that it needs an update, and the episodes stay
in the queue. Jam guests see podcast episodes of the host only as private items.
Queues synchronized from other Furumi players can contain podcast episodes;
the web player plays them (see [Podcasts](#podcasts)) and keeps them in order
with their position. A device that does not support podcasts yet (device-sync
protocol below 4, for example an older terminal player) cannot receive such a
queue: selecting it as the playback device shows that it needs an update, and
the episodes stay in the queue. Jam guests see podcast episodes of the host
only as private items they cannot play, add, remove or clear.
## Podcasts
**Podcasts** in the player sidebar lists your subscriptions with the number
of unplayed episodes and the result of the last feed check. Paste an RSS or
Atom feed URL to subscribe: the server loads the feed first and shows why when
it cannot (not a feed, not reachable, or an address that is not public). A
podcast page shows the description, the seasons (episodes without a season
form their own group), a search over episode titles and descriptions, filters
for unplayed, started and played episodes, and the recent listening history.
- **Play / Resume** continues an episode where you stopped, on whichever of
your Furumi players last saved it. **Start over** plays it from the
beginning; **Mark played** and **Mark unplayed** change its status without
playing (marking does not count as a listen). An episode counts as listened
when playback reaches its end, or its last 30 seconds or last 5 %, whichever
is shorter.
- The web player saves the position about every 20 seconds and when you
pause, seek, switch items, close the page or hand playback to another
device, without interrupting the audio. Reloading the page, or pressing
Play after a pause while another player went on with the episode,
continues from the newest saved position. Previous on an episode goes
back 15 seconds instead of to the start. Marking the episode that is
playing played or unplayed pauses it.
- Handing playback to another device keeps the episode, its position and
whether it was paused. A later save of an older listening session never
moves the position back after you seek or start over elsewhere.
- **Unsubscribe** stops checking the feed and keeps your progress and history;
subscribing again continues where you stopped. **Refresh** checks the feed
now (at most every 30 seconds).
Privacy: a new subscription is private. Subscriptions, progress and listening
history are synchronized only between your own paired Furumi players (trusted
devices), never published to the federation, and Jam guests cannot see your
episodes. Only podcasts you share explicitly are published (see below).
Episodes are not stored on the server or shared with peers: the browser
streams the audio and loads the artwork directly from the publisher, so the
publisher sees your browser's address. With the audio proxy (below) the
publisher sees the server instead.
### Sharing a podcast in the federation
**Share in the federation** on a podcast page publishes a public card of the
podcast so that members of the federation network outside your own devices
can find it and subscribe. The card holds only the podcast's public
metadata: title, author, description, artwork and site links, the feed URL,
the number of episodes and the newest publication date. It never contains
your progress, history, listening counts, device names or your account.
**Stop sharing** or **Unsubscribe** revokes it; your progress is kept.
- Sharing is your decision and is synchronized with your other Furumi
players, like the subscription. A podcast shared on another player is
shared here too, and turning it off anywhere turns it off everywhere. A
new subscription, including subscribing again, is always private.
- If the feed URL looks like it may contain private data (an access token or
key in the address, a host of personal subscriber feeds, a long random
part), the web player shows a confirmation dialog with the exact address
before publishing it. Such feeds are often personal: publishing one can
let others use your access. The confirmation belongs to this server and to
this one decision; it is asked again when you share again, when the feed
moves to another address, and when sharing was turned on on another
player.
- The **server** publishes the card with its federation identity (the same
node key that publishes its music library), not with an identity of your
account. When several accounts of this server share the same podcast,
the server publishes one card for it, built from one of these accounts'
copies of the feed, and keeps it while any of them still shares it. The
account whose copy the published card shows keeps providing it while it
shares (also after a server restart, recognised by the card's text), so
another account sharing the podcast later does not replace the card's
text; a new card preferably uses a copy the server downloaded itself
from the podcast's original address. If that account's copy changes
while the server restarts, the card may be rebuilt from another sharing
account's copy. Other members see that this server shares the
podcast, not who. Podcasts shared by a deactivated account are no longer
published (within five minutes).
- The podcast page says what this server does with the card: published,
publishing, waiting for your confirmation, held back by the server's
limit, or not possible because federation is off or public discovery is
unavailable. It shows only this server's publication: a Furumi player
such as the terminal player publishes its own card with its own identity
when you share there, so the card may be public through that player
while this server still waits for your confirmation. The podcast list
marks a podcast with **Sharing on** when your decision is to share it.
- Publishing needs federation to be enabled on the server (Settings →
Federation). Without it, the share decision is kept and the card is
published once federation runs.
- A server publishes at most 192 podcasts, shared fairly between the
accounts that share podcasts: every account gets its turn in rotation,
so an account whose shared podcasts fit in its part of the limit (192
divided by the number of sharing accounts) never loses a published card
because other accounts share many podcasts. When the limit is reached,
only podcasts of accounts beyond their part wait (newest shares first;
cards already published are kept before new ones), and the podcast page
says that the card is held back. Members' caches keep at most 256
records per server, including revocations of the last 24 hours, so a
new card also waits while recent revocations fill that room, and no
published card is given up for a card the federation would not store.
- A podcast whose stored data cannot be read (for example after a damaged
import) is treated as not shared, and its card is revoked; the other
podcasts are still published and revoked as usual.
- Revoking is eventually consistent: the server publishes a signed
revocation at once, but a federation peer that missed it may show the old
card for up to 12 hours. Subscriptions other members made from the card
stay theirs.
The server joins the separate podcast-discovery overlay
(`furumi-fd/podcast-dht/1`) of the federation network and keeps its routing
contacts and the cards it holds for others in `podcast-dht.sqlite3` in the
federation data directory (a cache that can be deleted while the server is
stopped). If the overlay cannot start, the server log says "public podcast
discovery disabled" and everything else keeps working. The admin
federation status (`/admin/v2/api/federation`) reports `podcast_discovery`
and `podcast_discovery_peers`.
### Finding podcasts in the federation
The search box searches the federation too. Podcasts that members shared
appear under **Federation → Podcasts**, marked **Podcast** and kept apart
from music; podcasts you already follow are marked **Already subscribed**
(also when you follow the same feed under an older address it moved
from). If you follow the feed only under the newer address of a card's
moved feed, the result list cannot tell yet; opening the card reads the
feed, sees the move and shows **Already subscribed** with **Open**, and
neither **Subscribe** nor **Add RSS** with the old address creates a
second subscription. A podcast you follow is changed only by its own
address: when a feed found under another address only claims to have
moved to yours (or is an old copy of it), nothing of that feed is written
into your podcast; if you had unsubscribed, it is subscribed again with
its own data and refreshed from its own address.
- Opening a card shows its title, author, description, artwork and the feed
URL, then loads the seasons and episodes from the feed itself through the
server: the newest 50 episodes, and the newest 50 of one season when you
pick its chip (pick it again for all seasons). Nothing is stored until
you subscribe. Closing the card, or picking another season, cancels a
loading preview.
- **Subscribe** adds your own private subscription: the server downloads the
feed from the card's address, as for **Add RSS**, and you can then play,
share or unsubscribe it like any other. The member who published the card
is never contacted and does not need to stay online; their card is only a
pointer to the public feed. The subscription keeps the podcast's identity,
so your other Furumi players recognise it, and nobody else sees it.
- Card texts and links come from other members and may be wrong. The
player shows them as plain text, opens only web links, and loads artwork
directly from its host only over HTTPS (like the artwork of subscribed
podcasts). Only the original feed address of a card is used: an address a
member claims the feed moved to is ignored unless the feed itself
redirects there.
- Search results show only cards; no member's subscriptions, progress or
account are ever visible. A card disappears from search results within
12 hours after it is revoked.
- Podcast search needs federation on this server. When public discovery is
off or unavailable, the search shows music only. When the podcast search
fails or times out, the music results are still shown, with a note that
the podcast search did not answer.
### Podcast feeds and audio
The `podcast_refresh` job (every five minutes, listed under Jobs) checks the
feeds of subscribed podcasts: each feed about once an hour with conditional
requests, failing feeds less and less often (up to once a day). It downloads
feed metadata only, never episode audio. A failing feed keeps its known
episodes. Accounts subscribed to the same feed share one request. Each run
checks at most 200 feeds (2400 an hour), taking accounts in turns: first the
most overdue feed of every account, then the second, and so on. An account
with many or slow feeds therefore does not delay the others, but can use the
capacity nobody else needs.
An account can add up to 1000 subscriptions and, from the browser, up to
10 000 episodes outside its feeds (handed-off queue items; episodes synced
from its own devices are not limited); beyond that the request is refused
with HTTP 422. It can have two feed downloads ("add RSS", Refresh) and four
proxied audio streams in progress at once (16 and 64 for the whole server);
beyond that the request is refused with HTTP 429.
The server therefore makes outgoing HTTP(S) requests to podcast publishers.
It only connects to public addresses on the web ports 80, 443, 8080 and 8443:
feed or audio URLs that point to loopback, private, link-local, CGNAT or cloud
metadata addresses, directly, through DNS or through a redirect, or to any
other port, are refused. System proxy settings are not used for these
requests. Users only see the kind of failure (address not allowed, could not
connect, HTTP status, not a feed); the details, without the URL, are in the
server log and, for scheduled refreshes, in the job log with the podcast id.
"Address not allowed" is only said for what the URL itself shows (scheme,
credentials, port, an IP address). A host name that resolves to a blocked
address gets the same answer as a name that does not resolve or a server that
does not answer ("could not connect"), so users cannot find out which names
exist in the server's internal DNS.
Episode audio is streamed by the browser directly from the publisher. Some
publishers cannot be played that way (plain HTTP while this server uses
HTTPS, or a server the browser refuses). For them, enable **Settings →
Podcasts → Podcast audio proxy** (`FURU_PODCAST_AUDIO_PROXY_ENABLED=true`):
the player then switches to the proxy when the direct URL fails or is plain
HTTP on an HTTPS page, and the server fetches the audio from the publisher
and passes it through without storing it. This uses the server's bandwidth.
Security model of the proxy:
- It takes an episode id, never a URL. The URL comes from the signed-in
account's own podcast catalog: its feeds, and episode references sent by
the account's own clients (its paired devices, or its browser when it plays
a handed-off episode the catalog does not know yet). A signed-in user
controls both (anyone can subscribe to a feed they host), so any signed-in
user can make the server fetch audio from any public URL, as with feed
refreshes.
- What limits that: the same public-address and port checks as for feeds
apply to every connection and redirect, and only audio is passed through.
The publisher's `Content-Type` must be audio; for a generic binary type
(`application/octet-stream` or none), the URL must have an audio file
extension or the file must start like an audio file (MP3, AAC, Ogg, FLAC,
WAV, M4A). The enclosure type written in the feed is never trusted. No
HTML, no video. Requests carry a single byte range, are limited to 1 GiB
and 4 hours each and to four streams per account (64 for the server), and
no cookies or credentials are sent.
- Responses are not cached or stored, and are marked `no-store`, `nosniff`
and sandboxed.
Leave it off unless your users need it.
### Upgrading from 0.13
Release 0.14 adds the podcast UI and playback described above; no setting is
needed besides the optional proxy.
Podcast state synchronized from other Furumi players is kept in per-account
podcast tables (`furumusic__podcast*`). The upgrade imports what 0.13 kept in
`furumusic__fed_podcast_relay`; that table is no longer used and can be
dropped by a later release. Release 0.13 does not read the new tables: after
a rollback it would serve the relay table as it was at the upgrade, and
devices would send their newer podcast state again in their snapshots.
### Rolling back to 0.12
+263 -14
View File
@@ -38,13 +38,12 @@ Update the pinned build tools in `Dockerfile`; changing them rebuilds the cache.
## Connected devices and podcast queue items
Trusted-device sync uses the shared `music_dht::device_sync` types (no local
copies) and announces protocol version 4. The web player has no podcast
features yet, but it keeps every v4 element it receives:
copies) and announces protocol version 4. Each account is one virtual device:
- podcast operations stay in the operation log and are relayed to v4 devices
only; their registers and the podcast snapshot sections are merged with the
shared `music_dht::podcast` rules into `furumusic__fed_podcast_relay`
(opaque contract JSON) and returned in this server's snapshots;
only. They are also applied to the account's podcast store
(`src/podcasts/`, `PodcastStore`), which also provides the podcast snapshot
sections;
- queue items that are not library tracks become external browser items
(`source` plus the verbatim `wire_track`), so podcast episodes and Jam
placeholders survive peer → server → browser → server → peer unchanged;
@@ -53,25 +52,162 @@ features yet, but it keeps every v4 element it receives:
typed value is used only while the legacy columns it recorded
(`queue_json`, `current_track_id`, `queue_position`) match the row. Jam
placeholders are never stored in a user's own queue.
- a browser that selects a podcast episode it cannot play does not know the
episode's position. It reports `position_unknown`, and the server sends the
resume position of the relayed progress register instead of 0. Jam
participant commands carry `jam_participant`; the host browser checks them
against its own queue.
- a browser that selects an item it cannot play (a Jam placeholder), or an
episode whose resume position the server has not sent yet, reports
`position_unknown`, and the server sends the resume position of the
episode's progress register instead of 0. Outgoing playback states raise
each episode's `progress_generation` to the register generation, so the
device that takes over writes above it. Jam participant commands carry
`jam_participant`; the host browser checks them against its own queue.
Rolling back to 0.12 needs the podcast operations removed from the operation
log first; see "Rolling back to 0.12" in `administration.md`.
### Podcast store
`PodcastStore` implements the podcast contract of `music_dht::podcast`
(Frid `PODCASTS.md`) on PostgreSQL, with the invariants of the SQLite
reference store in `furumi-library`; every table has a `user_id` key.
| Table | Content | Synced |
| --- | --- | --- |
| `furumusic__podcast` | channel metadata, fetch URL | no (catalog) |
| `furumusic__podcast_feed_alias` | bounded grow-only alias set | in subscription records and `podcast_feed_aliases_added` |
| `furumusic__podcast_episode` | `EpisodeMeta`, `source` `feed`/`sync`, `in_feed`, cached search text | no (catalog) |
| `furumusic__podcast_season` | view over the episodes | no |
| `furumusic__podcast_subscription`, `_share`, `_progress` | registers | yes |
| `furumusic__podcast_listen_session` | grow-only set of sessions | yes |
| `furumusic__podcast_fetch_state` | ETag, Last-Modified, check times, errors, cover cache | never |
- One apply path: received operations, snapshot records and local actions go
through the `SyncOpWire` materializers and the `merge_*` functions of
`music_dht::podcast`. Each write transaction holds a per-account advisory
lock.
- Local actions (`subscribe`, `unsubscribe`, `set_shared`, `start_session`,
`checkpoint`, `seek`, `complete_if_current`, `restart`, `mark_played`,
`mark_unplayed`, `record_listen_session`) append their operation to the
account's sync log with the next stamp (`devices::append_local_op`: `seq`,
`hlc_ms`, `op_id = <device>:<seq>`) and apply exactly that operation in
the same transaction.
- `apply_feed` resolves episode ids with the shared no-GUID rule against the
stored episodes, including those only known from synced references. It
updates the catalog and the alias set, never a register, progress or a
session.
- Snapshots export each section newest first up to its count and bound it
with `bound_podcast_state_to`. The catalog and the fetch state are never
exported.
Migrations `m_0056_create_podcast_store` and `m_0057_import_podcast_relay`
create the tables and import what 0.13.0 kept in `furumusic__fed_podcast_relay`
(registers as contract JSON, already merged) plus the catalog rows of the
episode references in the operation log. The relay table is no longer written
or read; it stays until a later cleanup release.
### Feed refresh and audio delivery
The runtime around the store lives next to it in `src/podcasts/`:
- `net.rs` — `SafeHttp`, the only HTTP client for podcast feeds and audio.
It accepts `http`/`https` URLs without userinfo, resolves names with
`GuardedResolver` and refuses the whole answer when any address is not
public (loopback, RFC 1918, CGNAT, link-local, cloud metadata, unique local,
multicast, unspecified, documentation, reserved; IPv4-mapped, NAT64 and
6to4 addresses are judged by their IPv4 part). The connector connects to
exactly the checked addresses, so a second DNS answer cannot be swapped in.
IP literals are checked before connecting, redirects are followed manually
(each hop checked, at most 5), and system proxies, cookies and `Referer`
are disabled. `GuardError::is_refusal` is true only for what the URL
shows (scheme, credentials, port, a blocked IP literal); a name that
resolves to a blocked address is reported to users like a transport
failure, so the answers do not reveal internal DNS names.
- `fetch.rs` — `FeedFetcher`: conditional GET (`If-None-Match`,
`If-Modified-Since`), `Accept-Encoding: identity`, 32 MiB and 60 s limits,
`Retry-After` on 429/503, the shared `music_dht` parser. Only a chain of
permanent redirects (301/308) moves a feed.
- `runner.rs` — scheduling: `podcasts_due_for_refresh`, one request per feed
URL for all accounts subscribed to it (conditional only when they share the
same validators), `apply_feed`, then the fetch state. Healthy feeds are
checked hourly plus a stable spread of up to 10 minutes; failures back off
15 min × 2ⁿ⁻¹ up to a day (a longer `Retry-After` wins). `refresh_now` is
the manual refresh: subscribed podcasts only, at most every 30 s.
`jobs/podcast_refresh.rs` runs `refresh_due` every five minutes (200
podcasts taken in turns across accounts without a per-account cap, 4
requests in flight, one run at a time via an advisory lock).
- `proxy.rs` and `http.rs` — the optional audio proxy and the endpoints under
`/api/podcasts`. The proxy takes an episode id, loads the URL from the
account's catalog (`PodcastStore::audio_source`), forwards a single
sanitized byte `Range`, passes 200/206/416 through, accepts audio content
types only (a generic binary type needs an audio URL extension or a file
that starts like audio, `sniff_audio`: read from the response when it
starts at byte 0 and has at least 16 bytes, otherwise with its own
`bytes=0-15` request; the feed's enclosure type is never trusted), and
stops at 1 GiB, 20 s without headers, 30 s
without data or 4 h per response. It is off unless
`podcast_audio_proxy_enabled` is set; by default the browser streams from
the publisher URL.
### Podcast UI and playback
`api.rs` holds what the endpoints do, independent of HTTP (`http.rs` only
authenticates and maps statuses), so the Postgres tests drive it directly:
| Endpoint (`/api/podcasts`) | Does |
| --- | --- |
| `GET /feeds[?all=1]`, `POST /feeds {url}` | list; subscribe after fetching the feed (`runner::subscribe_url`) |
| `GET /feeds/{id}`, `GET /feeds/{id}/episodes?season=&played=&q=&sort=&limit=&offset=` | podcast, seasons, history; episode pages with the browser queue item |
| `POST /feeds/{id}/subscribe`, `/unsubscribe`, `POST /{id}/refresh` | subscription and manual refresh |
| `POST /episodes/{id}/restart`, `/played`, `/unplayed` | explicit progress actions |
| `POST /episodes/{id}/playback` | source, resume position; with `start` a new session (explicit, above `carried_generation`); records an unknown handed-off reference first |
| `POST /episodes/{id}/progress` | `checkpoint`/`pause`/`stop`/`handoff` at the session generation, `seek` explicit, completion once (`ended` or `reaches_completion`), listening session record |
Request bodies must be `application/json` (415 otherwise, like cot's `Json`
extractor): other types need no CORS preflight and would let a same-site page
post with the session cookie.
In the browser (`templates/player/scripts.html`), `_playEpisodeLocal` asks
for the playback, loads the publisher URL (or the proxy: mixed content, or a
failed direct load) and seeks before playing; when the source cannot seek to
the saved position it shows that instead of playing from elsewhere. Only a
handoff or the user's own seek sends a position (`explicitPosition`); an
ordinary Play, including after a reload, resumes from the register, and Play
after a pause first checks that the register is still this session's
(`register_session_id`). A seek made while the source is being prepared is
applied when it arrives. Writes of one session run in order
(`_reportEpisode`), read the session generation at send time and carry the
position of their own episode. After `superseded`, checkpoints stop; if the
user keeps listening here, the next write (one checkpoint interval later, or
at once on pause, stop or the end of the stream) records the old session and
starts a new explicit one (`_takeOverEpisode`). A browser that loses the
active role sends one final `stop` at its session generation (it records the
listening; the new device's start keeps the register), unless it handed off
itself and the transfer went through (a failed transfer clears `handedOff` in
the devices store's `select`). Preparing the same episode again (Play after
its source failed) waits for the old session's final report, so the server
resumes from where the audio stopped. Marking the loaded episode played or unplayed in the same tab saves
and closes its session and pauses it. Prev on an episode goes back 15 s
locally instead of resetting it. Jam guests never play or save episodes
(they only ever see placeholders). The `podcasts` store backs the Podcasts
views in `shell.html`.
Tests never reach the internet: `test_server.rs` serves on 127.0.0.1 and
provides a scripted resolver. Loopback is reachable only through
`AddressPolicy::test_allow_loopback`, which exists only under `#[cfg(test)]`.
Everything sent to a device is gated by the version that device reported itself
(`profile_protocol_version`; relayed profiles only raise it). See the podcast
contract (`PODCASTS.md`) and `PLAYBACK_PROTOCOL.md` in the Frid repository.
```bash
cargo test devices:: queue_items podcast_relay queue_persistence
node --test tests/player_queue_sources.test.cjs
# Migrated disposable PostgreSQL (start the server against it once):
cargo test devices:: queue_items podcasts:: queue_persistence
node --test tests/player_queue_sources.test.cjs tests/player_podcasts.test.cjs
# Disposable PostgreSQL; the tests run the production migrations on it, and
# the migration test creates and drops a database next to it:
FURU_DEVICE_SYNC_TEST_DATABASE_URL=postgresql://... \
cargo test devices::postgres_tests -- --ignored
cargo test postgres_tests -- --ignored
# Sharing, aggregation and a loopback publication from the web node to
# another iroh node (temporary directories):
FURU_DEVICE_SYNC_TEST_DATABASE_URL=postgresql://... \
cargo test discovery -- --include-ignored
FURU_METADATA_FIX_TEST_DATABASE_URL=postgresql://... \
cargo test undo_accepts_playback_archives_written_before_the_typed_queue -- --ignored
```
@@ -80,6 +216,119 @@ FURU_METADATA_FIX_TEST_DATABASE_URL=postgresql://... \
0.12.x byte for byte; `fixtures/v3_hello.golden.json` is the v3 shape. Rewrite
it only for a deliberate change, with `FURUMUSIC_BLESS_GOLDEN=1`.
### Public podcast discovery
`src/podcasts/discovery.rs` decides what the server node publishes on the
`music_dht::podcast_dht` overlay; `src/federation/podcast_discovery.rs` runs
the overlay on the federation node (ALPN `furumi-fd/podcast-dht/1`, cache
`podcast-dht.sqlite3` in the federation data directory, `serve`,
`maintenance`, the reconciler and a capability probe that calls
`note_peer_capabilities`). The capability manifest advertises `podcast_dht`
only while the overlay runs.
- The server's node key is the only publisher. Each account's
`AccountShare` is built like the frid integration pattern for SQLite
clients: the merged subscription register (with its aliases) and share
register, and a card from `PodcastCardSpec::from_subscription` with the
account's catalog fetch URL as `feed_url`, the origin URL as `id_feed_url`
and the feed's episode count and newest date.
- `load_shared` reads active accounts only (`furumusic__user.is_active`).
An `(account, podcast)` whose stored data does not decode (a URL, the
register JSON, an id) is left out and counted (`unreadable`): only a
failing connection or query skips the round, so revocation stays
fail-safe for each podcast on its own.
- `aggregate_share_states` keeps the accounts for which `effective_share`
holds and whose card has no privacy warnings or a matching confirmation,
and builds one `PodcastShareState` per podcast with
`privacy_confirmed = true`. Representative: no warnings, then the
incumbent (the previous round's representative, kept in memory, or after
a restart an account whose card text equals the active card's text from
`PodcastDht::published`), then `AccountShare::trusted_meta`
(`meta_from_feed` and the catalog URL is the origin; an account can point
its synced catalog URL anywhere before the server fetches it), then the
smallest account id.
- At most 192 podcasts (replicas keep 256 records per publisher, tombstones
included), allocated round-robin over the sharing accounts (`allocate`):
each account's queue holds its podcasts the node already publishes
first, then by its own share time; in every round each account takes its
next podcast not taken yet, published ones first inside a round. An
account's share times therefore only order its own podcasts. A podcast
without a live record of the node (an active card or an unexpired
tombstone, `Incumbents::live_keys`) needs a new replica key; at most
`256 - live keys` of them are taken per round (`Incumbents::key_headroom`),
the others wait in `over_limit`, so a rebalance never displaces more
cards than replicas accept new ones for.
`PodcastDht::sync_shares` publishes the result and tombstones everything
else. The podcasts left out (`over_limit`) and those `sync_shares`
reported as failed are kept in memory for `ShareView.held`.
- `furumusic__podcast_share_confirmation` (`m_0058`) is the client-held
`privacy_confirmed`: per account and podcast, bound to the share
register's `op_id` and the exact card URLs. Never synced.
- The reconciler runs on `discovery::request_reconcile()` (after local share,
subscribe and unsubscribe actions, received subscription/share/alias
operations and snapshot sections, refreshes of shared podcasts; 2 s
debounce) and every five minutes. Connection and query errors skip a
round. Only counts are logged.
- Its first round in a federation run waits for the first device-sync round
(`devices::sync_loop` reports it through a `watch` channel once every
exchange of the first complete device listing has answered, failed or
timed out; at most `FIRST_DEVICE_SYNC_WAIT`, 45 s). The overlay withholds
the cards stored before the restart until the first `sync_shares`, which
re-sends the cards still shared, so a share turned off on a TUI while the
server was stopped is not announced again. A device that cannot be
reached at start delivers its decision only when it syncs.
- `ShareView` (in `GET /api/podcasts/feeds/{id}` and the share response):
`discovery` is `running`, `unavailable` (the node runs, the overlay did
not start; `Federation::podcast_discovery_unavailable`) or `off`;
`published` is this server node's own active card only; `held` is
`over_limit` or `failed` after the last round. An unreadable overlay
cache degrades to `published: false` instead of failing the page.
- `POST /api/podcasts/feeds/{id}/share` requires a JSON body
(`{"shared": bool, "confirm"?: {"feed_url", "id_feed_url"}}`); a card with
warnings is shared only with `confirm` naming exactly its current URLs
(otherwise 409 `needs_confirmation` with the warnings, nothing written).
Search and subscription from a card (`src/podcasts/search.rs`):
- `GET /api/player/federation/search/events` starts `PodcastDht::search`
(`Federation::spawn_podcast_search`, at most 20 hits, 20 s) next to the
music DHT search. Music events stream first and are unchanged
(`federation.track|artist|release`); every event now carries `"type":
"music"` or `"type": "podcast"`. Podcast hits follow as
`federation.podcast` events with a `PodcastHitView` entity: the card's
public fields, the origin `feed_url` (never `claimed_feed_urls`),
`publisher_count`, `rank` and `exact_match` from `merge_search_results`
with the music items, and the searching account's `subscribed` /
`local_podcast_id` (by id, or a known feed URL or alias through
`resolve_podcast_id`). `search.completed` adds `podcast_discovery`
(`running`, `off`, `failed`); a failed podcast search, or a failure to
prepare its hits for the account (`Federation::podcast_search_events`
never fails), only drops the podcast hits and reports `failed`.
- `POST /api/podcasts/discovery/preview` and `/discovery/subscribe` take
`{"podcast_id", "feed_url"}` from a hit and refuse a URL the id does not
hash from (422). The preview (optional `"season"`: a number as a
string, `"none"` or `"all"`) fetches the feed through the SSRF-guarded
fetcher and returns the channel, season counts, `listed_count` and the
newest 50 episodes of the season, without storing anything; subscribe
fetches the origin URL, then `runner::subscribe_fetched` writes a
private subscription with the card's id, or the account's existing
podcast for a known URL of `runner::document_urls` (the origin, the end
of a permanent redirect chain, the feed's `itunes:new-feed-url`; a
subscribed podcast first, `search::known_podcast`; also used by "add
RSS"). The document is
ingested only into a new podcast or when it came from the existing
podcast's own fetch URL (requested there, or a permanent redirect ended
there); otherwise the podcast is subscribed again with its stored
metadata and nothing of the document is written. Nothing is written when
the account is already subscribed, by id or known URL, before or after
the fetch. Both share
the "add RSS" request limits; the browser aborts a preview it no longer
shows (`AbortController`), so a closed card frees its slot once the
server sees the connection close.
- The browser keeps only expected, typed card fields (`podcasts.foundHit`):
ids must look like podcast ids, URLs must be absolute `http(s)` without
credentials (artwork: `https` only), and texts render with `x-text`.
## Contributing
Bug reports, design discussions, and patches are welcome. Before submitting a
+11
View File
@@ -545,6 +545,8 @@ struct AdminSettingsValues {
youtube_proxy_id: String,
#[serde(default)]
youtube_cookie_id: String,
#[serde(default)]
podcast_audio_proxy_enabled: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
@@ -615,6 +617,7 @@ struct AdminSettingsSources {
torrent_proxy_id: &'static str,
youtube_proxy_id: &'static str,
youtube_cookie_id: &'static str,
podcast_audio_proxy_enabled: &'static str,
}
#[derive(Debug, Deserialize)]
@@ -667,6 +670,8 @@ pub(super) struct UpdateSettingsRequest {
youtube_proxy_id: String,
#[serde(default)]
youtube_cookie_id: String,
#[serde(default)]
podcast_audio_proxy_enabled: bool,
}
fn default_similarity_model() -> String {
@@ -1370,6 +1375,10 @@ pub async fn update_settings(
("torrent_proxy_id", torrent_proxy_id),
("youtube_proxy_id", youtube_proxy_id),
("youtube_cookie_id", youtube_cookie_id),
(
"podcast_audio_proxy_enabled",
body.podcast_audio_proxy_enabled.to_string(),
),
];
for (key, value) in fields {
let mut entry = ConfigEntry::new(key.to_string(), value);
@@ -1714,6 +1723,7 @@ fn settings_dto(
torrent_proxy_id: config.torrent_proxy_id,
youtube_proxy_id: config.youtube_proxy_id,
youtube_cookie_id: config.youtube_cookie_id,
podcast_audio_proxy_enabled: config.podcast_audio_proxy_enabled,
},
sources: AdminSettingsSources {
auth_password_enabled: sources.auth_password_enabled.code(),
@@ -1750,6 +1760,7 @@ fn settings_dto(
torrent_proxy_id: sources.torrent_proxy_id.code(),
youtube_proxy_id: sources.youtube_proxy_id.code(),
youtube_cookie_id: sources.youtube_cookie_id.code(),
podcast_audio_proxy_enabled: sources.podcast_audio_proxy_enabled.code(),
},
}
}
+10
View File
@@ -149,6 +149,7 @@ pub struct ConfigSources {
pub torrent_proxy_id: ConfigSource,
pub youtube_proxy_id: ConfigSource,
pub youtube_cookie_id: ConfigSource,
pub podcast_audio_proxy_enabled: ConfigSource,
}
impl Default for ConfigSources {
@@ -190,6 +191,7 @@ impl Default for ConfigSources {
torrent_proxy_id: ConfigSource::Default,
youtube_proxy_id: ConfigSource::Default,
youtube_cookie_id: ConfigSource::Default,
podcast_audio_proxy_enabled: ConfigSource::Default,
}
}
}
@@ -407,6 +409,10 @@ pub struct AppConfig {
pub youtube_proxy_id: String,
/// Saved cookie-file id used by yt-dlp; empty means no cookies.
pub youtube_cookie_id: String,
/// Whether the server proxies podcast episode audio from publishers for
/// browsers that cannot load it directly (CORS, mixed content). Off by
/// default: browsers stream from the publisher URL.
pub podcast_audio_proxy_enabled: bool,
}
impl Default for AppConfig {
@@ -451,6 +457,7 @@ impl Default for AppConfig {
torrent_proxy_id: String::new(),
youtube_proxy_id: String::new(),
youtube_cookie_id: String::new(),
podcast_audio_proxy_enabled: false,
}
}
}
@@ -493,6 +500,7 @@ impl_env_overrides!(
torrent_proxy_id,
youtube_proxy_id,
youtube_cookie_id,
podcast_audio_proxy_enabled,
);
impl AppConfig {
@@ -636,6 +644,7 @@ impl AppConfig {
apply_db_field!(torrent_proxy_id);
apply_db_field!(youtube_proxy_id);
apply_db_field!(youtube_cookie_id);
apply_db_field!(podcast_audio_proxy_enabled);
}
pub fn parsed_download_proxies(&self) -> anyhow::Result<Vec<DownloadProxy>> {
@@ -693,6 +702,7 @@ mod tests {
assert!(cfg.torrent_downloads_enabled);
assert!(cfg.youtube_downloads_enabled);
assert!(cfg.parsed_download_proxies().unwrap().is_empty());
assert!(!cfg.podcast_audio_proxy_enabled);
}
#[test]
+115 -14
View File
@@ -1,43 +1,64 @@
//! Informational publication of the protocol versions exposed by this peer.
//! Informational publication of the protocol versions exposed by this peer,
//! and observation of the peers' manifests for the podcast-discovery overlay.
use std::time::Duration;
use std::collections::{BTreeSet, HashMap};
use std::sync::Arc;
use std::time::{Duration, Instant};
use anyhow::Result;
use music_dht::StreamAcceptor;
use anyhow::{Context, Result};
use futures_util::StreamExt;
use music_dht::capabilities::{
CAPABILITIES_PROTOCOL_VERSION, CapabilityManifest, CapabilityMessage, JAM_ID, SIMILARITY_ID,
read_message, write_message,
CAPABILITIES_ALPN, CAPABILITIES_PROTOCOL_VERSION, CapabilityManifest, CapabilityMessage,
JAM_ID, SIMILARITY_ID, read_message, write_message,
};
use music_dht::podcast_dht::PodcastDht;
use music_dht::{EndpointId, MusicDhtService, StreamAcceptor};
use super::serve::AUDIO_PROTOCOL_VERSION;
fn local_manifest() -> CapabilityManifest {
CapabilityManifest::frid("furumusic", env!("CARGO_PKG_VERSION"))
/// How often known peers are considered for a manifest probe.
const PROBE_INTERVAL: Duration = Duration::from_secs(5 * 60);
/// A peer's manifest is probed again after this long.
const REPROBE_AFTER: Duration = Duration::from_secs(60 * 60);
/// Peers probed per round and in parallel.
const PROBES_PER_ROUND: usize = 64;
const PROBE_CONCURRENCY: usize = 8;
const PROBE_TIMEOUT: Duration = Duration::from_secs(5);
/// The manifest of this node. The public podcast-discovery overlay is
/// advertised only while the node serves it (`podcast_dht`).
fn local_manifest(podcast_dht: bool) -> CapabilityManifest {
let manifest = CapabilityManifest::frid("furumusic", env!("CARGO_PKG_VERSION"))
// The web server does not expose federation Jam yet.
.without_protocol(JAM_ID)
.with_protocol("audio", AUDIO_PROTOCOL_VERSION)
.with_protocol(
SIMILARITY_ID,
music_dht::similarity::SIMILARITY_PROTOCOL_VERSION,
)
);
if podcast_dht {
manifest.with_podcast_dht()
} else {
manifest
}
}
pub async fn serve(mut acceptor: StreamAcceptor) {
pub async fn serve(mut acceptor: StreamAcceptor, podcast_dht: bool) {
while let Some(stream) = acceptor.accept().await {
tokio::spawn(async move {
if let Err(error) = serve_one(stream).await {
if let Err(error) = serve_one(stream, podcast_dht).await {
tracing::debug!("capability stream failed: {error:#}");
}
});
}
}
async fn serve_one(mut stream: music_dht::ByteStream) -> Result<()> {
async fn serve_one(mut stream: music_dht::ByteStream, podcast_dht: bool) -> Result<()> {
let response = match read_message(&mut stream).await? {
CapabilityMessage::Get {
version: CAPABILITIES_PROTOCOL_VERSION,
} => CapabilityMessage::Manifest {
manifest: local_manifest(),
manifest: local_manifest(podcast_dht),
},
CapabilityMessage::Get { version } => CapabilityMessage::Error {
message: format!("unsupported capability protocol {version}"),
@@ -52,13 +73,93 @@ async fn serve_one(mut stream: music_dht::ByteStream) -> Result<()> {
Ok(())
}
/// Probes the manifests of known peers and tells the podcast overlay which
/// of them lack it ([`PodcastDht::note_peer_capabilities`]), so bootstrap
/// stops dialing them. A peer that does not answer is left alone: the
/// overlay only learns contacts that answer on its own protocol anyway.
pub async fn probe_loop(service: Arc<MusicDhtService>, dht: Arc<PodcastDht>) {
let mut probed: HashMap<EndpointId, Instant> = HashMap::new();
let mut interval = tokio::time::interval(PROBE_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
interval.tick().await;
let own = service.endpoint_id();
probed.retain(|_, at| at.elapsed() < REPROBE_AFTER);
let peers: Vec<EndpointId> = service
.connected_peers()
.into_iter()
.chain(
service
.known_peers()
.into_iter()
.map(|contact| contact.peer_id),
)
.filter(|peer| *peer != own && !probed.contains_key(peer))
.collect::<BTreeSet<_>>()
.into_iter()
.take(PROBES_PER_ROUND)
.collect();
let results = futures_util::stream::iter(peers.into_iter().map(|peer| {
let service = Arc::clone(&service);
async move { (peer, probe_peer(&service, peer).await) }
}))
.buffer_unordered(PROBE_CONCURRENCY)
.collect::<Vec<_>>()
.await;
for (peer, result) in results {
probed.insert(peer, Instant::now());
match result {
Ok(manifest) => {
if let Err(error) = dht.note_peer_capabilities(peer, &manifest).await {
tracing::debug!(%peer, "podcast overlay capability note failed: {error}");
}
}
Err(error) => {
tracing::trace!(%peer, "peer capability probe unavailable: {error:#}");
}
}
}
}
}
async fn probe_peer(service: &MusicDhtService, peer: EndpointId) -> Result<CapabilityManifest> {
let mut stream =
tokio::time::timeout(PROBE_TIMEOUT, service.open_stream(peer, CAPABILITIES_ALPN))
.await
.context("capability stream timed out")??;
write_message(
&mut stream,
&CapabilityMessage::Get {
version: CAPABILITIES_PROTOCOL_VERSION,
},
)
.await?;
stream.send.finish()?;
let response = tokio::time::timeout(PROBE_TIMEOUT, read_message(&mut stream))
.await
.context("capability request timed out")??;
let CapabilityMessage::Manifest { manifest } = response else {
anyhow::bail!("peer did not return a capability manifest");
};
manifest.validate()?;
Ok(manifest)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_podcast_overlay_is_advertised_only_while_it_runs() {
assert!(!local_manifest(false).supports_podcast_dht());
let manifest = local_manifest(true);
assert!(manifest.supports_podcast_dht());
manifest.validate().unwrap();
}
#[test]
fn manifest_describes_only_supported_player_protocols() {
let manifest = local_manifest();
let manifest = local_manifest(false);
assert_eq!(manifest.application, "furumusic");
assert_eq!(
manifest.protocols.get("audio"),
+326 -151
View File
@@ -9,6 +9,7 @@ use anyhow::{Context, Result};
use music_dht::{ItemKind, LibraryItem, normalize_content_id};
use serde::Serialize;
use serde_json::{Value, json};
use sqlx::PgPool;
use sqlx::Row as _;
use tokio::io::AsyncReadExt;
@@ -94,6 +95,9 @@ pub struct SearchEvent {
pub search_id: String,
pub sequence: u64,
pub kind: &'static str,
/// `music` or `podcast` (`music_dht::podcast_dht::FederatedSearchHit`).
#[serde(rename = "type")]
pub hit_type: &'static str,
pub peer: Option<String>,
pub entity_key: Value,
pub entity: Value,
@@ -218,7 +222,12 @@ impl Federation {
/// Performs one bounded DHT search and returns entity upserts. The HTTP
/// layer streams each upsert independently; catalog fan-out can append
/// events to the same contract without changing the browser model.
pub async fn search_events(&self, search_id: &str, query: &str) -> Result<Vec<SearchEvent>> {
///
/// Music only (`federation.track|artist|release`, marked `"type":
/// "music"`). Public podcasts are searched next to it
/// ([`Self::spawn_podcast_search`]) and follow as `federation.podcast`
/// events ([`Self::podcast_search_events`]).
pub async fn search_events(&self, search_id: &str, query: &str) -> Result<MusicSearch> {
let query = query.trim();
anyhow::ensure!(!query.is_empty(), "search query is empty");
anyhow::ensure!(query.chars().count() <= 200, "search query is too long");
@@ -233,13 +242,10 @@ impl Federation {
"federated search started"
);
let own = service.endpoint_id();
let result = tokio::time::timeout(
std::time::Duration::from_secs(20),
service.search_network(query),
)
.await
.map_err(|_| anyhow::anyhow!("federated search timed out after 20 seconds"))?
.map_err(|err| anyhow::anyhow!("federated search failed: {err}"))?;
let result = tokio::time::timeout(SEARCH_TIMEOUT, service.search_network(query))
.await
.map_err(|_| anyhow::anyhow!("federated search timed out after 20 seconds"))?
.map_err(|err| anyhow::anyhow!("federated search failed: {err}"))?;
tracing::info!(
search_id,
query,
@@ -249,154 +255,14 @@ impl Federation {
elapsed_ms = started.elapsed().as_millis() as u64,
"federated DHT search finished"
);
let all_items: Vec<LibraryItem> = result
let items: Vec<LibraryItem> = result
.local_results
.into_iter()
.chain(result.network_results)
.collect();
let pool = self.pool().await?;
let mut by_content: HashMap<String, TrackDto> = HashMap::new();
for item in all_items.iter().filter(|item| item.kind == ItemKind::Track) {
let Some(content_id) = item.content_id.as_deref().and_then(normalize_content_id) else {
// A globally usable track reference must be verifiable.
continue;
};
let local = local_availability(&pool, &content_id).await?;
let source = FederationSourceDto {
owner: item.owner.to_string(),
item_id: hex(item.id.as_bytes()),
};
let entry = by_content.entry(content_id.clone()).or_insert_with(|| {
track_from_item(content_id.clone(), item, local, item.owner == own)
});
if !entry.availability.federation.iter().any(|candidate| {
candidate.owner == source.owner && candidate.item_id == source.item_id
}) {
entry.availability.federation.push(source);
}
if entry.availability.local.is_some() {
entry.availability.state = "local";
}
}
let mut tracks: Vec<_> = by_content.into_values().collect();
tracks.sort_by(|left, right| {
left.metadata
.title
.to_lowercase()
.cmp(&right.metadata.title.to_lowercase())
});
let mut events = Vec::with_capacity(all_items.len());
for (index, track) in tracks.into_iter().enumerate() {
persist_track_ref(&pool, &track).await?;
let peer = track
.availability
.federation
.first()
.map(|source| source.owner.clone());
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence: index as u64 + 1,
kind: "federation.track",
peer,
entity_key: serde_json::to_value(&track.key)?,
entity: serde_json::to_value(track)?,
});
}
let mut artist_peers: HashMap<String, (String, Vec<String>)> = HashMap::new();
let mut releases: HashMap<String, Value> = HashMap::new();
for item in &all_items {
match item.kind {
ItemKind::Artist => {
let key = music_dht::normalize_name(&item.name);
let entry = artist_peers
.entry(key)
.or_insert_with(|| (item.name.clone(), Vec::new()));
let owner = item.owner.to_string();
if !entry.1.contains(&owner) {
entry.1.push(owner);
}
}
ItemKind::Release => {
let artist_keys: Vec<String> = item
.artist_names
.iter()
.map(|name| music_dht::normalize_name(name))
.collect();
let normalized_title = music_dht::normalize_name(&item.name);
let cover_url = all_items
.iter()
.find(|track| {
track.kind == ItemKind::Track
&& track.release_title.as_deref().is_some_and(|title| {
music_dht::normalize_name(title) == normalized_title
})
&& track.year == item.year
})
.map(|track| {
format!(
"/api/player/federation/tracks/artwork?owner={}&item_id={}",
track.owner,
hex(track.id.as_bytes())
)
});
let key = format!(
"{}|{}|{}",
normalized_title,
artist_keys.join(","),
item.year.map_or_else(String::new, |year| year.to_string())
);
releases.entry(key.clone()).or_insert_with(|| {
json!({
"key": {
"normalized_title": music_dht::normalize_name(&item.name),
"primary_artists": artist_keys,
"release_type": null,
"year": item.year,
},
"title": item.name,
"artists": item.artist_names,
"year": item.year,
"cover_url": cover_url,
"sources": [{
"owner": item.owner.to_string(),
"item_id": hex(item.id.as_bytes()),
}],
})
});
}
ItemKind::Track => {}
}
}
for (key, (name, peers)) in artist_peers {
let sequence = events.len() as u64 + 1;
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.artist",
peer: peers.first().cloned(),
entity_key: json!({ "normalized_name": key }),
entity: json!({
"key": { "normalized_name": key },
"name": name,
"image_url": null,
"peers": peers,
}),
});
}
for (key, release) in releases {
let sequence = events.len() as u64 + 1;
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.release",
peer: None,
entity_key: json!({ "composite": key }),
entity: release,
});
}
let events = music_search_events(&pool, search_id, &items, own).await?;
tracing::info!(
search_id,
query,
@@ -404,8 +270,317 @@ impl Federation {
elapsed_ms = started.elapsed().as_millis() as u64,
"federated search response ready"
);
Ok(events)
Ok(MusicSearch { events, items })
}
/// Starts the public podcast search of `query` on the discovery
/// overlay, if it runs, next to the music search.
pub fn spawn_podcast_search(&self, query: &str) -> tokio::task::JoinHandle<PodcastSearch> {
let discovery = self.podcast_discovery();
let query = query.trim().to_owned();
tokio::spawn(async move { search_podcasts(discovery.as_deref(), &query).await })
}
/// The `federation.podcast` events of a search for `user_id`, numbered
/// after the music events (see [`podcast_events`]), and the
/// `podcast_discovery` state for `search.completed`.
///
/// Never fails: when the hits cannot be prepared for the account (a
/// database error) they are dropped and the state is `failed`, so the
/// music results stand and the browser can say the podcast search
/// failed.
pub async fn podcast_search_events(
&self,
search_id: &str,
query: &str,
user_id: i64,
music: &MusicSearch,
podcasts: PodcastSearch,
) -> (&'static str, Vec<SearchEvent>) {
let state = podcasts.state();
let PodcastSearch::Found(hits) = podcasts else {
return (state, Vec::new());
};
if hits.is_empty() {
return (state, Vec::new());
}
let events = async {
let pool = self.pool().await?;
let first_sequence = music.events.len() as u64 + 1;
podcast_events(
&pool,
user_id,
search_id,
query.trim(),
&music.items,
first_sequence,
hits,
)
.await
};
match events.await {
Ok(events) => {
tracing::info!(
search_id,
podcasts = events.len(),
"public podcast hits ready"
);
(state, events)
}
Err(error) => {
// Only the error: never the query or the hits.
tracing::warn!(
search_id,
"public podcast hits could not be prepared: {error:#}"
);
("failed", Vec::new())
}
}
}
}
/// Deadline of the music DHT and the podcast overlay searches.
const SEARCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
/// Music results of a federated search: the events to stream and the items
/// (podcast hits are ranked among them).
#[derive(Debug)]
pub struct MusicSearch {
pub events: Vec<SearchEvent>,
pub items: Vec<LibraryItem>,
}
/// Outcome of the public podcast search of one federated search.
#[derive(Debug)]
pub enum PodcastSearch {
/// Public discovery does not run on this server.
Off,
/// The overlay search failed or timed out (music results still count).
Failed,
/// Hits, possibly none.
Found(Vec<music_dht::podcast_dht::PodcastSearchHit>),
}
impl PodcastSearch {
/// `running`, `off` or `failed` (reported in `search.completed`).
pub fn state(&self) -> &'static str {
match self {
Self::Off => "off",
Self::Failed => "failed",
Self::Found(_) => "running",
}
}
}
async fn search_podcasts(
discovery: Option<&music_dht::podcast_dht::PodcastDht>,
query: &str,
) -> PodcastSearch {
let Some(dht) = discovery else {
return PodcastSearch::Off;
};
let limit = crate::podcasts::search::PODCAST_SEARCH_LIMIT;
match tokio::time::timeout(SEARCH_TIMEOUT, dht.search(query, limit)).await {
Ok(Ok(hits)) => PodcastSearch::Found(hits),
Ok(Err(error)) => {
tracing::info!("public podcast search failed: {error}");
PodcastSearch::Failed
}
Err(_) => {
tracing::info!("public podcast search timed out");
PodcastSearch::Failed
}
}
}
/// One `federation.podcast` event per public podcast hit, for `user_id`
/// (`subscribed`, `local_podcast_id`), ordered by the hit's place in
/// `merge_search_results` with the music items and numbered from
/// `first_sequence`.
pub(crate) async fn podcast_events(
pool: &PgPool,
user_id: i64,
search_id: &str,
query: &str,
music: &[LibraryItem],
first_sequence: u64,
hits: Vec<music_dht::podcast_dht::PodcastSearchHit>,
) -> Result<Vec<SearchEvent>> {
let store = crate::podcasts::PodcastStore::new(pool.clone());
let views = crate::podcasts::search::hit_views(&store, user_id, query, music, hits).await?;
views
.into_iter()
.zip(first_sequence..)
.map(|(view, sequence)| {
Ok(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.podcast",
hit_type: "podcast",
// Subscribing never contacts a publisher: no peer to name.
peer: None,
entity_key: json!({ "podcast_id": view.podcast_id }),
entity: serde_json::to_value(view)?,
})
})
.collect()
}
/// Music events of a search (tracks, then artists and releases).
pub(crate) async fn music_search_events(
pool: &PgPool,
search_id: &str,
all_items: &[LibraryItem],
own: music_dht::EndpointId,
) -> Result<Vec<SearchEvent>> {
let mut by_content: HashMap<String, TrackDto> = HashMap::new();
for item in all_items.iter().filter(|item| item.kind == ItemKind::Track) {
let Some(content_id) = item.content_id.as_deref().and_then(normalize_content_id) else {
// A globally usable track reference must be verifiable.
continue;
};
let local = local_availability(pool, &content_id).await?;
let source = FederationSourceDto {
owner: item.owner.to_string(),
item_id: hex(item.id.as_bytes()),
};
let entry = by_content
.entry(content_id.clone())
.or_insert_with(|| track_from_item(content_id.clone(), item, local, item.owner == own));
if !entry
.availability
.federation
.iter()
.any(|candidate| candidate.owner == source.owner && candidate.item_id == source.item_id)
{
entry.availability.federation.push(source);
}
if entry.availability.local.is_some() {
entry.availability.state = "local";
}
}
let mut tracks: Vec<_> = by_content.into_values().collect();
tracks.sort_by(|left, right| {
left.metadata
.title
.to_lowercase()
.cmp(&right.metadata.title.to_lowercase())
});
let mut events = Vec::with_capacity(all_items.len());
for (index, track) in tracks.into_iter().enumerate() {
persist_track_ref(pool, &track).await?;
let peer = track
.availability
.federation
.first()
.map(|source| source.owner.clone());
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence: index as u64 + 1,
kind: "federation.track",
hit_type: "music",
peer,
entity_key: serde_json::to_value(&track.key)?,
entity: serde_json::to_value(track)?,
});
}
let mut artist_peers: HashMap<String, (String, Vec<String>)> = HashMap::new();
let mut releases: HashMap<String, Value> = HashMap::new();
for item in all_items {
match item.kind {
ItemKind::Artist => {
let key = music_dht::normalize_name(&item.name);
let entry = artist_peers
.entry(key)
.or_insert_with(|| (item.name.clone(), Vec::new()));
let owner = item.owner.to_string();
if !entry.1.contains(&owner) {
entry.1.push(owner);
}
}
ItemKind::Release => {
let artist_keys: Vec<String> = item
.artist_names
.iter()
.map(|name| music_dht::normalize_name(name))
.collect();
let normalized_title = music_dht::normalize_name(&item.name);
let cover_url = all_items
.iter()
.find(|track| {
track.kind == ItemKind::Track
&& track.release_title.as_deref().is_some_and(|title| {
music_dht::normalize_name(title) == normalized_title
})
&& track.year == item.year
})
.map(|track| {
format!(
"/api/player/federation/tracks/artwork?owner={}&item_id={}",
track.owner,
hex(track.id.as_bytes())
)
});
let key = format!(
"{}|{}|{}",
normalized_title,
artist_keys.join(","),
item.year.map_or_else(String::new, |year| year.to_string())
);
releases.entry(key.clone()).or_insert_with(|| {
json!({
"key": {
"normalized_title": music_dht::normalize_name(&item.name),
"primary_artists": artist_keys,
"release_type": null,
"year": item.year,
},
"title": item.name,
"artists": item.artist_names,
"year": item.year,
"cover_url": cover_url,
"sources": [{
"owner": item.owner.to_string(),
"item_id": hex(item.id.as_bytes()),
}],
})
});
}
ItemKind::Track => {}
}
}
for (key, (name, peers)) in artist_peers {
let sequence = events.len() as u64 + 1;
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.artist",
hit_type: "music",
peer: peers.first().cloned(),
entity_key: json!({ "normalized_name": key }),
entity: json!({
"key": { "normalized_name": key },
"name": name,
"image_url": null,
"peers": peers,
}),
});
}
for (key, release) in releases {
let sequence = events.len() as u64 + 1;
events.push(SearchEvent {
search_id: search_id.to_owned(),
sequence,
kind: "federation.release",
hit_type: "music",
peer: None,
entity_key: json!({ "composite": key }),
entity: release,
});
}
Ok(events)
}
async fn fetch_artist_catalog(
+241 -62
View File
@@ -6,10 +6,10 @@
//! `furumi/sync/2` and maps operations into user-scoped Postgres state.
//!
//! The wire types are the shared definitions; there are no local copies.
//! Device-sync protocol v4 (podcasts) is announced because this adapter
//! keeps every v4 element it receives: podcast operations stay in the
//! generic operation log, podcast snapshot sections are merged into
//! [`super::podcast_relay`], and queue items keep their explicit
//! Device-sync protocol v4 (podcasts) is announced: podcast operations stay
//! in the generic operation log and are applied to the account's podcast
//! store ([`crate::podcasts::PodcastStore`]), which also provides the
//! podcast snapshot sections, and queue items keep their explicit
//! `PlaybackTrack.source` through the browser bridge
//! ([`crate::player::queue_items`]). Everything sent to a peer is gated by
//! the version that peer reported itself.
@@ -31,7 +31,7 @@ use music_dht::device_sync::{
random_hex, relayed_protocol_version, supports_listen_history, supports_podcasts,
ticket_endpoint_id,
};
use music_dht::podcast::ProgressRecord;
use music_dht::podcast::{EpisodeId, ProgressRecord};
use music_dht::{ByteStream, MusicDhtService, NetworkId, PeerTicket, SecretKey, StreamAcceptor};
use serde::Serialize;
use sqlx::Row;
@@ -39,8 +39,8 @@ use sqlx::Row;
use crate::player::PlayerDeviceHub;
use crate::player::queue_items;
use super::podcast_relay::{self, RelayRecord};
use super::{TransportStats, record_stream_transport};
use crate::podcasts::{ApplyOutcome, PodcastStore};
pub const SYNC_ALPN: &[u8] = b"furumi/sync/2";
@@ -800,16 +800,25 @@ pub async fn serve_peers(
}
}
/// Polls every trusted device of every account every
/// [`DEVICE_SYNC_INTERVAL`] (one exchange per device at a time).
///
/// `first_round` turns `true` once the first complete listing of the
/// devices has been polled: every exchange in flight then has answered,
/// failed or timed out (30 s). The podcast publication reconciler waits for
/// it (see [`crate::podcasts::discovery::run`]).
pub async fn sync_loop(
pool: sqlx::PgPool,
service: Arc<MusicDhtService>,
hub: Arc<PlayerDeviceHub>,
transport_stats: Arc<TransportStats>,
first_round: tokio::sync::watch::Sender<bool>,
) {
let mut interval = tokio::time::interval(DEVICE_SYNC_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut polls = tokio::task::JoinSet::new();
let mut active = std::collections::HashMap::new();
let mut first_round = FirstSyncRound::new(first_round);
loop {
tokio::select! {
_ = interval.tick() => {
@@ -819,10 +828,15 @@ pub async fn sync_loop(
Ok(users) => users,
Err(error) => { tracing::warn!("device poll listing failed: {error:#}"); continue; }
};
let mut listed_all = true;
for user_id in users {
let devices = match active_remote_devices(&pool, user_id).await {
Ok(devices) => devices,
Err(error) => { let _ = set_last_error(&pool, user_id, Some(&format!("{error:#}"))).await; continue; }
Err(error) => {
listed_all = false;
let _ = set_last_error(&pool, user_id, Some(&format!("{error:#}"))).await;
continue;
}
};
for device in devices {
let key = (user_id, device.device_id.clone());
@@ -838,12 +852,17 @@ pub async fn sync_loop(
active.insert(handle.id(), key);
}
}
if listed_all {
// Includes exchanges still running from an earlier tick.
first_round.started(active.keys().copied());
}
}
Some(completed) = polls.join_next_with_id(), if !polls.is_empty() => {
let (task, result) = match completed {
Ok((task, result)) => (task, result),
Err(error) => (error.id(), Err(anyhow::Error::from(error))),
};
first_round.finished(&task);
if let Some((user_id, device_id)) = active.remove(&task)
&& let Err(error) = result {
tracing::debug!(device = %device_id, "web fed device sync failed: {error:#}");
@@ -854,6 +873,50 @@ pub async fn sync_loop(
}
}
/// Tracks the first device-sync round of a federation run for
/// [`sync_loop`]: the exchanges started by the first complete listing of the
/// trusted devices (and those still running then). The watch turns `true`
/// when the last of them ends, or at once when there are none, and stays
/// `true`.
pub(crate) struct FirstSyncRound<K> {
pending: Option<std::collections::HashSet<K>>,
done: tokio::sync::watch::Sender<bool>,
}
impl<K: Eq + std::hash::Hash> FirstSyncRound<K> {
pub(crate) fn new(done: tokio::sync::watch::Sender<bool>) -> Self {
Self {
pending: None,
done,
}
}
/// The first complete listing started (or found running) `polls`;
/// later listings are ignored.
pub(crate) fn started(&mut self, polls: impl IntoIterator<Item = K>) {
if self.pending.is_some() || *self.done.borrow() {
return;
}
let pending: std::collections::HashSet<K> = polls.into_iter().collect();
if pending.is_empty() {
self.done.send_replace(true);
} else {
self.pending = Some(pending);
}
}
/// An exchange ended (answered, failed or timed out).
pub(crate) fn finished(&mut self, poll: &K) {
if let Some(pending) = &mut self.pending
&& pending.remove(poll)
&& pending.is_empty()
{
self.pending = None;
self.done.send_replace(true);
}
}
}
pub async fn sync_once(
pool: &sqlx::PgPool,
service: Arc<MusicDhtService>,
@@ -1782,20 +1845,7 @@ async fn record_local_op(
);
}
let now = now_ms();
let row = sqlx::query(
"UPDATE furumusic__fed_device_identity
SET local_seq = local_seq + 1,
last_hlc_ms = GREATEST($2, last_hlc_ms + 1)
WHERE user_id = $1
RETURNING local_seq, last_hlc_ms",
)
.bind(user_id)
.bind(now)
.fetch_one(pool)
.await?;
let seq: i64 = row.get("local_seq");
let hlc_ms: i64 = row.get("last_hlc_ms");
let (seq, hlc_ms) = reserve_local_stamp(pool, user_id).await?;
let op_id = format!("{}:{seq}", identity.device_id);
let op = SyncOpWire {
op_id,
@@ -1805,6 +1855,40 @@ async fn record_local_op(
payload,
};
insert_op(pool, user_id, &op).await?;
advance_vector(pool, user_id, &op.origin_device_id, op.seq).await?;
let _ = apply_op(pool, Arc::new(PlayerDeviceHub::default()), user_id, &op).await?;
Ok(())
}
/// Reserves the next local `(seq, hlc_ms)` of the account's web device.
///
/// `hlc_ms` is the wall clock, but always past the previous local
/// operation. Inside a transaction the identity row stays locked until it
/// ends, so local operations of one account are stamped in commit order.
async fn reserve_local_stamp<'e>(
db: impl sqlx::PgExecutor<'e>,
user_id: i64,
) -> Result<(i64, i64)> {
let row = sqlx::query(
"UPDATE furumusic__fed_device_identity
SET local_seq = local_seq + 1,
last_hlc_ms = GREATEST($2, last_hlc_ms + 1)
WHERE user_id = $1
RETURNING local_seq, last_hlc_ms",
)
.bind(user_id)
.bind(now_ms())
.fetch_one(db)
.await?;
Ok((row.get("local_seq"), row.get("last_hlc_ms")))
}
async fn advance_vector<'e>(
db: impl sqlx::PgExecutor<'e>,
user_id: i64,
device_id: &str,
seq: i64,
) -> Result<()> {
sqlx::query(
"INSERT INTO furumusic__fed_sync_vector (user_id, device_id, max_seq)
VALUES ($1, $2, $3)
@@ -1812,14 +1896,51 @@ async fn record_local_op(
max_seq = GREATEST(furumusic__fed_sync_vector.max_seq, EXCLUDED.max_seq)",
)
.bind(user_id)
.bind(&op.origin_device_id)
.bind(op.seq)
.execute(pool)
.bind(device_id)
.bind(seq)
.execute(db)
.await?;
let _ = apply_op(pool, Arc::new(PlayerDeviceHub::default()), user_id, &op).await?;
Ok(())
}
/// Device id of the account's web device, creating the identity on first
/// use. Local podcast actions stamp their operations with it.
pub(crate) async fn local_device_id(pool: &sqlx::PgPool, user_id: i64) -> Result<String> {
Ok(ensure_identity(pool, user_id, "").await?.device_id)
}
/// Appends a local operation of `device_id` (the account's web device, see
/// [`local_device_id`]) to the sync log inside the caller's transaction.
///
/// The stamp (`seq`, `hlc_ms`, `op_id = <device>:<seq>`) is reserved here;
/// `make` receives the `op_id`. The caller applies the returned operation
/// itself in the same transaction, so the log and the materialized state
/// change together (the podcast store does this for its local actions).
pub(crate) async fn append_local_op(
conn: &mut sqlx::PgConnection,
user_id: i64,
device_id: &str,
make: impl FnOnce(&str) -> SyncOpPayload,
) -> Result<SyncOpWire> {
let (seq, hlc_ms) = reserve_local_stamp(&mut *conn, user_id).await?;
let op_id = format!("{device_id}:{seq}");
let payload = make(&op_id);
let op = SyncOpWire {
op_id,
origin_device_id: device_id.to_string(),
seq,
hlc_ms,
payload,
};
anyhow::ensure!(
insert_op(&mut *conn, user_id, &op).await?,
"local operation {} already exists",
op.op_id
);
advance_vector(&mut *conn, user_id, device_id, seq).await?;
Ok(op)
}
async fn apply_ops(
pool: &sqlx::PgPool,
hub: Arc<PlayerDeviceHub>,
@@ -1837,23 +1958,17 @@ async fn apply_ops(
} else if insert_op(pool, user_id, &op).await? {
let _ = apply_op(pool, Arc::clone(&hub), user_id, &op).await?;
}
sqlx::query(
"INSERT INTO furumusic__fed_sync_vector (user_id, device_id, max_seq)
VALUES ($1, $2, $3)
ON CONFLICT (user_id, device_id) DO UPDATE SET
max_seq = GREATEST(furumusic__fed_sync_vector.max_seq, EXCLUDED.max_seq)",
)
.bind(user_id)
.bind(&op.origin_device_id)
.bind(op.seq)
.execute(pool)
.await?;
advance_vector(pool, user_id, &op.origin_device_id, op.seq).await?;
}
Ok(())
}
/// Appends an operation to the log; `false` when it was already stored.
async fn insert_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire) -> Result<bool> {
async fn insert_op<'e>(
db: impl sqlx::PgExecutor<'e>,
user_id: i64,
op: &SyncOpWire,
) -> Result<bool> {
let inserted = sqlx::query(
"INSERT INTO furumusic__fed_sync_ops
(user_id, op_id, origin_device_id, seq, kind, payload_json, hlc_ms,
@@ -1870,7 +1985,7 @@ async fn insert_op(pool: &sqlx::PgPool, user_id: i64, op: &SyncOpWire) -> Result
.bind(op.hlc_ms)
.bind(now_ms())
.bind(op.payload.is_tombstone())
.execute(pool)
.execute(db)
.await?
.rows_affected();
Ok(inserted > 0)
@@ -2035,12 +2150,19 @@ async fn apply_op(
| SyncOpPayload::PodcastFeedAliasesAdded { .. }
| SyncOpPayload::PodcastProgressSet { .. }
| SyncOpPayload::PodcastListenRecorded { .. } => {
// The web player has no podcast features yet: keep the registers
// for this device's snapshots; the operation itself is relayed
// from the log.
let records: Vec<_> = RelayRecord::from_op(op).into_iter().collect();
podcast_relay::merge_records(pool, user_id, &records).await?;
Ok(false)
// The operation itself is relayed from the log; the store keeps
// the registers (and the catalog rows of its episode reference).
match PodcastStore::new(pool.clone())
.apply_remote_op(user_id, op)
.await?
{
ApplyOutcome::Changed => Ok(true),
ApplyOutcome::Rejected(reason) => {
tracing::debug!(op_id = %op.op_id, "skipped invalid podcast operation: {reason}");
Ok(false)
}
ApplyOutcome::Unchanged | ApplyOutcome::Ignored => Ok(false),
}
}
}
}
@@ -2407,13 +2529,17 @@ async fn apply_playlist_item_state(
}
async fn apply_snapshot(pool: &sqlx::PgPool, user_id: i64, snapshot: SyncSnapshot) -> Result<()> {
let podcast = SyncSnapshot {
podcast_subscriptions: snapshot.podcast_subscriptions.clone(),
podcast_shares: snapshot.podcast_shares.clone(),
podcast_progress: snapshot.podcast_progress.clone(),
podcast_listen_sessions: snapshot.podcast_listen_sessions.clone(),
..SyncSnapshot::default()
};
let podcasts = PodcastStore::new(pool.clone())
.apply_snapshot(user_id, &snapshot)
.await?;
if podcasts.ignored_over_limits {
tracing::debug!("ignored podcast snapshot sections above the contract limits");
} else if podcasts.rejected > 0 {
tracing::debug!(
rejected = podcasts.rejected,
"skipped invalid podcast snapshot records"
);
}
for like in snapshot.likes {
apply_like_state(
pool,
@@ -2490,7 +2616,6 @@ async fn apply_snapshot(pool: &sqlx::PgPool, user_id: i64, snapshot: SyncSnapsho
)
.await?;
}
podcast_relay::merge_records(pool, user_id, &RelayRecord::from_snapshot(&podcast)).await?;
Ok(())
}
@@ -3658,7 +3783,7 @@ fn playback_clock() -> u64 {
///
/// When the browser does not know the position of its current item
/// ([`queue_items::POSITION_UNKNOWN_KEY`]) and that item is a podcast
/// episode, the position comes from the relayed progress register: handing
/// episode, the position comes from its synced progress register: handing
/// 0 to the next device would be written there as an explicit action and
/// restart the episode everywhere.
async fn playback_state_from_browser_json(
@@ -3668,19 +3793,64 @@ async fn playback_state_from_browser_json(
) -> Result<PlaybackStateWire> {
let unknown = browser_bool(&state, queue_items::POSITION_UNKNOWN_KEY) == Some(true);
let mut wire = browser_json_to_wire(pool, state).await?;
if unknown
&& let Some(episode) = wire
.queue
.get(wire.queue_pos)
.and_then(PlaybackTrack::podcast_episode_ref)
{
let record =
podcast_relay::progress_record(pool, user_id, episode.episode_id.as_str()).await?;
if let Some(episode_id) = episode_needing_resume_position(unknown, &wire) {
let record = PodcastStore::new(pool.clone())
.progress(user_id, &episode_id)
.await?;
apply_resume_position(&mut wire, record.as_ref());
}
let episodes: Vec<EpisodeId> = wire
.queue
.iter()
.filter_map(PlaybackTrack::podcast_episode_ref)
.map(|episode| episode.episode_id.clone())
.collect();
if !episodes.is_empty() {
let generations = PodcastStore::new(pool.clone())
.progress_generations(user_id, &episodes)
.await?;
apply_progress_generations(&mut wire, &generations);
}
Ok(wire)
}
/// Raises `PodcastEpisodeRef::progress_generation` of every episode in the
/// queue to the generation of its progress register here.
///
/// The browser keeps the hint its queue item came with, which can be older
/// than what this account has seen since (its own checkpoints, operations
/// from other devices). The device taking over writes its explicit action
/// above this value (frid `PODCASTS.md`, "Handoff").
fn apply_progress_generations(
state: &mut PlaybackStateWire,
generations: &std::collections::HashMap<EpisodeId, u64>,
) {
for track in &mut state.queue {
if let Some(PlaybackSource::PodcastEpisode(episode)) = track.source.as_mut()
&& let Some(generation) = generations.get(&episode.episode_id)
{
episode.progress_generation = Some(
episode
.progress_generation
.map_or(*generation, |seen| seen.max(*generation)),
);
}
}
}
/// The current episode whose position comes from its progress register:
/// only when the browser said it does not know the position. Known
/// positions (handoffs, seeks, playing) and music are never replaced.
fn episode_needing_resume_position(unknown: bool, wire: &PlaybackStateWire) -> Option<EpisodeId> {
if !unknown {
return None;
}
wire.queue
.get(wire.queue_pos)
.and_then(PlaybackTrack::podcast_episode_ref)
.map(|episode| episode.episode_id.clone())
}
/// Sets the position of the current podcast episode to where ordinary Play
/// resumes it (`ProgressRecord::resume_position_ms`, 0 without a register),
/// and records the register's generation as seen for the handoff.
@@ -4590,7 +4760,14 @@ async fn snapshot(pool: &sqlx::PgPool, user_id: i64) -> Result<SyncSnapshot> {
removed_playlist_items,
..SyncSnapshot::default()
};
podcast_relay::fill_snapshot(pool, user_id, &mut snapshot).await?;
// Bounded again per message by `fit_podcast_snapshot`.
PodcastStore::new(pool.clone())
.fill_snapshot(
user_id,
&mut snapshot,
music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES,
)
.await?;
Ok(snapshot)
}
@@ -4967,6 +5144,8 @@ fn short_id(value: &str) -> String {
value.chars().take(10).collect()
}
#[cfg(test)]
mod first_round_tests;
#[cfg(test)]
mod gating_tests;
#[cfg(test)]
@@ -0,0 +1,34 @@
//! The first device-sync round that the podcast publication reconciler
//! waits for ([`FirstSyncRound`]).
use super::FirstSyncRound;
#[test]
fn the_first_round_ends_when_its_last_exchange_ends() {
let (done, watch) = tokio::sync::watch::channel(false);
let mut round = FirstSyncRound::new(done);
// An exchange that ends before the first complete listing is not
// waited for.
round.finished(&7);
assert!(!*watch.borrow());
round.started([1, 2]);
// A later listing (with exchanges it started) changes nothing.
round.started([3]);
round.finished(&1);
round.finished(&3);
assert!(!*watch.borrow());
round.finished(&2);
assert!(*watch.borrow());
// Stays done.
round.started([4]);
round.finished(&4);
assert!(*watch.borrow());
}
#[test]
fn without_trusted_devices_the_first_round_ends_at_once() {
let (done, watch) = tokio::sync::watch::channel(false);
let mut round = FirstSyncRound::<u32>::new(done);
round.started([]);
assert!(*watch.borrow());
}
+61 -15
View File
@@ -365,7 +365,10 @@ async fn podcast_queue_survives_server_browser_server_round_trip() {
assert_eq!(payload["index"], 3);
assert_eq!(payload["position_seconds"], 1200.0);
assert_eq!(tracks[0]["source"]["kind"], "podcast_episode");
assert_eq!(tracks[0]["unavailable"], true);
assert_eq!(
tracks[0]["unavailable"], false,
"episodes play on the web player"
);
assert_eq!(tracks[1]["source"]["kind"], "private");
// The browser keeps its queue and sends it back (plus a library track
@@ -380,10 +383,10 @@ async fn podcast_queue_survives_server_browser_server_round_trip() {
"release_id": 9, "release_title": "Album",
}));
// browser -> server -> peer
let outgoing = playback_state_from_browser_json(&pool, 1, browser)
.await
.unwrap();
// browser -> server -> peer (the bridge itself; the progress-register
// lookups of `playback_state_from_browser_json` need a database and
// are covered by `apply_progress_generations_*` and the Postgres tests)
let outgoing = browser_json_to_wire(&pool, browser).await.unwrap();
assert_eq!(outgoing.queue[..4], queue[..]);
assert_eq!(outgoing.queue_pos, 3);
assert!(outgoing.paused);
@@ -400,6 +403,42 @@ async fn podcast_queue_survives_server_browser_server_round_trip() {
assert!(playback_allowed_for_peer(&outgoing, 4));
}
#[test]
fn apply_progress_generations_raises_stale_handoff_hints_only() {
let mut first = test_episode();
first.progress_generation = Some(9);
let mut second = test_episode();
second.episode_id = EpisodeId::from_guid(&second.podcast_id, "episode-43").unwrap();
second.progress_generation = None;
let mut third = test_episode();
third.episode_id = EpisodeId::from_guid(&third.podcast_id, "episode-44").unwrap();
third.progress_generation = Some(2);
let mut wire = state(vec![
PlaybackTrack::podcast_episode(first.clone()),
PlaybackTrack::podcast_episode(second.clone()),
music_track(3),
PlaybackTrack::podcast_episode(third.clone()),
]);
let generations = std::collections::HashMap::from([
(first.episode_id.clone(), 4),
(second.episode_id.clone(), 6),
(third.episode_id.clone(), 5),
]);
apply_progress_generations(&mut wire, &generations);
let hints: Vec<Option<u64>> = wire
.queue
.iter()
.map(|track| {
track
.podcast_episode_ref()
.and_then(|e| e.progress_generation)
})
.collect();
// A newer carried hint stays; missing and older ones are raised.
assert_eq!(hints, vec![Some(9), Some(6), None, Some(5)]);
assert_eq!(wire.queue[2], music_track(3), "music is untouched");
}
#[tokio::test]
async fn malformed_browser_items_keep_the_position_on_its_item() {
let pool = offline_pool();
@@ -555,21 +594,28 @@ async fn unknown_episode_position_resumes_from_the_progress_register() {
}
#[tokio::test]
async fn known_positions_and_music_never_read_the_progress_register() {
// The offline pool fails any query, so these conversions must not query.
async fn known_positions_and_music_never_take_the_register_position() {
// The bridge itself runs without queries (offline pool); which state
// reads the episode's progress register is decided on the result.
let pool = offline_pool();
let mut handed_off = song_then_episode();
handed_off["index"] = serde_json::json!(1);
handed_off["position_seconds"] = serde_json::json!(1200.0);
let wire = playback_state_from_browser_json(&pool, 1, handed_off)
.await
.unwrap();
let wire = browser_json_to_wire(&pool, handed_off).await.unwrap();
assert_eq!(wire.position_secs, 1200.0);
assert_eq!(episode_needing_resume_position(false, &wire), None);
assert_eq!(
episode_needing_resume_position(true, &wire),
Some(test_episode().episode_id),
"an episode the browser did not know the position of"
);
let mut music = song_then_episode();
music[queue_items::POSITION_UNKNOWN_KEY] = serde_json::json!(true);
let wire = playback_state_from_browser_json(&pool, 1, music)
.await
.unwrap();
let music = song_then_episode();
let wire = browser_json_to_wire(&pool, music).await.unwrap();
assert_eq!(wire.position_secs, 200.0);
assert_eq!(
episode_needing_resume_position(true, &wire),
None,
"music keeps its position even when flagged"
);
}
+137 -11
View File
@@ -1,7 +1,9 @@
//! Device-sync v4 storage paths against PostgreSQL.
//!
//! Requires a migrated disposable database (start the server once against
//! it); the test writes rows for a random user id and removes them again.
//! Requires a disposable database in `FURU_DEVICE_SYNC_TEST_DATABASE_URL`;
//! it is migrated with the production migrations on first use
//! (`crate::podcasts::postgres_tests::test_pool`). The tests write rows for
//! random user ids and remove them again.
use super::*;
use crate::player::queue_items::test_episode;
@@ -48,6 +50,14 @@ async fn cleanup(pool: &sqlx::PgPool, user_id: i64) {
"furumusic__fed_sync_ops",
"furumusic__fed_sync_vector",
"furumusic__playback_state",
"furumusic__podcast_fetch_state",
"furumusic__podcast_listen_session",
"furumusic__podcast_progress",
"furumusic__podcast_share",
"furumusic__podcast_subscription",
"furumusic__podcast_episode",
"furumusic__podcast_feed_alias",
"furumusic__podcast",
] {
sqlx::query(&format!("DELETE FROM {table} WHERE user_id = $1"))
.bind(user_id)
@@ -58,11 +68,9 @@ async fn cleanup(pool: &sqlx::PgPool, user_id: i64) {
}
#[tokio::test]
#[ignore = "requires a migrated disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_podcast_relay_gating_and_peer_versions() {
let pool = sqlx::PgPool::connect(&std::env::var("FURU_DEVICE_SYNC_TEST_DATABASE_URL").unwrap())
.await
.unwrap();
let pool = crate::podcasts::postgres_tests::test_pool().await;
let user_id = 8_000_000_000 + i64::from(rand_suffix());
cleanup(&pool, user_id).await;
let new = format!("tui_new_{user_id}");
@@ -320,11 +328,9 @@ async fn postgres_podcast_relay_gating_and_peer_versions() {
}
#[tokio::test]
#[ignore = "requires a migrated disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_unknown_episode_position_and_rollback_cleanup() {
let pool = sqlx::PgPool::connect(&std::env::var("FURU_DEVICE_SYNC_TEST_DATABASE_URL").unwrap())
.await
.unwrap();
let pool = crate::podcasts::postgres_tests::test_pool().await;
let user_id = 8_000_000_000 + i64::from(rand_suffix());
cleanup(&pool, user_id).await;
let tui = format!("tui_{user_id}");
@@ -406,6 +412,40 @@ async fn postgres_unknown_episode_position_and_rollback_cleanup() {
Some(9)
);
// A known position (the web player was playing the episode) is sent
// as is, but the stale handoff hint of the browser item (7) is raised
// to the register generation this account has seen (9).
let browser = serde_json::json!({
"tracks": [song.clone(), item.clone()], "index": 1, "track": item.clone(),
"position_seconds": 1500.0, "paused": true,
});
record_web_active_transfer(&pool, user_id, &tui, None, browser)
.await
.unwrap();
let sent: serde_json::Value = sqlx::query_scalar(
"SELECT payload_json FROM furumusic__fed_sync_ops
WHERE user_id = $1 AND kind = 'playback_command' ORDER BY seq DESC LIMIT 1",
)
.bind(user_id)
.fetch_one(&pool)
.await
.unwrap();
let SyncOpPayload::PlaybackCommand {
command: PlaybackCommand::ActiveChanged { state, .. },
..
} = serde_json::from_value(sent).unwrap()
else {
panic!("expected ActiveChanged");
};
assert_eq!(state.position_secs, 1500.0);
assert_eq!(
state.queue[1]
.podcast_episode_ref()
.unwrap()
.progress_generation,
Some(9)
);
// Remote Next onto the episode while the TUI plays the song.
let current = serde_json::json!({
"tracks": [song, item], "index": 0, "track": song,
@@ -463,12 +503,98 @@ async fn postgres_unknown_episode_position_and_rollback_cleanup() {
super::wire_compat_tests::legacy::SyncOpPayload::PlaybackCommand { .. }
))
.count(),
2
3
);
cleanup(&pool, user_id).await;
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_local_podcast_actions_reach_v4_peers_only() {
let pool = crate::podcasts::postgres_tests::test_pool().await;
let user_id = 8_000_000_000 + i64::from(rand_suffix());
cleanup(&pool, user_id).await;
let identity = ensure_identity(&pool, user_id, "tester").await.unwrap();
let old = format!("tui_old_{user_id}");
let new = format!("tui_new_{user_id}");
for (device, version) in [(&old, 3), (&new, 4)] {
apply_device_profile(
&pool,
user_id,
&profile(device, version),
VersionReport::SelfReported(Some(version)),
true,
)
.await
.unwrap();
}
// A local action lands in the log with this device's stamp and in the
// store, in one transaction.
let store = crate::podcasts::PodcastStore::new(pool.clone());
let episode = test_episode();
let meta: PodcastMeta = serde_json::from_value(serde_json::json!({
"feed_url": episode.feed_url.as_str(), "title": episode.podcast_title,
}))
.unwrap();
let subscribed = store.subscribe(user_id, &meta).await.unwrap();
let played = store.mark_played(user_id, &episode).await.unwrap();
assert!(
subscribed
.ops
.iter()
.chain([&played.op])
.all(|op| op.origin_device_id == identity.device_id)
);
// A music operation recorded afterwards keeps the same clock and sequence.
record_local_op(
&pool,
user_id,
SyncOpPayload::TrackLikeSet {
content_id: format!("b3:{}", "5".repeat(64)),
liked: false,
fed: None,
},
)
.await
.unwrap();
let own: Vec<SyncOpWire> = ops_for_peer(&pool, user_id, &new, 4)
.await
.unwrap()
.into_iter()
.filter(|op| op.origin_device_id == identity.device_id)
.collect();
let kinds: Vec<_> = own.iter().map(|op| payload_kind(&op.payload)).collect();
assert!(kinds.ends_with(&[
"podcast_subscription_set",
"podcast_progress_set",
"track_like_set"
]));
assert!(
own.windows(2)
.all(|pair| pair[1].seq > pair[0].seq && pair[1].hlc_ms > pair[0].hlc_ms)
);
assert_eq!(own[own.len() - 3], subscribed.ops[0]);
assert_eq!(own[own.len() - 2], played.op);
// The v3 peer gets the music operation only.
let to_old = ops_for_peer(&pool, user_id, &old, 3).await.unwrap();
assert!(to_old.iter().all(|op| !op.payload.is_podcast()));
assert!(
to_old
.iter()
.any(|op| payload_kind(&op.payload) == "track_like_set")
);
// The snapshot carries the store's registers for v4 only.
let full = snapshot(&pool, user_id).await.unwrap();
assert_eq!(full.podcast_subscriptions.len(), 1);
assert_eq!(full.podcast_progress.len(), 1);
assert_eq!(full.podcast_progress[0].status, ProgressStatus::Completed);
assert!(!full.for_peer(3).has_podcast_state());
cleanup(&pool, user_id).await;
}
fn rand_suffix() -> u32 {
u32::from_le_bytes(
blake3::hash(random_hex(8).as_bytes()).as_bytes()[..4]
+74 -15
View File
@@ -9,6 +9,11 @@
//! for catalog metadata and, when enabled, compatible similarity embeddings;
//! local playback and the local library remain independent of the network.
//!
//! The node also runs the public podcast-discovery overlay
//! ([`podcast_discovery`], `music_dht::podcast_dht`): it publishes cards of
//! the podcasts that accounts explicitly shared (`crate::podcasts::discovery`)
//! under the server's node identity.
//!
//! Settings are the regular admin config entries (`federation_enabled`,
//! `federation_network_id`, `federation_save_on_listen`) and apply on the fly —
//! saving the settings starts, stops or re-joins the node without a server
@@ -17,7 +22,7 @@
mod capabilities;
pub mod client;
pub mod devices;
mod podcast_relay;
pub(crate) mod podcast_discovery;
mod receive;
mod serve;
mod similarity;
@@ -31,6 +36,7 @@ use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use music_dht::capabilities::CAPABILITIES_ALPN;
use music_dht::podcast_dht::{PODCAST_DHT_ALPN, PodcastDht};
use music_dht::similarity_dht::SimilarityDht;
use music_dht::similarity_lsh::SIMILARITY_DHT_ALPN;
use music_dht::{
@@ -61,6 +67,8 @@ const TRANSPORT_SAMPLE_LIMIT: usize = 16;
struct Running {
service: Arc<MusicDhtService>,
similarity_dht: Arc<SimilarityDht>,
/// Public podcast discovery; `None` when the overlay failed to start.
podcast_dht: Option<Arc<PodcastDht>>,
network_name: String,
tasks: Vec<tokio::task::JoinHandle<()>>,
}
@@ -291,6 +299,11 @@ pub struct Federation {
download_locks: std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
pool: tokio::sync::OnceCell<PgPool>,
running: tokio::sync::Mutex<Option<Running>>,
/// The running podcast-discovery overlay, readable without waiting for
/// `running` (held while the node starts).
podcast_dht: std::sync::Mutex<Option<Arc<PodcastDht>>>,
/// The node runs, but the podcast-discovery overlay failed to start.
podcast_discovery_failed: AtomicBool,
sync_lock: tokio::sync::Mutex<()>,
supervisor_started: AtomicBool,
recovery_count: AtomicU64,
@@ -326,6 +339,8 @@ pub fn handle() -> Arc<Federation> {
download_locks: std::sync::Mutex::new(Default::default()),
pool: tokio::sync::OnceCell::new(),
running: tokio::sync::Mutex::new(None),
podcast_dht: std::sync::Mutex::new(None),
podcast_discovery_failed: AtomicBool::new(false),
sync_lock: tokio::sync::Mutex::new(()),
supervisor_started: AtomicBool::new(false),
recovery_count: AtomicU64::new(0),
@@ -475,6 +490,9 @@ impl Federation {
} else if recovery_only {
return Ok(false);
}
*lock(&self.podcast_dht) = None;
self.podcast_discovery_failed
.store(false, Ordering::Relaxed);
stop_running(guard.take()).await;
} else if recovery_only {
tracing::warn!(network = %network_name, "retrying stopped federation service");
@@ -499,6 +517,7 @@ impl Federation {
.schema_independent_stream_protocol(SIMILARITY_ALPN)
.schema_independent_stream_protocol(SIMILARITY_DHT_ALPN)
.schema_independent_stream_protocol(CAPABILITIES_ALPN)
.schema_independent_stream_protocol(PODCAST_DHT_ALPN)
.build()
.map_err(|err| anyhow::anyhow!("invalid federation config: {err}"))?;
let (service, mut events) =
@@ -531,6 +550,17 @@ impl Federation {
Arc::clone(&similarity_manager),
));
// Public podcast discovery: the server node publishes the podcasts
// accounts explicitly shared (see `crate::podcasts::discovery`),
// after the first device-sync round of this run.
let (first_device_sync_done, first_device_sync) = tokio::sync::watch::channel(false);
let podcast_discovery =
podcast_discovery::start(&service, &self.data_dir, pool.clone(), first_device_sync)
.await;
let podcast_dht = podcast_discovery
.as_ref()
.map(|discovery| Arc::clone(&discovery.dht));
// Drain DHT events into the log; the channel is bounded.
let event_task = tokio::spawn(async move {
while let Some(event) = events.recv().await {
@@ -585,11 +615,15 @@ impl Federation {
Arc::clone(&service),
device_hub,
Arc::clone(&self.transport_stats),
first_device_sync_done,
));
let capabilities_acceptor = service
.stream_acceptor(CAPABILITIES_ALPN)
.map_err(|err| anyhow::anyhow!("failed to take the capabilities acceptor: {err}"))?;
let capabilities_task = tokio::spawn(capabilities::serve(capabilities_acceptor));
let capabilities_task = tokio::spawn(capabilities::serve(
capabilities_acceptor,
podcast_dht.is_some(),
));
let similarity_acceptor = service
.stream_acceptor(SIMILARITY_ALPN)
.map_err(|err| anyhow::anyhow!("failed to take the similarity acceptor: {err}"))?;
@@ -600,23 +634,31 @@ impl Federation {
Arc::clone(&self.transport_stats),
));
let mut tasks = vec![
event_task,
sync_task,
audio_task,
catalog_task,
device_task,
device_sync_task,
capabilities_task,
similarity_task,
similarity_dht_serve_task,
similarity_dht_maintenance_task,
similarity_dht_sync_task,
];
if let Some(discovery) = podcast_discovery {
tasks.extend(discovery.tasks);
}
*lock(&self.podcast_dht) = podcast_dht.clone();
self.podcast_discovery_failed
.store(podcast_dht.is_none(), Ordering::Relaxed);
*guard = Some(Running {
service,
similarity_dht,
podcast_dht,
network_name,
tasks: vec![
event_task,
sync_task,
audio_task,
catalog_task,
device_task,
device_sync_task,
capabilities_task,
similarity_task,
similarity_dht_serve_task,
similarity_dht_maintenance_task,
similarity_dht_sync_task,
],
tasks,
});
self.set_error(None);
drop(guard);
@@ -660,6 +702,9 @@ impl Federation {
async fn stop(&self) {
let mut guard = self.running.lock().await;
*lock(&self.podcast_dht) = None;
self.podcast_discovery_failed
.store(false, Ordering::Relaxed);
stop_running(guard.take()).await;
}
@@ -686,6 +731,18 @@ impl Federation {
.context("federation is not running")
}
/// The public podcast-discovery overlay, while the node runs it. Does
/// not wait for a node that is starting or stopping.
pub fn podcast_discovery(&self) -> Option<Arc<PodcastDht>> {
lock(&self.podcast_dht).clone()
}
/// Whether the node runs but its podcast-discovery overlay could not
/// start (`podcast_discovery()` is then `None` too).
pub fn podcast_discovery_unavailable(&self) -> bool {
self.podcast_discovery_failed.load(Ordering::Relaxed)
}
async fn spawn_sync_soon(self: &Arc<Self>) {
if let Ok(service) = self.service().await {
let fed = Arc::clone(self);
@@ -1019,6 +1076,8 @@ impl Federation {
"last_rendezvous_error": health.last_rendezvous_error,
"recovery_count": self.recovery_count.load(Ordering::Relaxed),
"similarity_routing_peers": running.similarity_dht.known_peers(),
"podcast_discovery": running.podcast_dht.is_some(),
"podcast_discovery_peers": running.podcast_dht.as_ref().map(|dht| dht.known_peers()),
"published_items": published,
"transport": transport,
})
+88
View File
@@ -0,0 +1,88 @@
//! Runs the public podcast-discovery overlay (`music_dht::podcast_dht`) on
//! the web server's federation node.
//!
//! The overlay shares the node's iroh endpoint and identity with the music
//! DHT, like `similarity_dht`: its ALPN is registered as a
//! schema-independent stream protocol, its routing contacts and replica
//! cache live in `podcast-dht.sqlite3` in the federation data directory (a
//! replaceable cache, not durable state), and the capability manifest
//! advertises `podcast_dht` only while it runs. What the node publishes is
//! decided by [`crate::podcasts::discovery`] for all accounts.
//!
//! Failing to open the overlay disables public discovery and nothing else:
//! the federation node, the music DHT and private podcast sync keep running.
use std::path::Path;
use std::sync::Arc;
use music_dht::MusicDhtService;
use music_dht::podcast_dht::{PODCAST_DHT_ALPN, PodcastDht};
use sqlx::PgPool;
use tokio::sync::watch;
use tokio::task::JoinHandle;
/// Cache file of the overlay in the federation data directory.
pub(crate) const CACHE_FILE: &str = "podcast-dht.sqlite3";
/// The running overlay and its tasks (serving, maintenance, publication
/// reconciler, capability probes).
pub(crate) struct PodcastDiscovery {
pub(crate) dht: Arc<PodcastDht>,
pub(crate) tasks: Vec<JoinHandle<()>>,
}
/// Opens the overlay on `service` (whose config registered
/// [`PODCAST_DHT_ALPN`]) and starts its tasks. Returns `None`, with a
/// warning, when the cache cannot be opened or the acceptor is unavailable.
///
/// `first_device_sync` turns `true` after the first device-sync round of
/// this federation run; the publication reconciler waits for it.
pub(crate) async fn start(
service: &Arc<MusicDhtService>,
data_dir: &Path,
pool: PgPool,
first_device_sync: watch::Receiver<bool>,
) -> Option<PodcastDiscovery> {
let (dht, serve_task) = open(service, data_dir).await?;
let tasks = vec![
serve_task,
tokio::spawn(Arc::clone(&dht).maintenance()),
tokio::spawn(crate::podcasts::discovery::run(
pool,
Arc::clone(&dht),
first_device_sync,
)),
tokio::spawn(super::capabilities::probe_loop(
Arc::clone(service),
Arc::clone(&dht),
)),
];
tracing::info!("public podcast discovery started");
Some(PodcastDiscovery { dht, tasks })
}
/// Opens the overlay cache and serves overlay requests; the rest of
/// [`start`] (maintenance, reconciler, probes) is not started.
pub(crate) async fn open(
service: &Arc<MusicDhtService>,
data_dir: &Path,
) -> Option<(Arc<PodcastDht>, JoinHandle<()>)> {
let dht = match PodcastDht::open(Arc::clone(service), data_dir.join(CACHE_FILE)).await {
Ok(dht) => dht,
Err(error) => {
tracing::warn!("public podcast discovery disabled: cannot open its cache: {error}");
// Refuse overlay streams instead of queueing them.
drop(service.stream_acceptor(PODCAST_DHT_ALPN));
return None;
}
};
let acceptor = match service.stream_acceptor(PODCAST_DHT_ALPN) {
Ok(acceptor) => acceptor,
Err(error) => {
tracing::warn!("public podcast discovery disabled: {error}");
return None;
}
};
let serve_task = tokio::spawn(Arc::clone(&dht).serve(acceptor));
Some((dht, serve_task))
}
-575
View File
@@ -1,575 +0,0 @@
//! Lossless relay of podcast sync state (device-sync protocol v4).
//!
//! The web server does not implement podcasts yet, but it announces protocol
//! v4 and therefore must not lose podcast state that trusted devices send
//! through it:
//!
//! * podcast operations are stored verbatim in the generic operation log
//! (`furumusic__fed_sync_ops`) and forwarded to v4 peers only;
//! * the podcast registers carried by those operations and by snapshot
//! sections are kept here as opaque JSON documents, merged with the shared
//! `music_dht::podcast` rules, and returned in this device's snapshots.
//! Snapshots are the only recovery path for a peer that skipped podcast
//! operations while it ran an older version.
//!
//! The rows hold the shared contract types (`SubscriptionRecord`,
//! `ShareRecord`, `ProgressRecord`, `ListenSession`) as JSON and nothing
//! web-specific; a later podcast store can import them.
use anyhow::Result;
use music_dht::device_sync::{SyncOpWire, SyncSnapshot};
use music_dht::podcast::{
FeedUrl, ListenSession, PodcastId, ProgressRecord, ShareRecord, SubscriptionRecord,
merge_feed_aliases, merge_listen_session, merge_progress, merge_share, merge_subscription,
};
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::Value;
use sqlx::Row;
const SECTION_SUBSCRIPTION: &str = "subscription";
const SECTION_SHARE: &str = "share";
const SECTION_ALIASES: &str = "aliases";
const SECTION_PROGRESS: &str = "progress";
const SECTION_SESSION: &str = "session";
/// One podcast register update, keyed by its entity.
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum RelayRecord {
/// Subscription register of a podcast.
Subscription(SubscriptionRecord),
/// Share register of a podcast.
Share(ShareRecord),
/// Feed aliases learned before (or without) the subscription register.
Aliases(PodcastId, Vec<FeedUrl>),
/// Progress register of an episode.
Progress(ProgressRecord),
/// Listening session.
Session(ListenSession),
}
impl RelayRecord {
fn section(&self) -> &'static str {
match self {
RelayRecord::Subscription(_) => SECTION_SUBSCRIPTION,
RelayRecord::Share(_) => SECTION_SHARE,
RelayRecord::Aliases(..) => SECTION_ALIASES,
RelayRecord::Progress(_) => SECTION_PROGRESS,
RelayRecord::Session(_) => SECTION_SESSION,
}
}
fn key(&self) -> &str {
match self {
RelayRecord::Subscription(record) => record.podcast_id.as_str(),
RelayRecord::Share(record) => record.podcast_id.as_str(),
RelayRecord::Aliases(podcast_id, _) => podcast_id.as_str(),
RelayRecord::Progress(record) => record.episode_id.as_str(),
RelayRecord::Session(session) => session.session_id.as_str(),
}
}
/// Register updates carried by a podcast operation.
///
/// Operations that fail `validate_podcast` must be dropped before this.
pub(crate) fn from_op(op: &SyncOpWire) -> Option<Self> {
if let Some(record) = op.podcast_subscription() {
return Some(RelayRecord::Subscription(record));
}
if let Some(record) = op.podcast_share() {
return Some(RelayRecord::Share(record));
}
if let Some((podcast_id, aliases)) = op.podcast_feed_aliases() {
return Some(RelayRecord::Aliases(podcast_id.clone(), aliases.to_vec()));
}
if let Some(record) = op.podcast_progress() {
return Some(RelayRecord::Progress(record));
}
op.podcast_listen_session()
.map(|session| RelayRecord::Session(session.clone()))
}
/// Valid register records of a snapshot. Podcast sections that exceed
/// the contract counts are ignored as a whole; invalid records are
/// skipped one by one.
pub(crate) fn from_snapshot(snapshot: &SyncSnapshot) -> Vec<Self> {
if !snapshot.podcast_state_within_limits() {
tracing::debug!("ignored podcast snapshot sections above the contract limits");
return Vec::new();
}
let subscriptions = snapshot
.podcast_subscriptions
.iter()
.filter(|record| record.validate().is_ok())
.cloned()
.map(RelayRecord::Subscription);
let shares = snapshot
.podcast_shares
.iter()
.filter(|record| record.validate().is_ok())
.cloned()
.map(RelayRecord::Share);
let progress = snapshot
.podcast_progress
.iter()
.filter(|record| record.validate().is_ok())
.cloned()
.map(RelayRecord::Progress);
let sessions = snapshot
.podcast_listen_sessions
.iter()
.filter(|session| session.validate().is_ok())
.cloned()
.map(RelayRecord::Session);
subscriptions
.chain(shares)
.chain(progress)
.chain(sessions)
.collect()
}
}
/// Document address: `(section, record_key)`.
type DocKey = (&'static str, String);
/// Documents a record's merge reads.
fn read_keys(record: &RelayRecord) -> Vec<DocKey> {
let own = (record.section(), record.key().to_string());
match record {
RelayRecord::Aliases(podcast_id, _) => {
vec![(SECTION_SUBSCRIPTION, podcast_id.as_str().to_string()), own]
}
RelayRecord::Subscription(_) => {
vec![(SECTION_ALIASES, record.key().to_string()), own]
}
_ => vec![own],
}
}
/// Merges `incoming` into the stored documents with the shared rules.
///
/// Returns the writes: `Some(value)` upserts, `None` deletes. Aliases
/// learned before the subscription register are kept in their own document
/// and folded into the register when it arrives.
fn plan_merge(
docs: &std::collections::BTreeMap<DocKey, Value>,
incoming: &RelayRecord,
) -> Vec<(DocKey, Option<Value>)> {
let own = (incoming.section(), incoming.key().to_string());
let current = docs.get(&own);
let changed = |merged: Option<Value>| {
merged
.filter(|merged| current != Some(merged))
.map(|merged| (own.clone(), Some(merged)))
};
match incoming {
RelayRecord::Subscription(record) => {
let alias_key = (SECTION_ALIASES, incoming.key().to_string());
let pending = docs.get(&alias_key).and_then(parse::<Vec<FeedUrl>>);
let merged = match current.and_then(parse::<SubscriptionRecord>) {
Some(current) => merge_subscription(&current, record).ok(),
None => {
let mut record = record.clone();
record.normalize();
Some(record)
}
}
.map(|mut record| {
if let Some(aliases) = &pending {
record.add_feed_aliases(aliases);
}
record
});
let mut writes: Vec<_> = changed(merged.as_ref().and_then(to_value))
.into_iter()
.collect();
if docs.contains_key(&alias_key) && merged.is_some() {
writes.push((alias_key, None));
}
writes
}
RelayRecord::Aliases(podcast_id, aliases) => {
let sub_key = (SECTION_SUBSCRIPTION, podcast_id.as_str().to_string());
if let Some(subscription) = docs.get(&sub_key) {
let Some(mut record) = parse::<SubscriptionRecord>(subscription) else {
return Vec::new();
};
record.add_feed_aliases(aliases);
return to_value(&record)
.filter(|merged| merged != subscription)
.map(|merged| vec![(sub_key, Some(merged))])
.unwrap_or_default();
}
let current = current.and_then(parse::<Vec<FeedUrl>>).unwrap_or_default();
changed(to_value(&merge_feed_aliases(podcast_id, &current, aliases)))
.into_iter()
.collect()
}
RelayRecord::Share(record) => changed(match current.and_then(parse::<ShareRecord>) {
Some(current) => merge_share(&current, record)
.ok()
.as_ref()
.and_then(to_value),
None => to_value(record),
})
.into_iter()
.collect(),
RelayRecord::Progress(record) => changed(match current.and_then(parse::<ProgressRecord>) {
Some(current) => merge_progress(&current, record)
.ok()
.as_ref()
.and_then(to_value),
None => to_value(record),
})
.into_iter()
.collect(),
RelayRecord::Session(session) => changed(match current.and_then(parse::<ListenSession>) {
Some(current) => merge_listen_session(&current, session)
.ok()
.as_ref()
.and_then(to_value),
None => to_value(session),
})
.into_iter()
.collect(),
}
}
fn parse<T: DeserializeOwned>(value: &Value) -> Option<T> {
serde_json::from_value(value.clone()).ok()
}
fn to_value<T: Serialize>(value: &T) -> Option<Value> {
serde_json::to_value(value).ok()
}
/// Merges register updates into the user's relay state.
pub(crate) async fn merge_records(
pool: &sqlx::PgPool,
user_id: i64,
records: &[RelayRecord],
) -> Result<()> {
if records.is_empty() {
return Ok(());
}
let mut tx = pool.begin().await?;
// One writer per account: a merge reads, combines and writes documents.
sqlx::query(
"SELECT pg_advisory_xact_lock(hashtextextended('furumusic:podcast-relay:' || $1::text, 0))",
)
.bind(user_id)
.execute(&mut *tx)
.await?;
let mut docs = std::collections::BTreeMap::new();
for record in records {
for key in read_keys(record) {
if !docs.contains_key(&key)
&& let Some(value) = load(&mut tx, user_id, key.0, &key.1).await?
{
docs.insert(key, value);
}
}
for (key, value) in plan_merge(&docs, record) {
match &value {
Some(value) => store(&mut tx, user_id, key.0, &key.1, value).await?,
None => {
sqlx::query(
"DELETE FROM furumusic__fed_podcast_relay
WHERE user_id = $1 AND section = $2 AND record_key = $3",
)
.bind(user_id)
.bind(key.0)
.bind(&key.1)
.execute(&mut *tx)
.await?;
}
}
match value {
Some(value) => docs.insert(key, value),
None => docs.remove(&key),
};
}
}
tx.commit().await?;
Ok(())
}
async fn load(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
user_id: i64,
section: &str,
key: &str,
) -> Result<Option<Value>> {
Ok(sqlx::query_scalar(
"SELECT record_json FROM furumusic__fed_podcast_relay
WHERE user_id = $1 AND section = $2 AND record_key = $3",
)
.bind(user_id)
.bind(section)
.bind(key)
.fetch_optional(&mut **tx)
.await?)
}
async fn store(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
user_id: i64,
section: &str,
key: &str,
value: &Value,
) -> Result<()> {
sqlx::query(
"INSERT INTO furumusic__fed_podcast_relay
(user_id, section, record_key, record_json, updated_at_ms)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (user_id, section, record_key) DO UPDATE SET
record_json = EXCLUDED.record_json,
updated_at_ms = EXCLUDED.updated_at_ms",
)
.bind(user_id)
.bind(section)
.bind(key)
.bind(value)
.bind(chrono::Utc::now().timestamp_millis())
.execute(&mut **tx)
.await?;
Ok(())
}
/// Relayed progress register of an episode, if any.
pub(crate) async fn progress_record(
pool: &sqlx::PgPool,
user_id: i64,
episode_id: &str,
) -> Result<Option<ProgressRecord>> {
let value: Option<Value> = sqlx::query_scalar(
"SELECT record_json FROM furumusic__fed_podcast_relay
WHERE user_id = $1 AND section = $2 AND record_key = $3",
)
.bind(user_id)
.bind(SECTION_PROGRESS)
.bind(episode_id)
.fetch_optional(pool)
.await?;
Ok(value.as_ref().and_then(parse))
}
/// Adds the relayed podcast registers to `snapshot`.
///
/// The sections are bounded later, per message, with the contract budget
/// (`SyncSnapshot::bound_podcast_state_to`) and stripped for peers below v4
/// (`SyncSnapshot::for_peer`).
pub(crate) async fn fill_snapshot(
pool: &sqlx::PgPool,
user_id: i64,
snapshot: &mut SyncSnapshot,
) -> Result<()> {
let rows = sqlx::query(
"SELECT section, record_json FROM furumusic__fed_podcast_relay
WHERE user_id = $1 AND section <> $2
ORDER BY section, record_key",
)
.bind(user_id)
.bind(SECTION_ALIASES)
.fetch_all(pool)
.await?;
for row in rows {
let section: String = row.get("section");
let value: Value = row.get("record_json");
fill_section(snapshot, &section, &value);
}
Ok(())
}
fn fill_section(snapshot: &mut SyncSnapshot, section: &str, value: &Value) {
match section {
SECTION_SUBSCRIPTION => snapshot.podcast_subscriptions.extend(parse(value)),
SECTION_SHARE => snapshot.podcast_shares.extend(parse(value)),
SECTION_PROGRESS => snapshot.podcast_progress.extend(parse(value)),
SECTION_SESSION => snapshot.podcast_listen_sessions.extend(parse(value)),
_ => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
use music_dht::podcast::{EpisodeId, PodcastMeta, ProgressStatus};
use std::collections::BTreeMap;
fn feed(path: &str) -> FeedUrl {
FeedUrl::parse(&format!("https://podcasts.example/{path}")).unwrap()
}
fn meta() -> PodcastMeta {
serde_json::from_value(serde_json::json!({
"feed_url": feed("feed.xml").as_str(),
"title": "Example Cast",
"description": "Long channel description",
}))
.unwrap()
}
fn podcast() -> PodcastId {
PodcastId::from_feed_url(&feed("feed.xml"))
}
fn op(seq: i64, hlc_ms: i64, payload: SyncOpPayload) -> SyncOpWire {
SyncOpWire {
op_id: format!("tui_a:{seq}"),
origin_device_id: "tui_a".into(),
seq,
hlc_ms,
payload,
}
}
fn progress(seq: i64, generation: u64, hlc_ms: i64, position_ms: i64) -> SyncOpWire {
let podcast_id = podcast();
op(
seq,
hlc_ms,
SyncOpPayload::PodcastProgressSet {
episode_id: EpisodeId::from_guid(&podcast_id, "ep-1").unwrap(),
podcast_id,
position_ms,
status: ProgressStatus::InProgress,
generation,
session_id: format!("s{generation}"),
episode: None,
},
)
}
/// Applies records to an in-memory table with the production plan.
fn apply(state: &mut BTreeMap<DocKey, Value>, records: &[RelayRecord]) {
for record in records {
for (key, value) in plan_merge(state, record) {
match value {
Some(value) => state.insert(key, value),
None => state.remove(&key),
};
}
}
}
fn snapshot_of(state: &BTreeMap<DocKey, Value>) -> SyncSnapshot {
let mut snapshot = SyncSnapshot::default();
for ((section, _), value) in state {
fill_section(&mut snapshot, section, value);
}
snapshot
}
#[test]
fn progress_registers_follow_the_contract_in_any_order() {
// A late checkpoint of an older generation never wins, whatever its clock.
let ops = [
progress(1, 3, 9_000, 600_000),
progress(2, 4, 1_000, 10_000),
];
for order in [[0, 1], [1, 0]] {
let mut state = BTreeMap::new();
for index in order {
apply(&mut state, &[RelayRecord::from_op(&ops[index]).unwrap()]);
}
let snapshot = snapshot_of(&state);
assert_eq!(snapshot.podcast_progress.len(), 1);
assert_eq!(snapshot.podcast_progress[0].position_ms, 10_000);
assert_eq!(snapshot.podcast_progress[0].generation, 4);
}
}
#[test]
fn snapshot_sections_survive_the_relay() {
let subscribe = op(
1,
10,
SyncOpPayload::PodcastSubscriptionSet {
podcast_id: podcast(),
subscribed: true,
podcast: meta(),
feed_aliases: vec![],
generation: 1,
},
);
let share = op(
2,
11,
SyncOpPayload::PodcastShareSet {
podcast_id: podcast(),
shared: false,
generation: 1,
},
);
let mut source = BTreeMap::new();
for op in [&subscribe, &share, &progress(3, 2, 12, 5_000)] {
op.validate_podcast().unwrap();
apply(&mut source, &[RelayRecord::from_op(op).unwrap()]);
}
let mut sent = snapshot_of(&source);
sent.bound_podcast_state();
// A second relay that only saw the snapshot re-emits the same state.
let mut relay = BTreeMap::new();
apply(&mut relay, &RelayRecord::from_snapshot(&sent));
let mut relayed = snapshot_of(&relay);
relayed.bound_podcast_state();
assert_eq!(relayed, sent);
assert!(!relayed.podcast_shares[0].shared);
assert!(relayed.podcast_subscriptions[0].subscribed);
// Peers below v4 never receive the sections.
assert!(!relayed.for_peer(3).has_podcast_state());
assert!(relayed.for_peer(4).has_podcast_state());
}
#[test]
fn aliases_learned_before_the_subscription_are_merged_into_it() {
let aliases = op(
1,
10,
SyncOpPayload::PodcastFeedAliasesAdded {
podcast_id: podcast(),
feed_aliases: vec![feed("moved.xml")],
},
);
let subscribe = op(
2,
11,
SyncOpPayload::PodcastSubscriptionSet {
podcast_id: podcast(),
subscribed: true,
podcast: meta(),
feed_aliases: vec![],
generation: 1,
},
);
let mut state = BTreeMap::new();
apply(&mut state, &[RelayRecord::from_op(&aliases).unwrap()]);
apply(&mut state, &[RelayRecord::from_op(&subscribe).unwrap()]);
let snapshot = snapshot_of(&state);
assert_eq!(
snapshot.podcast_subscriptions[0].feed_urls(),
vec![feed("feed.xml"), feed("moved.xml")]
);
assert!(!state.keys().any(|(section, _)| *section == SECTION_ALIASES));
}
#[test]
fn oversized_snapshot_sections_are_ignored() {
let mut snapshot = SyncSnapshot::default();
let record = RelayRecord::from_op(&progress(1, 1, 1, 1)).unwrap();
let RelayRecord::Progress(record) = record else {
unreachable!()
};
snapshot.podcast_progress = vec![record; music_dht::podcast::MAX_SNAPSHOT_PROGRESS + 1];
assert!(RelayRecord::from_snapshot(&snapshot).is_empty());
snapshot
.podcast_progress
.truncate(music_dht::podcast::MAX_SNAPSHOT_PROGRESS);
assert_eq!(
RelayRecord::from_snapshot(&snapshot).len(),
music_dht::podcast::MAX_SNAPSHOT_PROGRESS
);
}
}
+94 -1
View File
@@ -382,8 +382,101 @@ translations! {
player_clear: "Clear" , "Очистить";
player_remove: "Remove" , "Удалить";
player_queue_empty: "Queue is empty" , "Очередь пуста";
player_queue_podcast_unplayable: "Podcast · not playable in the web player yet", "Подкаст · пока не воспроизводится в веб-плеере";
player_queue_podcast_item: "Podcast episode" , "Выпуск подкаста";
player_queue_private_item: "Private item · hidden by its owner", "Приватный элемент · скрыт владельцем";
player_podcasts: "Podcasts", "Подкасты";
player_podcast_add_placeholder: "RSS feed URL", "Адрес RSS-ленты";
player_podcast_add: "Add", "Добавить";
player_podcast_adding: "Loading the feed…", "Загружаем ленту…";
player_podcast_added: "Subscribed", "Подписка добавлена";
player_podcast_already: "Already subscribed", "Вы уже подписаны";
player_podcast_private_note: "New subscriptions are private: they sync only between your own devices.", "Новые подписки приватны: они синхронизируются только между вашими устройствами.";
player_podcasts_empty: "No podcasts yet. Add an RSS feed to subscribe.", "Подкастов пока нет. Добавьте RSS-ленту, чтобы подписаться.";
player_podcast_show_unsubscribed: "Show unsubscribed", "Показать без подписки";
player_podcast_unplayed: "unplayed", "не прослушано";
player_podcast_episodes: "episodes", "выпусков";
player_podcast_seasons: "seasons", "сезонов";
player_podcast_unsubscribed: "Not subscribed", "Нет подписки";
player_podcast_refresh: "Refresh", "Обновить";
player_podcast_refreshing: "Checking the feed…", "Проверяем ленту…";
player_podcast_refresh_updated: "Feed updated", "Лента обновлена";
player_podcast_refresh_not_modified: "No changes in the feed", "Изменений в ленте нет";
player_podcast_refresh_failed: "Could not refresh the feed", "Не удалось обновить ленту";
player_podcast_last_checked: "Checked", "Проверено";
player_podcast_never_checked: "Not checked yet", "Ещё не проверялась";
player_podcast_fetch_error: "Last check failed", "Последняя проверка не удалась";
player_podcast_next_check: "next check", "следующая проверка";
player_podcast_unsubscribe: "Unsubscribe", "Отписаться";
player_podcast_unsubscribe_confirm: "Unsubscribe from “{title}”? The feed will no longer be checked. Your progress and listening history are kept: subscribing again continues where you stopped.", "Отписаться от «{title}»? Лента больше не будет проверяться. Прогресс и история прослушиваний сохранятся: при повторной подписке вы продолжите с того же места.";
player_podcast_subscribe: "Subscribe", "Подписаться";
player_podcast_share: "Share in the federation", "Поделиться в федерации";
player_podcast_unshare: "Stop sharing", "Перестать делиться";
player_podcast_sharing: "Saving…", "Сохраняем…";
player_podcast_shared_badge: "Sharing on", "Публикация включена";
player_podcast_share_private: "Private: only your own devices see this subscription. Sharing publishes the podcast card (title, description, artwork, feed URL) from this server so that other federation members can find it and subscribe. Your progress, history and account are never published.", "Приватно: подписку видят только ваши устройства. «Поделиться» публикует карточку подкаста (название, описание, обложку, адрес RSS-ленты) от имени этого сервера, чтобы другие участники федерации могли найти его и подписаться. Ваш прогресс, история и аккаунт не публикуются.";
player_podcast_share_published: "Published by this server: other federation members can find this podcast and subscribe. Your progress, history and account stay private.", "Опубликовано этим сервером: участники федерации могут найти этот подкаст и подписаться. Ваш прогресс, история и аккаунт остаются приватными.";
player_podcast_share_publishing: "Shared: the server is publishing the card…", "Включено: сервер публикует карточку…";
player_podcast_share_discovery_off: "Shared: this server publishes the card once federation is enabled on it.", "Включено: этот сервер опубликует карточку, когда на нём включат федерацию.";
player_podcast_share_discovery_unavailable: "Shared, but this server cannot publish the card: public podcast discovery failed to start (see the federation status).", "Включено, но этот сервер не может опубликовать карточку: публичный поиск подкастов не запустился (см. статус федерации).";
player_podcast_share_over_limit: "Shared, but not published: this server already publishes as many podcasts as it can, and your part of them is used. The card is published when a place frees up.", "Включено, но не опубликовано: сервер уже публикует столько подкастов, сколько может, и ваша доля занята. Карточка появится, когда освободится место.";
player_podcast_share_failed: "Shared, but this server could not publish the card. It tries again regularly.", "Включено, но этот сервер не смог опубликовать карточку. Он регулярно пробует снова.";
player_podcast_share_pending: "Sharing is on, but the feed URL may contain private data. This server does not publish it until you confirm here; your other players where you already confirmed may publish it themselves.", "Публикация включена, но адрес ленты может содержать личные данные. Этот сервер не опубликует его, пока вы не подтвердите здесь; ваши другие плееры, где вы уже подтвердили, могут публиковать его сами.";
player_podcast_share_review: "Review and publish", "Проверить и опубликовать";
player_podcast_share_unavailable: "This podcast cannot be published", "Этот подкаст нельзя опубликовать";
player_podcast_share_dialog_title: "Publish this feed URL?", "Опубликовать адрес ленты?";
player_podcast_share_dialog_text: "The podcast card is visible to everyone in the federation, and anyone can subscribe with its feed URL. This URL may contain private data:", "Карточку подкаста увидят все участники федерации, и любой сможет подписаться по адресу ленты. Похоже, этот адрес содержит личные данные:";
player_podcast_share_dialog_hint: "If this is a personal feed (for example, a paid or member subscription), do not publish it: others could use your access.", "Если это личная лента (например, платная подписка или лента для участников), не публикуйте её: другие смогут воспользоваться вашим доступом.";
player_podcast_share_dialog_confirm: "Publish anyway", "Всё равно опубликовать";
player_podcast_warning_secret_query: "the query parameter «{name}» looks like an access key", "параметр «{name}» похож на ключ доступа";
player_podcast_warning_private_host: "{host} usually serves personal subscriber feeds", "{host} обычно отдаёт личные ленты подписчиков";
player_podcast_warning_entropy_query: "the value of the parameter «{name}» looks like a secret token", "значение параметра «{name}» похоже на секретный токен";
player_podcast_warning_entropy_path: "a part of the address looks like a secret token", "часть адреса похожа на секретный токен";
player_podcast_warning_other: "the address looks like a personal feed", "адрес похож на личную ленту";
player_podcast_all_seasons: "All seasons", "Все сезоны";
player_podcast_season: "Season", "Сезон";
player_podcast_no_season: "No season", "Без сезона";
player_podcast_filter_all: "All episodes", "Все выпуски";
player_podcast_filter_unplayed: "Unplayed", "Не прослушано";
player_podcast_filter_in_progress: "In progress", "Начато";
player_podcast_filter_played: "Played", "Прослушано";
player_podcast_sort_newest: "Newest first", "Сначала новые";
player_podcast_sort_oldest: "Oldest first", "Сначала старые";
player_podcast_sort_number: "By number", "По номеру";
player_podcast_search: "Search episodes", "Поиск по выпускам";
player_podcast_no_episodes: "No episodes match.", "Подходящих выпусков нет.";
player_podcast_play: "Play", "Слушать";
player_podcast_resume: "Resume", "Продолжить";
player_podcast_restart: "Start over", "Начать сначала";
player_podcast_mark_played: "Mark played", "Прослушано";
player_podcast_mark_unplayed: "Mark unplayed", "Не прослушано";
player_podcast_add_queue: "Add to queue", "В очередь";
player_podcast_left: "left", "осталось";
player_podcast_saved_at: "Saved at", "Сохранено на";
player_podcast_played_badge: "Played", "Прослушано";
player_podcast_played_times: "Completed listens", "Полных прослушиваний";
player_podcast_not_in_feed: "No longer in the feed", "Больше нет в ленте";
player_podcast_load_more: "Load more", "Показать ещё";
player_podcast_history: "Recently listened", "Недавно прослушано";
player_podcast_jam_blocked: "Podcast episodes are private and cannot be added to a Jam.", "Выпуски подкастов приватны и не добавляются в Jam.";
player_podcast_source_failed: "Could not load this episode from the publisher.", "Не удалось загрузить выпуск у издателя.";
player_podcast_cannot_seek: "The publisher server cannot seek to the saved position. Press Play to listen from the start; until then the saved position is kept.", "Сервер издателя не умеет перематывать к сохранённой позиции. Нажмите Play, чтобы слушать с начала; до этого сохранённая позиция не меняется.";
player_podcast_loading: "Preparing the episode…", "Готовим выпуск…";
player_podcast_feed: "Feed", "Лента";
player_podcast_request_failed: "Request failed", "Запрос не выполнен";
player_podcast_found_title: "Podcasts", "Подкасты";
player_podcast_type: "Podcast", "Подкаст";
player_podcast_found_failed: "The podcast search did not answer this time; music results are complete.", "Поиск подкастов сейчас не ответил; результаты по музыке полные.";
player_podcast_found_subscribed: "Already subscribed", "Уже подписан";
player_podcast_found_open: "Open", "Открыть";
player_podcast_found_subscribing: "Subscribing…", "Подписываемся…";
player_podcast_found_done: "Subscribed. The subscription is private: only your devices see it.", "Вы подписались. Подписка приватная: её видят только ваши устройства.";
player_podcast_found_note: "This card was published by a federation member; its title, description and artwork are their claims. Subscribing reads the episodes from the feed below, never from the member, and your subscription stays private.", "Эту карточку опубликовал участник федерации; название, описание и обложка — его данные. При подписке выпуски загружаются из ленты по адресу ниже, а не у участника, и ваша подписка остаётся приватной.";
player_podcast_found_feed: "Feed URL", "Адрес ленты";
player_podcast_found_loading: "Loading seasons and episodes from the feed…", "Загружаем сезоны и выпуски из ленты…";
player_podcast_found_retry: "Try again", "Повторить";
player_podcast_found_latest: "Latest episode", "Последний выпуск";
player_podcast_found_more: "and {count} more", "и ещё {count}";
player_podcast_found_moved: "The feed has moved; subscribing follows it.", "Лента переехала; подписка последует за ней.";
player_shuffle: "Shuffle" , "Перемешать";
player_repeat: "Repeat" , "Повтор";
player_volume: "Volume" , "Громкость";
+1
View File
@@ -6,6 +6,7 @@ pub mod lastfm_popularity;
pub mod lastfm_scrobble;
pub mod metadata_backfill;
pub mod musicbrainz;
pub mod podcast_refresh;
use std::path::{Component, Path, PathBuf};
+73
View File
@@ -0,0 +1,73 @@
//! Background refresh of subscribed podcast feeds.
//!
//! Every run takes the podcasts whose next check is due (see
//! [`crate::podcasts::runner`]): healthy feeds are checked about hourly with
//! conditional requests, failing ones back off exponentially up to a day.
//! One request per feed URL serves every account subscribed to it, a
//! bounded number of requests run concurrently, and a failing feed never
//! blocks the others or clears the stored catalog.
use crate::podcasts::{FeedFetcher, PodcastStore, now_ms, runner};
use crate::scheduler::{Job, JobContext, JobLog};
pub struct PodcastRefreshJob;
/// Session-level advisory lock: one run at a time across processes.
const ADVISORY_LOCK_ID: i64 = 0x504f_4443_5246_5348; // "PODCRFSH"
/// Failures listed individually in the run log.
const LOGGED_FAILURES: usize = 20;
#[async_trait::async_trait]
impl Job for PodcastRefreshJob {
fn name(&self) -> &'static str {
"podcast_refresh"
}
fn description(&self) -> &'static str {
"Check subscribed podcast feeds for new episodes (metadata only, no audio)"
}
fn default_cron(&self) -> &'static str {
// Every five minutes; each feed is checked only when due.
"0 */5 * * * *"
}
async fn run(&self, ctx: &JobContext, log: &mut JobLog) -> anyhow::Result<()> {
let mut conn = ctx.pool.acquire().await?;
// A session lock must never return to the pool, including on
// cancellation; the store below uses other pool connections.
conn.close_on_drop();
let locked: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock($1)")
.bind(ADVISORY_LOCK_ID)
.fetch_one(&mut *conn)
.await?;
if !locked {
log.info("Podcast refresh is already running; skipping this run");
return Ok(());
}
let store = PodcastStore::new(ctx.pool.clone());
let fetcher = FeedFetcher::new()?;
let result = runner::refresh_due(
&store,
&fetcher,
now_ms(),
runner::RUN_LIMIT,
runner::RUN_CONCURRENCY,
)
.await;
conn.close().await?;
let summary = result?;
for (podcast_id, error) in summary.failures.iter().take(LOGGED_FAILURES) {
log.warn(&format!("podcast {podcast_id}: {error}"));
}
if summary.failures.len() > LOGGED_FAILURES {
log.warn(&format!(
"... and {} more failed feeds",
summary.failures.len() - LOGGED_FAILURES
));
}
log.info(&format!("Podcast refresh: {summary}"));
Ok(())
}
}
+10
View File
@@ -15,6 +15,7 @@ mod metrics;
mod music;
mod oidc;
mod player;
mod podcasts;
mod scheduler;
mod similarity;
mod torrents;
@@ -63,6 +64,7 @@ fn build_registry() -> Arc<JobRegistry> {
registry.register(jobs::metadata_backfill::MetadataBackfillJob);
registry.register(jobs::lastfm_popularity::LastfmPopularityJob);
registry.register(jobs::lastfm_scrobble::LastfmScrobbleJob);
registry.register(jobs::podcast_refresh::PodcastRefreshJob);
Arc::new(registry)
}
@@ -475,6 +477,10 @@ impl Project for FuruProject {
" API:\n",
" FURU_SWAGGER_ENABLED Enable Swagger UI at /swagger/ (default: false)\n",
"\n",
" Podcasts:\n",
" FURU_PODCAST_AUDIO_PROXY_ENABLED Stream episode audio through this\n",
" server (default: false)\n",
"\n",
"QUICK START\n",
" export FURU_DATABASE_URL=postgres://user:pass@localhost/furumusic\n",
" furumusic run",
@@ -579,6 +585,10 @@ impl Project for FuruProject {
"/admin",
);
apps.register_with_views(api::ApiApp, "/api");
apps.register_with_views(
podcasts::http::PodcastsApp::new(Arc::clone(&self.app_config)),
"/api/podcasts",
);
apps.register_with_views(
player::PlayerApp::new(
Arc::clone(&self.app_config),
+113
View File
@@ -2942,6 +2942,116 @@ pub mod db_migrations {
.build()];
}
// -- M0056: PostgreSQL podcast store (device-sync protocol v4) ---------
// Per-account podcast catalog, synced registers and local fetch state;
// see `crate::podcasts` and `crate::podcasts::schema`.
#[cot::db::migrations::migration_op]
async fn create_podcast_store(ctx: migrations::MigrationContext<'_>) -> cot::db::Result<()> {
for sql in crate::podcasts::schema::CREATE_PODCAST_STORE {
ctx.db.raw(sql).await?;
}
Ok(())
}
#[cot::db::migrations::migration_op]
async fn drop_podcast_store(ctx: migrations::MigrationContext<'_>) -> cot::db::Result<()> {
for sql in crate::podcasts::schema::DROP_PODCAST_STORE {
ctx.db.raw(sql).await?;
}
Ok(())
}
#[derive(Debug, Copy, Clone)]
pub struct M0056CreatePodcastStore;
impl migrations::Migration for M0056CreatePodcastStore {
const APP_NAME: &'static str = "furumusic";
const MIGRATION_NAME: &'static str = "m_0056_create_podcast_store";
const DEPENDENCIES: &'static [migrations::MigrationDependency] =
&[migrations::MigrationDependency::migration(
"furumusic",
"m_0055_create_fed_podcast_relay",
)];
const OPERATIONS: &'static [Operation] = &[Operation::custom(create_podcast_store)
.backwards(drop_podcast_store)
.build()];
}
// -- M0057: import the 0.13.0 podcast relay state into the store --------
// The relay table stays (unused) until a later cleanup release, so a
// rollback to 0.13.0 still finds the state it relayed.
#[cot::db::migrations::migration_op]
async fn import_podcast_relay(ctx: migrations::MigrationContext<'_>) -> cot::db::Result<()> {
for sql in crate::podcasts::schema::import_statements("TRUE") {
ctx.db.raw(&sql).await?;
}
Ok(())
}
#[cot::db::migrations::migration_op]
async fn keep_podcast_store_rows(
_ctx: migrations::MigrationContext<'_>,
) -> cot::db::Result<()> {
// The imported rows go with the tables (`m_0056` backwards).
Ok(())
}
#[derive(Debug, Copy, Clone)]
pub struct M0057ImportPodcastRelay;
impl migrations::Migration for M0057ImportPodcastRelay {
const APP_NAME: &'static str = "furumusic";
const MIGRATION_NAME: &'static str = "m_0057_import_podcast_relay";
const DEPENDENCIES: &'static [migrations::MigrationDependency] =
&[migrations::MigrationDependency::migration(
"furumusic",
"m_0056_create_podcast_store",
)];
const OPERATIONS: &'static [Operation] = &[Operation::custom(import_podcast_relay)
.backwards(keep_podcast_store_rows)
.build()];
}
// -- M0058: local privacy confirmations of shared podcasts -------------
// Client-held `privacy_confirmed` of public podcast discovery (MR-W3);
// see `crate::podcasts::discovery`. Never synced.
#[cot::db::migrations::migration_op]
async fn create_podcast_share_confirmation(
ctx: migrations::MigrationContext<'_>,
) -> cot::db::Result<()> {
for sql in crate::podcasts::schema::CREATE_SHARE_CONFIRMATION {
ctx.db.raw(sql).await?;
}
Ok(())
}
#[cot::db::migrations::migration_op]
async fn drop_podcast_share_confirmation(
ctx: migrations::MigrationContext<'_>,
) -> cot::db::Result<()> {
for sql in crate::podcasts::schema::DROP_SHARE_CONFIRMATION {
ctx.db.raw(sql).await?;
}
Ok(())
}
#[derive(Debug, Copy, Clone)]
pub struct M0058CreatePodcastShareConfirmation;
impl migrations::Migration for M0058CreatePodcastShareConfirmation {
const APP_NAME: &'static str = "furumusic";
const MIGRATION_NAME: &'static str = "m_0058_create_podcast_share_confirmation";
const DEPENDENCIES: &'static [migrations::MigrationDependency] =
&[migrations::MigrationDependency::migration(
"furumusic",
"m_0057_import_podcast_relay",
)];
const OPERATIONS: &'static [Operation] =
&[Operation::custom(create_podcast_share_confirmation)
.backwards(drop_podcast_share_confirmation)
.build()];
}
pub const MIGRATIONS: &[&SyncDynMigration] = &[
&M0006CreateMediaFile,
&M0007CreateArtist,
@@ -2987,5 +3097,8 @@ pub mod db_migrations {
&M0052AddImportPlaylistDestinations,
&M0054AddTypedPlaybackQueue,
&M0055CreateFedPodcastRelay,
&M0056CreatePodcastStore,
&M0057ImportPodcastRelay,
&M0058CreatePodcastShareConfirmation,
];
}
+32 -8
View File
@@ -7472,7 +7472,7 @@ async fn federation_search_events_handler(
db: Database,
query: cot::request::extractors::UrlQuery<SearchQuery>,
) -> cot::Result<cot::response::Response> {
let Some(_user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
let q = query.0.q.trim().to_owned();
@@ -7482,6 +7482,7 @@ async fn federation_search_events_handler(
let search_id = uuid::Uuid::new_v4().to_string();
let body_search_id = search_id.clone();
let user_id = user.id;
let body = Body::streaming(async_stream::try_stream! {
let started = serde_json::json!({
"search_id": body_search_id,
@@ -7492,25 +7493,40 @@ async fn federation_search_events_handler(
});
yield Bytes::from(format!("event: search.started\ndata: {started}\n\n"));
match crate::federation::handle().search_events(&body_search_id, &q).await {
Ok(events) => {
let federation = crate::federation::handle();
// Public podcasts are searched next to the music DHT; their events
// follow the music events.
let podcasts = federation.spawn_podcast_search(&q);
match federation.search_events(&body_search_id, &q).await {
Ok(music) => {
let mut last_sequence = 0;
for event in events {
for event in &music.events {
last_sequence = event.sequence;
let data = serde_json::to_string(&event)
.map_err(|err| cot::Error::internal(err.to_string()))?;
yield Bytes::from(format!("event: {}\ndata: {data}\n\n", event.kind));
yield search_event_frame(event)?;
}
let podcasts = podcasts
.await
.unwrap_or(crate::federation::client::PodcastSearch::Failed);
// A podcast search or ranking failure only drops the
// podcast hits: `search.completed` reports it.
let (podcast_discovery, podcast_events) = federation
.podcast_search_events(&body_search_id, &q, user_id, &music, podcasts)
.await;
for event in &podcast_events {
last_sequence = event.sequence;
yield search_event_frame(event)?;
}
let completed = serde_json::json!({
"search_id": body_search_id,
"sequence": last_sequence + 1,
"kind": "search.completed",
"entity_key": null,
"entity": { "partial": false },
"entity": { "partial": false, "podcast_discovery": podcast_discovery },
});
yield Bytes::from(format!("event: search.completed\ndata: {completed}\n\n"));
}
Err(err) => {
podcasts.abort();
let failed = serde_json::json!({
"search_id": body_search_id,
"sequence": 1,
@@ -7531,6 +7547,14 @@ async fn federation_search_events_handler(
.map_err(|err| cot::Error::internal(err.to_string()))
}
/// One server-sent event of a federated search: `event: <kind>` and the
/// event as JSON.
fn search_event_frame(event: &crate::federation::client::SearchEvent) -> cot::Result<Bytes> {
let data = serde_json::to_string(event).map_err(|err| cot::Error::internal(err.to_string()))?;
let kind = event.kind;
Ok(Bytes::from(format!("event: {kind}\ndata: {data}\n\n")))
}
async fn federation_artist_events_handler(
auth_ctx: auth::AuthContext,
session: Session,
+18 -9
View File
@@ -8,7 +8,10 @@
//! plus the verbatim device-sync DTO under [`WIRE_TRACK_KEY`] and its explicit
//! [`PlaybackSource`] under [`SOURCE_KEY`]. Converting back to the wire uses
//! the verbatim DTO, so such items survive browser -> server -> peers ->
//! server -> browser unchanged even though this web player cannot play them.
//! server -> browser unchanged. The web player plays podcast episodes
//! through `/api/podcasts/episodes/{id}/playback` (publisher URL or the
//! optional proxy); placeholders and unresolved federated music stay
//! unplayable here.
//!
//! Gating and Jam decisions are delegated to `music_dht::device_sync`; this
//! module only maps browser JSON onto the shared DTOs.
@@ -34,9 +37,6 @@ pub(crate) const POSITION_UNKNOWN_KEY: &str = "position_unknown";
/// Reason shown for federated music that is not in this library.
pub(crate) const FED_UNAVAILABLE_REASON: &str = "Not available on this web player yet";
/// Reason shown for podcast episodes, which this web player cannot play yet.
pub(crate) const PODCAST_UNAVAILABLE_REASON: &str =
"Podcast episodes cannot be played on the web player yet";
/// Reason shown for Jam placeholders of someone else's private item.
pub(crate) const PRIVATE_UNAVAILABLE_REASON: &str = "Private item of the Jam host";
@@ -68,13 +68,15 @@ pub(crate) fn external_track_json(
"release_id": 0,
"release_title": episode.podcast_title,
"release_year": serde_json::Value::Null,
"cover_url": serde_json::Value::Null,
// Artwork is loaded from the publisher, like the audio.
"cover_url": episode.image_url,
// The player asks `/api/podcasts/episodes/{id}/playback` for the
// audio source when it starts the episode.
"stream_url": "",
"uploader_name": "Podcast",
"audio_format": episode.mime,
"content_id": serde_json::Value::Null,
"unavailable": true,
"unavailable_reason": PODCAST_UNAVAILABLE_REASON,
"unavailable": false,
}),
Some(PlaybackSource::Private) => serde_json::json!({
"id": placeholder_track_id(Some("private"), index),
@@ -490,8 +492,15 @@ mod tests {
let mut wire = PlaybackTrack::podcast_episode(episode());
wire.play_count = 3;
let json = external_track_json(&wire, 4, None);
assert_eq!(json["unavailable"], true);
assert_eq!(json["unavailable_reason"], PODCAST_UNAVAILABLE_REASON);
// Playable on the web player (the source is resolved on play).
assert_eq!(json["unavailable"], false);
assert!(json.get("unavailable_reason").is_none());
assert_eq!(json["stream_url"], "");
assert_eq!(
json["cover_url"],
Value::Null,
"the test episode has no artwork"
);
assert_eq!(json["title"], "Episode 42");
assert_eq!(json[SOURCE_KEY]["kind"], "podcast_episode");
assert!(json["id"].as_i64().unwrap() < 0);
+790
View File
@@ -0,0 +1,790 @@
//! Local user actions: they build the device-sync operation, append it to
//! the account's sync log with its stamp and apply it through the shared
//! code path, all in one transaction.
use anyhow::Result;
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
use music_dht::podcast::{
EpisodeId, FeedUrl, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta, ProgressRecord,
ProgressStatus, effective_share, merge_feed_aliases, merge_listen_session,
next_podcast_generation, resolve_podcast_id,
};
use sqlx::{PgConnection, Row};
use super::apply::{self, ApplyOutcome};
use super::{PodcastStore, PodcastStoreError, invalid};
use crate::federation::devices;
/// Result of [`PodcastStore::subscribe`].
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct SubscribeOutcome {
/// Podcast the URL resolved to (an existing one when the URL is a known
/// feed URL or alias).
pub(crate) podcast_id: PodcastId,
/// The podcast was already subscribed; no operation was emitted.
pub(crate) already_subscribed: bool,
/// Operations appended to the sync log (and applied).
pub(crate) ops: Vec<SyncOpWire>,
}
/// A listening session of this account as seen by the progress register.
///
/// Returned by explicit actions; periodic checkpoints of the session use its
/// `generation` ([`PodcastStore::checkpoint`]).
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ProgressSession {
/// Episode being played.
pub(crate) episode: PodcastEpisodeRef,
/// Session id (also the `ListenSession::session_id`).
pub(crate) session_id: String,
/// Generation of the session's latest explicit action.
pub(crate) generation: u64,
}
/// A progress register written by a local action.
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ProgressWrite {
/// The register as written.
pub(crate) record: ProgressRecord,
/// The session after the action (its generation for later checkpoints).
pub(crate) session: ProgressSession,
/// Operation appended to the sync log.
pub(crate) op: SyncOpWire,
}
/// Result of [`PodcastStore::checkpoint`].
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum CheckpointOutcome {
/// The checkpoint was written.
Written(Box<ProgressWrite>),
/// The register holds a greater generation (another device or an
/// explicit action took over). Nothing was written: stop writing at the
/// old generation, or continue locally with an explicit action
/// ([`PodcastStore::start_session`] / [`PodcastStore::seek`]).
Superseded {
/// Current register.
current: ProgressRecord,
},
/// This session already completed the episode at its generation; a
/// checkpoint would turn it back into "in progress". Nothing was written.
AlreadyCompleted {
/// Current register.
current: ProgressRecord,
},
}
#[derive(Debug, Clone, Copy)]
enum ExplicitAction {
Start(i64),
Seek(i64),
Restart,
MarkPlayed,
MarkUnplayed,
}
/// Validates the payload (with a provisional stamp), appends it to the sync
/// log with the next stamp of `device_id` and applies exactly that operation
/// through [`apply::apply_op`]. `make` receives the operation id, which
/// explicit actions without a session use as their session id.
pub(crate) async fn emit(
conn: &mut PgConnection,
user_id: i64,
device_id: &str,
make: impl Fn(&str) -> SyncOpPayload,
) -> Result<SyncOpWire> {
SyncOpWire {
op_id: "provisional".to_string(),
origin_device_id: device_id.to_string(),
seq: 0,
hlc_ms: 0,
payload: make("provisional"),
}
.validate_podcast()
.map_err(invalid)?;
let op = devices::append_local_op(conn, user_id, device_id, &make).await?;
match apply::apply_op(conn, user_id, &op).await? {
ApplyOutcome::Rejected(reason) => Err(invalid(reason)),
_ => Ok(op),
}
}
/// Resolves a feed URL against every known URL of every podcast of the
/// account: the alias sets, the catalog fetch URLs and the registers'
/// `podcast.feed_url` (`resolve_podcast_id`).
pub(crate) async fn resolve_podcast(
conn: &mut PgConnection,
user_id: i64,
url: &FeedUrl,
) -> Result<PodcastId> {
let rows = sqlx::query(
"SELECT podcast_id FROM furumusic__podcast_feed_alias
WHERE user_id = $1 AND feed_url = $2
UNION SELECT podcast_id FROM furumusic__podcast
WHERE user_id = $1 AND feed_url = $2
UNION SELECT podcast_id FROM furumusic__podcast_subscription
WHERE user_id = $1 AND podcast_json->>'feed_url' = $2",
)
.bind(user_id)
.bind(url.as_str())
.fetch_all(&mut *conn)
.await?;
let ids = rows
.iter()
.map(|row| super::podcast_id_col(row, "podcast_id"))
.collect::<sqlx::Result<Vec<_>>>()?;
Ok(resolve_podcast_id(url, ids.iter().map(|id| (id, url))))
}
async fn catalog_meta(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<PodcastMeta>> {
let row = sqlx::query(
"SELECT feed_url, title, author, description, link, image_url
FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
let Some(row) = row else {
return Ok(None);
};
Ok(Some(PodcastMeta {
feed_url: super::feed_url_col(&row, "feed_url")?,
title: row.try_get("title")?,
author: row.try_get("author")?,
description: row.try_get("description")?,
link: row.try_get("link")?,
image_url: row.try_get("image_url")?,
}))
}
async fn share_payload(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
shared: bool,
) -> Result<SyncOpPayload> {
let subscription = apply::load_subscription_register(conn, user_id, podcast_id).await?;
let share = apply::load_share(conn, user_id, podcast_id).await?;
Ok(SyncOpPayload::PodcastShareSet {
podcast_id: podcast_id.clone(),
shared,
generation: next_podcast_generation(subscription.as_ref(), share.as_ref()),
})
}
/// The reference carried by operations: identity and display only.
fn op_ref(episode: &PodcastEpisodeRef) -> PodcastEpisodeRef {
let mut episode = episode.clone();
episode.progress_generation = None;
episode
}
impl PodcastStore {
/// Subscribes to the podcast of `podcast.feed_url` (explicit user action).
///
/// The URL is resolved against known feed URLs and aliases first, so
/// subscribing again, through a redirect target or from someone else's
/// card reuses the existing podcast with its progress and history. When
/// the podcast is already subscribed nothing is emitted. Otherwise one
/// `podcast_subscription_set` with generation
/// `next_podcast_generation(..)` and the known aliases is appended and
/// applied. The new subscription is private until
/// [`PodcastStore::set_shared`].
///
/// Pass the channel metadata of the fetched feed
/// (`ParsedFeed::to_podcast_meta(url)`), then ingest the episodes with
/// [`PodcastStore::apply_feed`].
pub(crate) async fn subscribe(
&self,
user_id: i64,
podcast: &PodcastMeta,
) -> Result<SubscribeOutcome> {
let meta = podcast.bounded();
meta.validate().map_err(invalid)?;
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let podcast_id = resolve_podcast(&mut tx, user_id, &meta.feed_url).await?;
let current = apply::load_subscription_register(&mut tx, user_id, &podcast_id).await?;
if current.as_ref().is_some_and(|record| record.subscribed) {
return Ok(SubscribeOutcome {
podcast_id,
already_subscribed: true,
ops: Vec::new(),
});
}
let active: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM furumusic__podcast_subscription
WHERE user_id = $1 AND subscribed",
)
.bind(user_id)
.fetch_one(&mut *tx)
.await?;
if usize::try_from(active).unwrap_or(usize::MAX) >= super::MAX_SUBSCRIPTIONS_PER_ACCOUNT {
return Err(
PodcastStoreError::SubscriptionLimit(super::MAX_SUBSCRIPTIONS_PER_ACCOUNT).into(),
);
}
let share = apply::load_share(&mut tx, user_id, &podcast_id).await?;
let generation = next_podcast_generation(current.as_ref(), share.as_ref());
let aliases = merge_feed_aliases(
&podcast_id,
&apply::load_aliases(&mut tx, user_id, &podcast_id).await?,
std::slice::from_ref(&meta.feed_url),
);
let op = emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastSubscriptionSet {
podcast_id: podcast_id.clone(),
subscribed: true,
podcast: meta.clone(),
feed_aliases: aliases.clone(),
generation,
}
})
.await?;
tx.commit().await?;
// A new subscription is private, but it supersedes an earlier share
// decision of a former subscription.
super::discovery::request_reconcile();
Ok(SubscribeOutcome {
podcast_id,
already_subscribed: false,
ops: vec![op],
})
}
/// Unsubscribes (explicit user action).
///
/// Emits `podcast_subscription_set { subscribed: false }` followed by
/// `podcast_share_set { shared: false }`, each with the next podcast
/// generation. Progress, sessions and the catalog are kept, so
/// subscribing again continues where the user stopped. Does nothing when
/// the podcast is not subscribed.
pub(crate) async fn unsubscribe(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<SyncOpWire>> {
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let Some(current) = apply::load_subscription_register(&mut tx, user_id, podcast_id).await?
else {
return Ok(Vec::new());
};
if !current.subscribed {
return Ok(Vec::new());
}
let share = apply::load_share(&mut tx, user_id, podcast_id).await?;
let generation = next_podcast_generation(Some(&current), share.as_ref());
let mut meta = catalog_meta(&mut tx, user_id, podcast_id)
.await?
.map(|meta| meta.bounded())
.unwrap_or_else(|| current.podcast.clone());
meta.feed_url = current.podcast.feed_url.clone();
let aliases = current.feed_aliases.clone();
let unsubscribe = emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastSubscriptionSet {
podcast_id: podcast_id.clone(),
subscribed: false,
podcast: meta.clone(),
feed_aliases: aliases.clone(),
generation,
}
})
.await?;
let revoke_payload = share_payload(&mut tx, user_id, podcast_id, false).await?;
let revoke = emit(&mut tx, user_id, &device_id, |_| revoke_payload.clone()).await?;
super::discovery::clear_confirmation(&mut tx, user_id, podcast_id).await?;
tx.commit().await?;
super::discovery::request_reconcile();
Ok(vec![unsubscribe, revoke])
}
/// Sets the decision to publish the podcast to the federation.
///
/// Sharing needs an active subscription and is a no-op when the podcast
/// is already effectively shared (`effective_share`). Revoking is a no-op
/// when no share decision says `true`. The write uses the next podcast
/// generation, so it orders after the subscription it observed.
///
/// Revoking also forgets the account's privacy confirmation, so sharing
/// again asks again. Publishing and revoking the card is the
/// reconciler's job (`super::discovery`), which this requests.
pub(crate) async fn set_shared(
&self,
user_id: i64,
podcast_id: &PodcastId,
shared: bool,
) -> Result<Vec<SyncOpWire>> {
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let subscription = apply::load_subscription_register(&mut tx, user_id, podcast_id).await?;
let share = apply::load_share(&mut tx, user_id, podcast_id).await?;
if shared {
if !subscription
.as_ref()
.is_some_and(|record| record.subscribed)
{
return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into());
}
if effective_share(subscription.as_ref(), share.as_ref()) {
return Ok(Vec::new());
}
} else if !share.as_ref().is_some_and(|record| record.shared) {
return Ok(Vec::new());
}
let payload = share_payload(&mut tx, user_id, podcast_id, shared).await?;
let op = emit(&mut tx, user_id, &device_id, |_| payload.clone()).await?;
if !shared {
super::discovery::clear_confirmation(&mut tx, user_id, podcast_id).await?;
}
tx.commit().await?;
super::discovery::request_reconcile();
Ok(vec![op])
}
/// Shares a podcast whose card has privacy warnings, with the
/// account's confirmation of exactly `feed_url` / `id_feed_url`.
///
/// In one transaction: the card the account would publish now is
/// rebuilt and compared with the confirmed URLs (a feed that moved
/// meanwhile returns `Ok(None)` and writes nothing, so the user is asked
/// about the new URLs); the share decision is written like
/// [`PodcastStore::set_shared`] when the podcast is not already
/// effectively shared; the confirmation is stored for the resulting
/// share decision (`ShareRecord::op_id`). Returns the emitted operations.
pub(crate) async fn share_confirmed(
&self,
user_id: i64,
podcast_id: &PodcastId,
feed_url: &FeedUrl,
id_feed_url: Option<&FeedUrl>,
now_ms: i64,
) -> Result<Option<Vec<SyncOpWire>>> {
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let Some(account) =
super::discovery::load_account_share(&mut tx, user_id, podcast_id).await?
else {
return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into());
};
if !account.subscription.subscribed {
return Err(PodcastStoreError::NotSubscribed(podcast_id.clone()).into());
}
let card = account.card.as_ref().map_err(invalid)?;
if card.feed_url != *feed_url || card.id_feed_url.as_ref() != id_feed_url {
return Ok(None);
}
let mut ops = Vec::new();
let share_op_id = if account.is_shared() {
account
.share
.as_ref()
.map(|share| share.op_id.clone())
.expect("an effective share has a share register")
} else {
let payload = share_payload(&mut tx, user_id, podcast_id, true).await?;
let op = emit(&mut tx, user_id, &device_id, |_| payload.clone()).await?;
let op_id = op.op_id.clone();
ops.push(op);
op_id
};
let confirmation = super::discovery::ShareConfirmation {
share_op_id,
feed_url: card.feed_url.clone(),
id_feed_url: card.id_feed_url.clone(),
};
super::discovery::store_confirmation(&mut tx, user_id, podcast_id, &confirmation, now_ms)
.await?;
tx.commit().await?;
super::discovery::request_reconcile();
Ok(Some(ops))
}
/// Forgets the account's privacy confirmation of a podcast.
pub(crate) async fn clear_share_confirmation(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<()> {
let mut conn = self.pool.acquire().await?;
super::discovery::clear_confirmation(&mut conn, user_id, podcast_id).await
}
/// One account's publication view of a podcast (registers, card,
/// confirmation); `None` without a subscription register.
pub(crate) async fn account_share(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<super::discovery::AccountShare>> {
let mut conn = self.pool.acquire().await?;
super::discovery::load_account_share(&mut conn, user_id, podcast_id).await
}
async fn explicit_progress(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
session_id: Option<&str>,
carried_generation: Option<u64>,
action: ExplicitAction,
) -> Result<ProgressWrite> {
episode.validate().map_err(invalid)?;
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let current = apply::load_progress(&mut tx, user_id, &episode.episode_id).await?;
let max_seen = current
.as_ref()
.map(|record| record.generation)
.into_iter()
.chain(carried_generation)
.max();
let generation = ProgressRecord::explicit_generation(max_seen);
let current_position = current.as_ref().map_or(0, |record| record.position_ms);
let (position_ms, status) = match action {
ExplicitAction::Start(position) | ExplicitAction::Seek(position) => {
(position, ProgressStatus::InProgress)
}
ExplicitAction::Restart => (0, ProgressStatus::InProgress),
ExplicitAction::MarkPlayed => (current_position, ProgressStatus::Completed),
ExplicitAction::MarkUnplayed => (0, ProgressStatus::NotStarted),
};
let reference = op_ref(episode);
let op = emit(&mut tx, user_id, &device_id, |op_id| {
SyncOpPayload::PodcastProgressSet {
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
position_ms,
status,
generation,
session_id: session_id.unwrap_or(op_id).to_string(),
episode: Some(reference.clone()),
}
})
.await?;
tx.commit().await?;
let record = op.podcast_progress().expect("progress payload");
Ok(ProgressWrite {
session: ProgressSession {
episode: episode.clone(),
session_id: record.session_id.clone(),
generation,
},
record,
op,
})
}
/// Starts a listening session at `position_ms` (explicit action).
///
/// The generation is `explicit_generation(max(local, carried_generation))`;
/// pass `PodcastEpisodeRef::progress_generation` of a queue item received
/// through a handoff as `carried_generation`. Ordinary Play resumes from
/// [`PodcastStore::resume_position_ms`].
pub(crate) async fn start_session(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
session_id: &str,
position_ms: i64,
carried_generation: Option<u64>,
) -> Result<ProgressWrite> {
self.explicit_progress(
user_id,
episode,
Some(session_id),
carried_generation,
ExplicitAction::Start(position_ms),
)
.await
}
/// Periodic checkpoint (or pause/stop save) of a running session.
///
/// Keeps the session generation. When the register already holds a
/// greater generation nothing is written ([`CheckpointOutcome::Superseded`]).
pub(crate) async fn checkpoint(
&self,
user_id: i64,
session: &ProgressSession,
position_ms: i64,
) -> Result<CheckpointOutcome> {
let episode = &session.episode;
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
if let Some(current) = apply::load_progress(&mut tx, user_id, &episode.episode_id).await? {
if current.generation > session.generation {
return Ok(CheckpointOutcome::Superseded { current });
}
if current.generation == session.generation
&& current.session_id == session.session_id
&& current.status == ProgressStatus::Completed
{
return Ok(CheckpointOutcome::AlreadyCompleted { current });
}
}
let reference = op_ref(episode);
let op = emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastProgressSet {
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
position_ms,
status: ProgressStatus::InProgress,
generation: session.generation,
session_id: session.session_id.clone(),
episode: Some(reference.clone()),
}
})
.await?;
tx.commit().await?;
let record = op.podcast_progress().expect("progress payload");
Ok(CheckpointOutcome::Written(Box::new(ProgressWrite {
record,
session: session.clone(),
op,
})))
}
/// User seek during a session (explicit action; the returned session
/// carries the new generation for later checkpoints).
pub(crate) async fn seek(
&self,
user_id: i64,
session: &ProgressSession,
position_ms: i64,
) -> Result<ProgressWrite> {
self.explicit_progress(
user_id,
&session.episode,
Some(&session.session_id),
Some(session.generation),
ExplicitAction::Seek(position_ms),
)
.await
}
/// Completes a running session (end of stream or `reaches_completion`)
/// unless another write took over. Record the session itself with
/// [`PodcastStore::record_listen_session`].
///
/// Like [`PodcastStore::checkpoint`], nothing is written when the
/// register holds a greater generation ([`CheckpointOutcome::Superseded`])
/// or when this session already completed the episode
/// ([`CheckpointOutcome::AlreadyCompleted`]). Otherwise `completed` is
/// written with an explicit generation; the check and the write
/// happen in one transaction, so a periodic save after the end never
/// completes the episode twice.
pub(crate) async fn complete_if_current(
&self,
user_id: i64,
session: &ProgressSession,
position_ms: i64,
) -> Result<CheckpointOutcome> {
let episode = &session.episode;
episode.validate().map_err(invalid)?;
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let current = apply::load_progress(&mut tx, user_id, &episode.episode_id).await?;
if let Some(current) = current.as_ref() {
if current.generation > session.generation {
return Ok(CheckpointOutcome::Superseded {
current: current.clone(),
});
}
if current.session_id == session.session_id
&& current.status == ProgressStatus::Completed
{
return Ok(CheckpointOutcome::AlreadyCompleted {
current: current.clone(),
});
}
}
let generation = ProgressRecord::explicit_generation(
current
.as_ref()
.map(|record| record.generation)
.into_iter()
.chain(Some(session.generation))
.max(),
);
let reference = op_ref(episode);
let op = emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastProgressSet {
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
position_ms,
status: ProgressStatus::Completed,
generation,
session_id: session.session_id.clone(),
episode: Some(reference.clone()),
}
})
.await?;
tx.commit().await?;
let record = op.podcast_progress().expect("progress payload");
Ok(CheckpointOutcome::Written(Box::new(ProgressWrite {
record,
session: ProgressSession {
episode: episode.clone(),
session_id: session.session_id.clone(),
generation,
},
op,
})))
}
/// "Start over": position 0, in progress, explicit generation.
pub(crate) async fn restart(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
session_id: &str,
) -> Result<ProgressWrite> {
self.explicit_progress(
user_id,
episode,
Some(session_id),
None,
ExplicitAction::Restart,
)
.await
}
/// "Mark played": status completed, explicit generation. Creates no
/// listening session and invents no listening time.
pub(crate) async fn mark_played(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
) -> Result<ProgressWrite> {
self.explicit_progress(user_id, episode, None, None, ExplicitAction::MarkPlayed)
.await
}
/// "Mark unplayed": status not started at position 0, explicit generation.
pub(crate) async fn mark_unplayed(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
) -> Result<ProgressWrite> {
self.explicit_progress(user_id, episode, None, None, ExplicitAction::MarkUnplayed)
.await
}
/// Records (or grows) a listening session of this account's device.
///
/// Sessions below the history threshold (`ListenSession::should_record`)
/// and versions that do not grow the stored session emit nothing.
/// `session.device_id` must be the account's web device id. The episode
/// reference defaults to the catalog's.
pub(crate) async fn record_listen_session(
&self,
user_id: i64,
session: &ListenSession,
episode: Option<&PodcastEpisodeRef>,
) -> Result<Option<SyncOpWire>> {
let device_id = devices::local_device_id(&self.pool, user_id).await?;
if session.device_id != device_id {
return Err(invalid(
"session device_id differs from the local device id",
));
}
session.validate().map_err(invalid)?;
if !session.should_record() {
return Ok(None);
}
let mut tx = self.begin_write(user_id).await?;
if let Some(current) = apply::load_session(&mut tx, user_id, &session.session_id).await? {
let merged = merge_listen_session(&current, session).map_err(invalid)?;
if merged == current {
return Ok(None);
}
}
let reference = match episode {
Some(episode) => Some(op_ref(episode)),
None => super::queries::episode_ref_in(&mut tx, user_id, &session.episode_id)
.await?
.map(|reference| op_ref(&reference)),
};
let op = emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastListenRecorded {
session: session.clone(),
episode: reference.clone(),
}
})
.await?;
tx.commit().await?;
Ok(Some(op))
}
/// Makes sure the catalog knows an episode a queue item refers to.
///
/// Queue items received through a handoff carry a self-contained
/// `PodcastEpisodeRef`; the account may not have refreshed that feed yet.
/// The reference is recorded like one learned through sync
/// (`source = 'sync'`, never replacing feed metadata). Returns whether a
/// row was added. No register is written. A new row is refused with
/// [`PodcastStoreError::ReferenceLimit`] once the account has
/// [`super::MAX_REFERENCE_EPISODES_PER_ACCOUNT`] such episodes.
pub(crate) async fn ensure_episode(
&self,
user_id: i64,
episode: &PodcastEpisodeRef,
) -> Result<bool> {
episode.validate().map_err(invalid)?;
let mut tx = self.begin_write(user_id).await?;
let known = super::queries::episode_ref_in(&mut tx, user_id, &episode.episode_id)
.await?
.is_some();
if !known {
// Counted under the account's write lock: concurrent requests
// cannot pass the limit together.
let references: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM furumusic__podcast_episode
WHERE user_id = $1 AND source = 'sync'",
)
.bind(user_id)
.fetch_one(&mut *tx)
.await?;
if usize::try_from(references).unwrap_or(usize::MAX)
>= super::MAX_REFERENCE_EPISODES_PER_ACCOUNT
{
return Err(PodcastStoreError::ReferenceLimit(
super::MAX_REFERENCE_EPISODES_PER_ACCOUNT,
)
.into());
}
apply::ensure_episode_from_ref(&mut tx, user_id, episode).await?;
}
tx.commit().await?;
Ok(!known)
}
/// Resolves a feed URL to a podcast id: the existing podcast when the
/// URL is a known feed URL or alias (`resolve_podcast_id`), otherwise
/// the id a first subscription through this URL would derive.
pub(crate) async fn resolve_podcast_id(
&self,
user_id: i64,
url: &FeedUrl,
) -> Result<PodcastId> {
let mut conn = self.pool.acquire().await?;
resolve_podcast(&mut conn, user_id, url).await
}
/// The episode reference for queueing or progress actions (with
/// `progress_generation` set to the stored register's generation), or
/// `None` when the episode is unknown.
pub(crate) async fn episode_ref(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<PodcastEpisodeRef>> {
let mut conn = self.pool.acquire().await?;
super::queries::episode_ref_in(&mut conn, user_id, episode_id).await
}
}
+1390
View File
File diff suppressed because it is too large. Load diff
+896
View File
@@ -0,0 +1,896 @@
//! The single code path that applies podcast records to the store.
//!
//! Remote operations, local actions and snapshot sections all end here.
//! Records come from the `SyncOpWire` materializers of
//! `music_dht::device_sync` and are merged with the functions of
//! `music_dht::podcast`, so applying is idempotent and order-independent.
use anyhow::Result;
use music_dht::device_sync::{SyncOpPayload, SyncOpWire, SyncSnapshot};
use music_dht::podcast::{
EpisodeId, EpisodeIdentityHints, EpisodeMeta, EpisodeType, FeedUrl, KnownEpisode,
ListenSession, PodcastEpisodeRef, PodcastError, PodcastId, ProgressRecord, ShareRecord,
SubscriptionRecord, merge_feed_aliases, merge_listen_session,
};
use sqlx::{PgConnection, Row};
use super::{
PodcastStore, PodcastStoreError, SESSION_COLUMNS, episode_id_col, episode_search_norm,
feed_url_col, generation_from_sql, generation_to_sql, invalid, now_ms, podcast_meta_col,
progress_columns, progress_from_row, session_from_row, status_to_sql,
};
/// Result of applying one operation.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ApplyOutcome {
/// Stored state changed.
Changed,
/// The operation was already reflected (duplicate, or older than the
/// stored register).
Unchanged,
/// Not a podcast operation; nothing to do here.
Ignored,
/// The operation violates the contract and was skipped. Acknowledge it
/// anyway so the log keeps moving (`PODCASTS.md`).
Rejected(String),
}
/// Counters returned by [`PodcastStore::apply_snapshot`].
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) struct SnapshotApplyStats {
/// Records that changed stored state.
pub(crate) changed: usize,
/// Valid records already reflected locally.
pub(crate) unchanged: usize,
/// Records that failed validation and were skipped.
pub(crate) rejected: usize,
/// Whether the podcast sections were ignored for exceeding the counts.
pub(crate) ignored_over_limits: bool,
}
/// Where an episode row came from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum EpisodeSource {
/// Parsed from the podcast's feed by this account.
Feed,
/// Learned from a `PodcastEpisodeRef` (sync operation or queue item).
Sync,
}
impl EpisodeSource {
fn as_sql(self) -> &'static str {
match self {
Self::Feed => "feed",
Self::Sync => "sync",
}
}
}
pub(crate) async fn load_subscription_register(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<SubscriptionRecord>> {
let row = sqlx::query(
"SELECT subscribed, podcast_json, generation, hlc_ms, op_id
FROM furumusic__podcast_subscription WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
let Some(row) = row else {
return Ok(None);
};
let record = SubscriptionRecord {
podcast_id: podcast_id.clone(),
subscribed: row.try_get("subscribed")?,
podcast: podcast_meta_col(&row, "podcast_json")?,
feed_aliases: load_aliases(conn, user_id, podcast_id).await?,
generation: generation_from_sql(row.try_get("generation")?),
hlc_ms: row.try_get("hlc_ms")?,
op_id: row.try_get("op_id")?,
};
Ok(Some(record))
}
pub(crate) async fn load_aliases(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<FeedUrl>> {
let rows = sqlx::query(
"SELECT feed_url FROM furumusic__podcast_feed_alias
WHERE user_id = $1 AND podcast_id = $2 ORDER BY feed_url COLLATE \"C\"",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_all(&mut *conn)
.await?;
let mut aliases = rows
.iter()
.map(|row| feed_url_col(row, "feed_url"))
.collect::<sqlx::Result<Vec<_>>>()?;
// `merge_feed_aliases` returns byte order; the database collation may
// differ (ICU, glibc locales), and the comparisons below need equality.
aliases.sort();
Ok(aliases)
}
pub(crate) fn share_from_row(row: &sqlx::postgres::PgRow) -> sqlx::Result<ShareRecord> {
Ok(ShareRecord {
podcast_id: super::podcast_id_col(row, "podcast_id")?,
shared: row.try_get("shared")?,
generation: generation_from_sql(row.try_get("generation")?),
hlc_ms: row.try_get("hlc_ms")?,
op_id: row.try_get("op_id")?,
})
}
pub(crate) async fn load_share(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<ShareRecord>> {
let row = sqlx::query(
"SELECT podcast_id, shared, generation, hlc_ms, op_id
FROM furumusic__podcast_share WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
Ok(row.as_ref().map(share_from_row).transpose()?)
}
pub(crate) async fn load_progress(
conn: &mut PgConnection,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<ProgressRecord>> {
let row = sqlx::query(&format!(
"SELECT {} FROM furumusic__podcast_progress p
WHERE p.user_id = $1 AND p.episode_id = $2",
progress_columns("p", "")
))
.bind(user_id)
.bind(episode_id.as_str())
.fetch_optional(&mut *conn)
.await?;
Ok(row
.as_ref()
.map(|row| progress_from_row(row, ""))
.transpose()?)
}
pub(crate) async fn load_session(
conn: &mut PgConnection,
user_id: i64,
session_id: &str,
) -> Result<Option<ListenSession>> {
let row = sqlx::query(&format!(
"SELECT {SESSION_COLUMNS} FROM furumusic__podcast_listen_session s
WHERE s.user_id = $1 AND s.session_id = $2"
))
.bind(user_id)
.bind(session_id)
.fetch_optional(&mut *conn)
.await?;
Ok(row.as_ref().map(session_from_row).transpose()?)
}
/// Replaces the alias set of a podcast with `merged` when it differs.
async fn store_aliases(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
merged: &[FeedUrl],
) -> Result<bool> {
let current = load_aliases(conn, user_id, podcast_id).await?;
if current == merged {
return Ok(false);
}
sqlx::query("DELETE FROM furumusic__podcast_feed_alias WHERE user_id = $1 AND podcast_id = $2")
.bind(user_id)
.bind(podcast_id.as_str())
.execute(&mut *conn)
.await?;
let urls: Vec<&str> = merged.iter().map(FeedUrl::as_str).collect();
sqlx::query(
"INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url)
SELECT $1, $2, url FROM UNNEST($3::text[]) AS url",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(&urls)
.execute(&mut *conn)
.await?;
Ok(true)
}
/// Merges feed URLs into the bounded grow-only alias set of a podcast.
///
/// Returns whether the stored set changed.
pub(crate) async fn merge_aliases(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
urls: &[FeedUrl],
) -> Result<bool> {
let current = load_aliases(conn, user_id, podcast_id).await?;
let merged = merge_feed_aliases(podcast_id, &current, urls);
store_aliases(conn, user_id, podcast_id, &merged).await
}
/// Inserts the catalog row of a podcast if it does not exist yet.
pub(crate) async fn ensure_podcast_row(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
feed_url: &FeedUrl,
title: &str,
now: i64,
) -> Result<()> {
sqlx::query(
"INSERT INTO furumusic__podcast
(user_id, podcast_id, feed_url, title, meta_from_feed, created_at_ms, updated_at_ms)
VALUES ($1, $2, $3, $4, FALSE, $5, $5)
ON CONFLICT (user_id, podcast_id) DO NOTHING",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(feed_url.as_str())
.bind(title)
.bind(now)
.execute(&mut *conn)
.await?;
Ok(())
}
/// Applies a subscription register record (operation or snapshot).
///
/// The register is last-writer-wins ([`SubscriptionRecord::supersedes`]);
/// the alias set is a bounded grow-only set merged independently, so the
/// stored aliases are the bounded union of every URL seen (what
/// `merge_subscription` computes). The catalog row is created from the
/// register metadata and follows the winning register until this account
/// has refreshed the feed itself.
pub(crate) async fn apply_subscription_record(
conn: &mut PgConnection,
user_id: i64,
record: &SubscriptionRecord,
now: i64,
) -> Result<bool> {
let generation = generation_to_sql(record.generation)?;
let podcast_id = &record.podcast_id;
let current = load_subscription_register(conn, user_id, podcast_id).await?;
let wins = current
.as_ref()
.is_none_or(|current| record.supersedes(current));
let mut changed = false;
if wins {
sqlx::query(
"INSERT INTO furumusic__podcast_subscription
(user_id, podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (user_id, podcast_id) DO UPDATE SET
subscribed = EXCLUDED.subscribed, podcast_json = EXCLUDED.podcast_json,
generation = EXCLUDED.generation, hlc_ms = EXCLUDED.hlc_ms,
op_id = EXCLUDED.op_id",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(record.subscribed)
.bind(serde_json::to_value(&record.podcast)?)
.bind(generation)
.bind(record.hlc_ms)
.bind(&record.op_id)
.execute(&mut *conn)
.await?;
changed = true;
}
let mut urls = record.feed_aliases.clone();
urls.push(record.podcast.feed_url.clone());
changed |= merge_aliases(conn, user_id, podcast_id, &urls).await?;
ensure_podcast_row(
conn,
user_id,
podcast_id,
&record.podcast.feed_url,
&record.podcast.title,
now,
)
.await?;
if wins {
let meta = &record.podcast;
sqlx::query(
"UPDATE furumusic__podcast
SET feed_url = $3, title = $4, author = $5,
description = COALESCE($6, description), link = $7, image_url = $8,
updated_at_ms = $9
WHERE user_id = $1 AND podcast_id = $2 AND NOT meta_from_feed",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(meta.feed_url.as_str())
.bind(&meta.title)
.bind(&meta.author)
.bind(&meta.description)
.bind(&meta.link)
.bind(&meta.image_url)
.bind(now)
.execute(&mut *conn)
.await?;
}
Ok(changed)
}
/// Applies a share register record.
pub(crate) async fn apply_share_record(
conn: &mut PgConnection,
user_id: i64,
record: &ShareRecord,
) -> Result<bool> {
let generation = generation_to_sql(record.generation)?;
let current = load_share(conn, user_id, &record.podcast_id).await?;
if current
.as_ref()
.is_some_and(|current| !record.supersedes(current))
{
return Ok(false);
}
sqlx::query(
"INSERT INTO furumusic__podcast_share
(user_id, podcast_id, shared, generation, hlc_ms, op_id)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (user_id, podcast_id) DO UPDATE SET
shared = EXCLUDED.shared, generation = EXCLUDED.generation,
hlc_ms = EXCLUDED.hlc_ms, op_id = EXCLUDED.op_id",
)
.bind(user_id)
.bind(record.podcast_id.as_str())
.bind(record.shared)
.bind(generation)
.bind(record.hlc_ms)
.bind(&record.op_id)
.execute(&mut *conn)
.await?;
Ok(true)
}
/// Applies a progress register record. Never touches the catalog.
pub(crate) async fn apply_progress_record(
conn: &mut PgConnection,
user_id: i64,
record: &ProgressRecord,
) -> Result<bool> {
let generation = generation_to_sql(record.generation)?;
let current = load_progress(conn, user_id, &record.episode_id).await?;
if current
.as_ref()
.is_some_and(|current| !record.supersedes(current))
{
return Ok(false);
}
sqlx::query(
"INSERT INTO furumusic__podcast_progress
(user_id, episode_id, podcast_id, position_ms, status, generation, session_id,
hlc_ms, device_id, op_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (user_id, episode_id) DO UPDATE SET
podcast_id = EXCLUDED.podcast_id, position_ms = EXCLUDED.position_ms,
status = EXCLUDED.status, generation = EXCLUDED.generation,
session_id = EXCLUDED.session_id, hlc_ms = EXCLUDED.hlc_ms,
device_id = EXCLUDED.device_id, op_id = EXCLUDED.op_id",
)
.bind(user_id)
.bind(record.episode_id.as_str())
.bind(record.podcast_id.as_str())
.bind(record.position_ms)
.bind(status_to_sql(record.status))
.bind(generation)
.bind(&record.session_id)
.bind(record.hlc_ms)
.bind(&record.device_id)
.bind(&record.op_id)
.execute(&mut *conn)
.await?;
Ok(true)
}
/// Adds or grows a listening session (grow-only set keyed by `session_id`).
pub(crate) async fn apply_listen_session(
conn: &mut PgConnection,
user_id: i64,
session: &ListenSession,
) -> Result<bool> {
let current = load_session(conn, user_id, &session.session_id).await?;
let merged = match &current {
Some(current) => merge_listen_session(current, session).map_err(invalid)?,
None => session.clone(),
};
if current.as_ref() == Some(&merged) {
return Ok(false);
}
sqlx::query(
"INSERT INTO furumusic__podcast_listen_session
(user_id, session_id, episode_id, podcast_id, device_id, started_at_ms,
ended_at_ms, start_position_ms, end_position_ms, listened_ms, completed)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (user_id, session_id) DO UPDATE SET
started_at_ms = EXCLUDED.started_at_ms, ended_at_ms = EXCLUDED.ended_at_ms,
start_position_ms = EXCLUDED.start_position_ms,
end_position_ms = EXCLUDED.end_position_ms,
listened_ms = EXCLUDED.listened_ms, completed = EXCLUDED.completed",
)
.bind(user_id)
.bind(&merged.session_id)
.bind(merged.episode_id.as_str())
.bind(merged.podcast_id.as_str())
.bind(&merged.device_id)
.bind(merged.started_at_ms)
.bind(merged.ended_at_ms)
.bind(merged.start_position_ms)
.bind(merged.end_position_ms)
.bind(merged.listened_ms)
.bind(merged.completed)
.execute(&mut *conn)
.await?;
Ok(true)
}
pub(crate) fn episode_type_to_sql(kind: EpisodeType) -> &'static str {
match kind {
EpisodeType::Full => "full",
EpisodeType::Trailer => "trailer",
EpisodeType::Bonus => "bonus",
}
}
/// Inserts an episode row.
pub(crate) async fn insert_episode(
conn: &mut PgConnection,
user_id: i64,
meta: &EpisodeMeta,
source: EpisodeSource,
in_feed: bool,
now: i64,
) -> Result<()> {
sqlx::query(
"INSERT INTO furumusic__podcast_episode
(user_id, episode_id, podcast_id, guid, title, description, season,
episode_number, episode_type, published_at_ms, duration_ms, audio_url, mime,
byte_len, image_url, link, source, in_feed, search_norm, first_seen_at_ms,
updated_at_ms)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17,
$18, $19, $20, $20)",
)
.bind(user_id)
.bind(meta.episode_id.as_str())
.bind(meta.podcast_id.as_str())
.bind(&meta.guid)
.bind(&meta.title)
.bind(&meta.description)
.bind(meta.season.map(i64::from))
.bind(meta.episode_number.map(i64::from))
.bind(episode_type_to_sql(meta.episode_type))
.bind(meta.published_at_ms)
.bind(meta.duration_ms)
.bind(&meta.audio_url)
.bind(&meta.mime)
.bind(meta.byte_len.and_then(|len| i64::try_from(len).ok()))
.bind(&meta.image_url)
.bind(&meta.link)
.bind(source.as_sql())
.bind(in_feed)
.bind(episode_search_norm(
&meta.title,
meta.description.as_deref(),
))
.bind(now)
.execute(&mut *conn)
.await?;
Ok(())
}
/// Episode metadata of a reference learned through sync.
pub(crate) fn meta_from_ref(episode: &PodcastEpisodeRef) -> EpisodeMeta {
EpisodeMeta {
episode_id: episode.episode_id.clone(),
podcast_id: episode.podcast_id.clone(),
guid: None,
title: episode.episode_title.clone(),
description: None,
season: episode.season,
episode_number: episode.episode_number,
episode_type: EpisodeType::default(),
published_at_ms: episode.published_at_ms,
duration_ms: episode.duration_ms,
audio_url: episode.audio_url.clone(),
mime: episode.mime.clone(),
byte_len: None,
image_url: episode.image_url.clone(),
link: None,
}
}
/// Records an episode learned through sync when the catalog does not know it.
///
/// Feed metadata always wins: an existing row is left untouched. The
/// podcast catalog row is created from the reference when missing (no
/// register is written).
pub(crate) async fn ensure_episode_from_ref(
conn: &mut PgConnection,
user_id: i64,
episode: &PodcastEpisodeRef,
) -> Result<()> {
let exists: bool = sqlx::query_scalar(
"SELECT EXISTS (SELECT 1 FROM furumusic__podcast_episode
WHERE user_id = $1 AND episode_id = $2)",
)
.bind(user_id)
.bind(episode.episode_id.as_str())
.fetch_one(&mut *conn)
.await?;
if exists {
return Ok(());
}
let now = now_ms();
ensure_podcast_row(
conn,
user_id,
&episode.podcast_id,
&episode.feed_url,
&episode.podcast_title,
now,
)
.await?;
insert_episode(
conn,
user_id,
&meta_from_ref(episode),
EpisodeSource::Sync,
false,
now,
)
.await
}
/// Known episodes of a podcast with the identity hints they were last seen with.
pub(crate) async fn known_episodes(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<KnownEpisode>> {
let rows = sqlx::query(
"SELECT episode_id, guid, audio_url, link, published_at_ms, title
FROM furumusic__podcast_episode
WHERE user_id = $1 AND podcast_id = $2 ORDER BY episode_id COLLATE \"C\"",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_all(&mut *conn)
.await?;
let known = rows
.iter()
.map(|row| {
Ok(KnownEpisode {
episode_id: episode_id_col(row, "episode_id")?,
hints: EpisodeIdentityHints {
guid: row.try_get("guid")?,
enclosure_url: Some(row.try_get("audio_url")?),
link: row.try_get("link")?,
published_at_ms: row.try_get("published_at_ms")?,
title: Some(row.try_get("title")?),
},
})
})
.collect::<sqlx::Result<Vec<_>>>()?;
Ok(known)
}
/// Applies one podcast operation inside the caller's transaction (which
/// holds the account lock).
pub(crate) async fn apply_op(
conn: &mut PgConnection,
user_id: i64,
op: &SyncOpWire,
) -> Result<ApplyOutcome> {
if !op.payload.is_podcast() {
return Ok(ApplyOutcome::Ignored);
}
if let Err(error) = op.validate_podcast() {
return Ok(ApplyOutcome::Rejected(error.to_string()));
}
let result = match &op.payload {
SyncOpPayload::PodcastSubscriptionSet { .. } => {
let record = op.podcast_subscription().expect("subscription payload");
apply_subscription_record(conn, user_id, &record, now_ms()).await
}
SyncOpPayload::PodcastShareSet { .. } => {
apply_share_record(conn, user_id, &op.podcast_share().expect("share payload")).await
}
SyncOpPayload::PodcastFeedAliasesAdded { .. } => {
let (podcast_id, urls) = op.podcast_feed_aliases().expect("alias payload");
merge_aliases(conn, user_id, podcast_id, urls).await
}
SyncOpPayload::PodcastProgressSet { episode, .. } => {
let record = op.podcast_progress().expect("progress payload");
if let Err(error) = generation_to_sql(record.generation) {
return Ok(ApplyOutcome::Rejected(error.to_string()));
}
if let Some(episode) = episode {
ensure_episode_from_ref(conn, user_id, episode).await?;
}
apply_progress_record(conn, user_id, &record).await
}
SyncOpPayload::PodcastListenRecorded { session, episode } => {
if let Some(current) = load_session(conn, user_id, &session.session_id).await?
&& let Err(error) = merge_listen_session(&current, session)
{
return Ok(ApplyOutcome::Rejected(error.to_string()));
}
if let Some(episode) = episode {
ensure_episode_from_ref(conn, user_id, episode).await?;
}
apply_listen_session(conn, user_id, session).await
}
_ => return Ok(ApplyOutcome::Ignored),
};
match result {
Ok(true) => Ok(ApplyOutcome::Changed),
Ok(false) => Ok(ApplyOutcome::Unchanged),
Err(error) => match error.downcast_ref::<PodcastStoreError>() {
Some(PodcastStoreError::Invalid(reason)) => Ok(ApplyOutcome::Rejected(reason.clone())),
_ => Err(error),
},
}
}
/// Whether an applied operation can change what is published
/// (`super::discovery`): subscription and share registers, and the alias set
/// the card's origin URL comes from.
fn affects_publication(payload: &SyncOpPayload) -> bool {
matches!(
payload,
SyncOpPayload::PodcastSubscriptionSet { .. }
| SyncOpPayload::PodcastShareSet { .. }
| SyncOpPayload::PodcastFeedAliasesAdded { .. }
)
}
fn record_outcome(stats: &mut SnapshotApplyStats, result: Result<bool>) -> Result<()> {
match result {
Ok(true) => stats.changed += 1,
Ok(false) => stats.unchanged += 1,
Err(error) => match error.downcast_ref::<PodcastStoreError>() {
Some(PodcastStoreError::Invalid(_)) => stats.rejected += 1,
_ => return Err(error),
},
}
Ok(())
}
fn check(result: Result<(), PodcastError>) -> Result<()> {
result.map_err(invalid)
}
impl PodcastStore {
/// Applies a device-sync operation received from a trusted device.
///
/// Non-podcast operations return [`ApplyOutcome::Ignored`]. Podcast
/// operations are validated (`SyncOpWire::validate_podcast`) and merged
/// with the contract functions in one transaction; applying the same
/// operations in any order, with duplicates, gives the same state.
pub(crate) async fn apply_remote_op(
&self,
user_id: i64,
op: &SyncOpWire,
) -> Result<ApplyOutcome> {
if !op.payload.is_podcast() {
return Ok(ApplyOutcome::Ignored);
}
let mut tx = self.begin_write(user_id).await?;
let outcome = apply_op(&mut tx, user_id, op).await?;
tx.commit().await?;
if outcome == ApplyOutcome::Changed && affects_publication(&op.payload) {
super::discovery::request_reconcile();
}
Ok(outcome)
}
/// Merges the podcast sections of a sync snapshot.
///
/// Sections exceeding the contract counts are ignored
/// (`SyncSnapshot::podcast_state_within_limits`); every record is
/// validated and merged with the same functions as operations, so a
/// snapshot never replaces newer local state. One transaction.
pub(crate) async fn apply_snapshot(
&self,
user_id: i64,
snapshot: &SyncSnapshot,
) -> Result<SnapshotApplyStats> {
let mut stats = SnapshotApplyStats::default();
if !snapshot.has_podcast_state() {
return Ok(stats);
}
if !snapshot.podcast_state_within_limits() {
stats.ignored_over_limits = true;
return Ok(stats);
}
let mut tx = self.begin_write(user_id).await?;
let now = now_ms();
for record in &snapshot.podcast_subscriptions {
let mut record = record.clone();
record.normalize();
let result = match check(record.validate()) {
Ok(()) => apply_subscription_record(&mut tx, user_id, &record, now).await,
Err(error) => Err(error),
};
record_outcome(&mut stats, result)?;
}
for record in &snapshot.podcast_shares {
let result = match check(record.validate()) {
Ok(()) => apply_share_record(&mut tx, user_id, record).await,
Err(error) => Err(error),
};
record_outcome(&mut stats, result)?;
}
for record in &snapshot.podcast_progress {
let result = match check(record.validate()) {
Ok(()) => apply_progress_record(&mut tx, user_id, record).await,
Err(error) => Err(error),
};
record_outcome(&mut stats, result)?;
}
for session in &snapshot.podcast_listen_sessions {
let result = match check(session.validate()) {
Ok(()) => apply_listen_session(&mut tx, user_id, session).await,
Err(error) => Err(error),
};
record_outcome(&mut stats, result)?;
}
tx.commit().await?;
if stats.changed > 0
&& !(snapshot.podcast_subscriptions.is_empty() && snapshot.podcast_shares.is_empty())
{
super::discovery::request_reconcile();
}
Ok(stats)
}
/// Fills the podcast sections of `snapshot` with this account's
/// registers and sessions, bounded by
/// `SyncSnapshot::bound_podcast_state_to(budget_bytes)`.
///
/// Records are read newest first and capped by the per-section counts
/// before the byte budget is applied. Fetch state and the catalog are
/// never part of the snapshot. Send it only to v4 peers
/// (`SyncSnapshot::for_peer`).
pub(crate) async fn fill_snapshot(
&self,
user_id: i64,
snapshot: &mut SyncSnapshot,
budget_bytes: usize,
) -> Result<()> {
use music_dht::podcast::{
MAX_SNAPSHOT_LISTEN_SESSIONS, MAX_SNAPSHOT_PROGRESS, MAX_SNAPSHOT_SHARES,
MAX_SNAPSHOT_SUBSCRIPTIONS,
};
// One consistent view of every section.
let mut tx = self.pool.begin().await?;
sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
.execute(&mut *tx)
.await?;
let rows = sqlx::query(
"SELECT s.podcast_id, s.subscribed, s.podcast_json, s.generation, s.hlc_ms, s.op_id,
COALESCE((SELECT array_agg(a.feed_url ORDER BY a.feed_url COLLATE \"C\")
FROM furumusic__podcast_feed_alias a
WHERE a.user_id = s.user_id AND a.podcast_id = s.podcast_id),
'{}') AS aliases
FROM furumusic__podcast_subscription s
WHERE s.user_id = $1
ORDER BY s.generation DESC, s.hlc_ms DESC, s.op_id COLLATE \"C\" DESC,
s.podcast_id COLLATE \"C\" DESC
LIMIT $2",
)
.bind(user_id)
.bind(MAX_SNAPSHOT_SUBSCRIPTIONS as i64)
.fetch_all(&mut *tx)
.await?;
let subscriptions = rows
.iter()
.map(|row| {
let aliases: Vec<String> = row.try_get("aliases")?;
Ok(SubscriptionRecord {
podcast_id: super::podcast_id_col(row, "podcast_id")?,
subscribed: row.try_get("subscribed")?,
podcast: podcast_meta_col(row, "podcast_json")?,
feed_aliases: aliases
.into_iter()
.map(FeedUrl::try_from)
.collect::<Result<_, _>>()
.map_err(|error| sqlx::Error::Decode(Box::new(error)))?,
generation: generation_from_sql(row.try_get("generation")?),
hlc_ms: row.try_get("hlc_ms")?,
op_id: row.try_get("op_id")?,
})
})
.collect::<sqlx::Result<Vec<_>>>()?;
let shares = sqlx::query(
"SELECT podcast_id, shared, generation, hlc_ms, op_id FROM furumusic__podcast_share
WHERE user_id = $1
ORDER BY generation DESC, hlc_ms DESC, op_id COLLATE \"C\" DESC,
podcast_id COLLATE \"C\" DESC
LIMIT $2",
)
.bind(user_id)
.bind(MAX_SNAPSHOT_SHARES as i64)
.fetch_all(&mut *tx)
.await?
.iter()
.map(share_from_row)
.collect::<sqlx::Result<Vec<_>>>()?;
let progress = sqlx::query(&format!(
"SELECT {} FROM furumusic__podcast_progress p
WHERE p.user_id = $1
ORDER BY p.generation DESC, p.hlc_ms DESC, p.op_id COLLATE \"C\" DESC,
p.episode_id COLLATE \"C\" DESC
LIMIT $2",
progress_columns("p", "")
))
.bind(user_id)
.bind(MAX_SNAPSHOT_PROGRESS as i64)
.fetch_all(&mut *tx)
.await?
.iter()
.map(|row| progress_from_row(row, ""))
.collect::<sqlx::Result<Vec<_>>>()?;
let sessions = sqlx::query(&format!(
"SELECT {SESSION_COLUMNS} FROM furumusic__podcast_listen_session s
WHERE s.user_id = $1
ORDER BY s.ended_at_ms DESC, s.session_id COLLATE \"C\" DESC
LIMIT $2"
))
.bind(user_id)
.bind(MAX_SNAPSHOT_LISTEN_SESSIONS as i64)
.fetch_all(&mut *tx)
.await?
.iter()
.map(session_from_row)
.collect::<sqlx::Result<Vec<_>>>()?;
tx.commit().await?;
snapshot.podcast_subscriptions = subscriptions;
snapshot.podcast_shares = shares;
snapshot.podcast_progress = progress;
snapshot.podcast_listen_sessions = sessions;
snapshot.bound_podcast_state_to(budget_bytes);
Ok(())
}
/// A snapshot holding only this account's podcast sections, bounded
/// with the default budget (`MAX_PODCAST_SNAPSHOT_BYTES`).
#[cfg(test)]
pub(crate) async fn export_snapshot(&self, user_id: i64) -> Result<SyncSnapshot> {
let mut snapshot = SyncSnapshot::default();
self.fill_snapshot(
user_id,
&mut snapshot,
music_dht::podcast::MAX_PODCAST_SNAPSHOT_BYTES,
)
.await?;
Ok(snapshot)
}
/// Progress register of an episode (also for episodes the catalog does
/// not know yet).
pub(crate) async fn progress(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<ProgressRecord>> {
let mut conn = self.pool.acquire().await?;
load_progress(&mut conn, user_id, episode_id).await
}
}
File diff suppressed because it is too large. Load diff
+800
View File
@@ -0,0 +1,800 @@
//! Public podcast discovery against PostgreSQL: the share toggle and its
//! privacy confirmation, the aggregation of several accounts, the
//! publication from the web's federation node to another loopback node and
//! the first publication after a restart.
//!
//! Ignored by default, like the other store tests: set
//! `FURU_DEVICE_SYNC_TEST_DATABASE_URL` to a disposable database. The
//! loopback test starts two iroh nodes in temporary directories (the same
//! setup as frid's `tests/podcast_dht.rs`).
use std::sync::Arc;
use std::time::Duration;
use music_dht::device_sync::{SyncOpPayload, SyncOpWire, random_hex};
use music_dht::podcast::{FeedUrl, PodcastId, PodcastMeta};
use music_dht::podcast_dht::{PODCAST_DHT_ALPN, PodcastDht};
use music_dht::{MusicDhtConfig, MusicDhtService, NetworkId};
use serde_json::json;
use super::api::{self, ShareOutcome};
use super::discovery::{self, Discovery, aggregate_share_states, load_shared};
use super::postgres_tests::{cleanup, random_user, test_pool};
use super::*;
fn url(value: &str) -> FeedUrl {
FeedUrl::parse(value).unwrap()
}
fn meta(feed: &str, title: &str) -> PodcastMeta {
PodcastMeta {
feed_url: url(feed),
title: title.into(),
author: Some("Host".into()),
description: Some("A public show".into()),
link: None,
image_url: None,
}
}
async fn subscribe(store: &PodcastStore, user: i64, feed: &str, title: &str) -> PodcastId {
store
.subscribe(user, &meta(feed, title))
.await
.unwrap()
.podcast_id
}
fn share_op(
origin: &str,
seq: i64,
podcast_id: &PodcastId,
shared: bool,
generation: u64,
) -> SyncOpWire {
SyncOpWire {
op_id: format!("{origin}:{seq}"),
origin_device_id: origin.into(),
seq,
hlc_ms: 9_000_000_000_000 + seq,
payload: SyncOpPayload::PodcastShareSet {
podcast_id: podcast_id.clone(),
shared,
generation,
},
}
}
async fn share(
store: &PodcastStore,
user: i64,
podcast_id: &PodcastId,
body: serde_json::Value,
) -> Result<ShareOutcome, api::ApiError> {
api::set_share(store, user, podcast_id, body, Discovery::Off, now_ms()).await
}
fn done(outcome: Result<ShareOutcome, api::ApiError>) -> Box<api::ShareResponse> {
match outcome.unwrap() {
ShareOutcome::Done(response) => response,
ShareOutcome::NeedsConfirmation(view) => panic!("unexpected confirmation: {view:?}"),
}
}
fn needs_confirmation(outcome: Result<ShareOutcome, api::ApiError>) -> api::ShareView {
match outcome.unwrap() {
ShareOutcome::NeedsConfirmation(view) => view,
ShareOutcome::Done(response) => panic!("shared without confirmation: {:?}", response.share),
}
}
fn confirm_body(view: &api::ShareView) -> serde_json::Value {
json!({
"shared": true,
"confirm": { "feed_url": view.feed_url, "id_feed_url": view.id_feed_url },
})
}
async fn published_ids(pool: &sqlx::PgPool, users: &[i64]) -> Vec<(PodcastId, i64)> {
let aggregation = aggregate_share_states(
load_shared(pool, Some(users)).await.unwrap().shares,
&Default::default(),
);
aggregation.representatives.into_iter().collect()
}
/// Active accounts (only they publish); random ids like the store tests.
async fn accounts<const N: usize>(pool: &sqlx::PgPool) -> [i64; N] {
let users: [i64; N] = std::array::from_fn(|_| random_user());
for user in users {
sqlx::query(
"INSERT INTO furumusic__user (id, username, role, is_active)
VALUES ($1, $2, 'user', TRUE)",
)
.bind(user)
.bind(format!("podcast-discovery-test-{user}"))
.execute(pool)
.await
.unwrap();
}
users
}
async fn remove_accounts(pool: &sqlx::PgPool, users: &[i64]) {
cleanup(pool, users.iter().copied()).await;
sqlx::query("DELETE FROM furumusic__user WHERE id = ANY($1)")
.bind(users)
.execute(pool)
.await
.unwrap();
}
async fn log_len(pool: &sqlx::PgPool, user: i64) -> i64 {
sqlx::query_scalar("SELECT COUNT(*) FROM furumusic__fed_sync_ops WHERE user_id = $1")
.bind(user)
.fetch_one(pool)
.await
.unwrap()
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_share_toggle_asks_before_publishing_a_private_looking_feed() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let [user, other] = accounts(&pool).await;
let secret = format!(
"https://members.example/feed.xml?token=8f3a9c2e7d1b4a6f{}",
random_hex(4)
);
let podcast_id = subscribe(&store, user, &secret, "Members Only").await;
// Private by default; the dialog needs the warnings and the URL.
let view = api::share_view(&store, user, &podcast_id, Discovery::Off)
.await
.unwrap();
assert!(!view.shared && !view.pending_confirmation && view.can_share);
assert_eq!(view.discovery, "off");
assert!(!view.warnings.is_empty());
assert_eq!(view.feed_url.as_deref(), Some(secret.as_str()));
// Enabling without confirming writes nothing.
let before = log_len(&pool, user).await;
let asked = needs_confirmation(share(&store, user, &podcast_id, json!({"shared": true})).await);
assert!(!asked.warnings.is_empty());
assert!(store.share(user, &podcast_id).await.unwrap().is_none());
// Confirming other URLs than the card's writes nothing either.
let wrong =
json!({"shared": true, "confirm": {"feed_url": "https://members.example/feed.xml"}});
needs_confirmation(share(&store, user, &podcast_id, wrong).await);
assert_eq!(log_len(&pool, user).await, before);
assert!(published_ids(&pool, &[user]).await.is_empty());
// Confirming exactly the shown URLs shares and publishes.
let response = done(share(&store, user, &podcast_id, confirm_body(&asked)).await);
assert!(response.podcast.shared);
assert!(response.share.shared && !response.share.pending_confirmation);
assert_eq!(log_len(&pool, user).await, before + 1);
assert_eq!(
published_ids(&pool, &[user]).await,
vec![(podcast_id.clone(), user)]
);
// Another account cannot see or change this account's podcast.
let error = share(&store, other, &podcast_id, json!({"shared": false}))
.await
.unwrap_err();
assert_eq!(error.status, 404);
// Malformed requests.
let error = share(&store, user, &podcast_id, json!({"share": true}))
.await
.unwrap_err();
assert_eq!(error.status, 400);
// Turning sharing off revokes and forgets the confirmation: sharing
// again asks again.
let response = done(share(&store, user, &podcast_id, json!({"shared": false})).await);
assert!(!response.share.shared);
assert!(published_ids(&pool, &[user]).await.is_empty());
let confirmations: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM furumusic__podcast_share_confirmation WHERE user_id = $1",
)
.bind(user)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(confirmations, 0);
needs_confirmation(share(&store, user, &podcast_id, json!({"shared": true})).await);
// Shared on another trusted device (which asked its own user): the web
// still asks this account before its server publishes the URL.
let remote = share_op("tui_a", 1, &podcast_id, true, 100);
assert_eq!(
store.apply_remote_op(user, &remote).await.unwrap(),
ApplyOutcome::Changed
);
let view = api::share_view(&store, user, &podcast_id, Discovery::Off)
.await
.unwrap();
assert!(view.shared && view.pending_confirmation);
assert!(published_ids(&pool, &[user]).await.is_empty());
let before = log_len(&pool, user).await;
let response = done(share(&store, user, &podcast_id, confirm_body(&view)).await);
assert!(response.share.shared && !response.share.pending_confirmation);
// Confirming an existing decision emits no operation.
assert_eq!(log_len(&pool, user).await, before);
assert_eq!(published_ids(&pool, &[user]).await.len(), 1);
// A feed that moves to another URL that looks private asks again.
let moved = format!(
"https://members.example/v2/feed.xml?key=9a8b7c6d5e4f3a2b{}",
random_hex(4)
);
sqlx::query(
"UPDATE furumusic__podcast SET feed_url = $3 WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user)
.bind(podcast_id.as_str())
.bind(&moved)
.execute(&pool)
.await
.unwrap();
let view = api::share_view(&store, user, &podcast_id, Discovery::Off)
.await
.unwrap();
assert!(view.shared && view.pending_confirmation);
assert_eq!(view.feed_url.as_deref(), Some(moved.as_str()));
assert_eq!(view.id_feed_url.as_deref(), Some(secret.as_str()));
assert!(published_ids(&pool, &[user]).await.is_empty());
// Unsubscribing revokes and forgets; the podcast is private again.
api::unsubscribe(&store, user, &podcast_id).await.unwrap();
let view = api::share_view(&store, user, &podcast_id, Discovery::Off)
.await
.unwrap();
assert!(!view.shared && !view.can_share);
let error = share(&store, user, &podcast_id, json!({"shared": true}))
.await
.unwrap_err();
assert_eq!(error.status, 409);
remove_accounts(&pool, &[user, other]).await;
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_shared_podcasts_aggregate_across_accounts() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let users = accounts::<3>(&pool).await;
let [a, b, c] = users;
let feed = format!("https://podcasts.example/shared-{}.xml", random_hex(4));
let private = format!("https://podcasts.example/private-{}.xml", random_hex(4));
let podcast_id = subscribe(&store, a, &feed, "Shared Show").await;
assert_eq!(subscribe(&store, b, &feed, "Shared Show").await, podcast_id);
assert_eq!(subscribe(&store, c, &feed, "Shared Show").await, podcast_id);
let private_id = subscribe(&store, a, &private, "Private Show").await;
// Nothing shared: nothing published, not even the private podcast.
assert!(published_ids(&pool, &users).await.is_empty());
done(share(&store, a, &podcast_id, json!({"shared": true})).await);
done(share(&store, b, &podcast_id, json!({"shared": true})).await);
let published = published_ids(&pool, &users).await;
assert_eq!(published, vec![(podcast_id.clone(), a.min(b))]);
let aggregation = aggregate_share_states(
load_shared(&pool, Some(&users)).await.unwrap().shares,
&Default::default(),
);
let state = &aggregation.states[0];
assert!(state.is_shared() && state.privacy_confirmed);
let card = state.card.as_ref().unwrap();
assert_eq!(card.feed_url, url(&feed));
assert_eq!(card.title, "Shared Show");
// A deactivated account publishes nothing; the other one still shares.
let set_active = async |user: i64, active: bool| {
sqlx::query("UPDATE furumusic__user SET is_active = $2 WHERE id = $1")
.bind(user)
.bind(active)
.execute(&pool)
.await
.unwrap();
};
set_active(a.min(b), false).await;
assert_eq!(
published_ids(&pool, &users).await,
vec![(podcast_id.clone(), a.max(b))]
);
set_active(a.max(b), false).await;
assert!(published_ids(&pool, &users).await.is_empty());
set_active(a, true).await;
set_active(b, true).await;
// One account stops sharing: the other still shares the same card.
done(share(&store, a, &podcast_id, json!({"shared": false})).await);
assert_eq!(
published_ids(&pool, &users).await,
vec![(podcast_id.clone(), b)]
);
// A stale `shared = true` of A (delivered late from an old device log)
// cannot republish for A.
let stale = share_op("old_device", 1, &podcast_id, true, 1);
assert_eq!(
store.apply_remote_op(a, &stale).await.unwrap(),
ApplyOutcome::Unchanged
);
let view = api::share_view(&store, a, &podcast_id, Discovery::Off)
.await
.unwrap();
assert!(!view.shared);
// The last sharing account unsubscribes: the card is revoked, and
// subscribing again is private.
api::unsubscribe(&store, b, &podcast_id).await.unwrap();
assert!(published_ids(&pool, &users).await.is_empty());
api::resubscribe(&store, b, &podcast_id).await.unwrap();
assert!(published_ids(&pool, &users).await.is_empty());
// Share decisions of C never existed; the private podcast of A stays
// private throughout.
let view = api::share_view(&store, a, &private_id, Discovery::Off)
.await
.unwrap();
assert!(!view.shared);
remove_accounts(&pool, &users).await;
}
// ---------------------------------------------------------------------------
// Loopback: publish from the web node, find from another node
// ---------------------------------------------------------------------------
struct Node {
service: Arc<MusicDhtService>,
dht: Arc<PodcastDht>,
tasks: Vec<tokio::task::JoinHandle<()>>,
}
impl Node {
async fn stop(self) {
for task in &self.tasks {
task.abort();
}
self.service.shutdown().await.unwrap();
}
}
async fn start_node(directory: &std::path::Path, network: NetworkId) -> Node {
std::fs::create_dir_all(directory).unwrap();
let config = MusicDhtConfig::builder()
.data_dir(directory)
.network_id(network)
.request_timeout(Duration::from_secs(2))
.lookup_timeout(Duration::from_secs(5))
.transport_timeout(Duration::from_secs(5))
.dial_timeout(Duration::from_secs(2))
.schema_independent_stream_protocol(PODCAST_DHT_ALPN)
.build()
.unwrap();
let (service, mut events) = MusicDhtService::start(config).await.unwrap();
let service = Arc::new(service);
let events = tokio::spawn(async move { while events.recv().await.is_some() {} });
// The web's own wiring: the overlay on the node's endpoint, its cache
// in the federation data directory.
let (dht, serve) = crate::federation::podcast_discovery::open(&service, directory)
.await
.expect("overlay opens");
assert!(
directory
.join(crate::federation::podcast_discovery::CACHE_FILE)
.exists()
);
Node {
service,
dht,
tasks: vec![events, serve],
}
}
async fn wait_for<T>(what: &str, mut check: impl AsyncFnMut() -> Option<T>) -> T {
let started = std::time::Instant::now();
loop {
if let Some(value) = check().await {
return value;
}
assert!(
started.elapsed() < Duration::from_secs(90),
"timed out waiting for {what}"
);
tokio::time::sleep(Duration::from_millis(300)).await;
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_web_node_publishes_shared_podcasts_to_another_node() {
tokio::time::timeout(Duration::from_secs(240), loopback_scenario())
.await
.unwrap();
}
async fn loopback_scenario() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let users = accounts::<2>(&pool).await;
let [u1, u2] = users;
// An account of the stranger's side (another server in real life) that
// subscribes from the card it found.
let finder = random_user();
let tag = random_hex(6);
let directory = std::env::temp_dir().join(format!("furumusic-podcast-dht-{tag}"));
let network = NetworkId::from_name(&format!("furumusic-podcast-loopback-{tag}"));
let web = start_node(&directory.join("web"), network).await;
let stranger = start_node(&directory.join("stranger"), network).await;
// The stranger reaches the web node through the music-DHT transport
// only: no device-sync pairing, another identity.
meet(&web, &stranger).await;
// Two accounts share one podcast; one of them also has a private one.
let title = format!("Loopback Radio {tag}");
let hidden = format!("Hidden Garden {tag}");
let server = super::test_server::TestServer::start();
server.set(
"/loopback.xml",
super::test_server::Route::new(
format!(
r#"<?xml version="1.0"?><rss version="2.0"><channel><title>{title}</title>
<item><title>First</title><guid>1</guid><pubDate>Mon, 01 Jan 2024 10:00:00 GMT</pubDate>
<enclosure url="/audio/1.mp3" type="audio/mpeg"/></item></channel></rss>"#
),
"application/rss+xml",
),
);
let feed = server.url("/loopback.xml");
let podcast_id = subscribe(&store, u1, &feed, &title).await;
subscribe(&store, u2, &feed, &title).await;
subscribe(
&store,
u1,
&format!("https://podcasts.example/hidden-{tag}.xml"),
&hidden,
)
.await;
// Sharing while the stranger already looks: nothing before.
assert!(stranger.dht.search(&title, 10).await.unwrap().is_empty());
done(share(&store, u1, &podcast_id, json!({"shared": true})).await);
done(share(&store, u2, &podcast_id, json!({"shared": true})).await);
let report = discovery::reconcile_once(&pool, &web.dht, Some(&users))
.await
.unwrap();
assert_eq!(report.aggregation.states.len(), 1, "one card per podcast");
assert_eq!(report.stats.published, 1);
assert!(report.stats.held_back.is_empty() && report.stats.failed.is_empty());
let view = api::share_view(&store, u1, &podcast_id, Discovery::Running(&web.dht))
.await
.unwrap();
assert!(view.published && view.held.is_none());
assert_eq!(view.discovery, "running");
let hit = wait_for("the shared card", async || {
stranger
.dht
.search(&title, 10)
.await
.unwrap()
.into_iter()
.find(|hit| hit.podcast_id == podcast_id)
})
.await;
assert_eq!(hit.feed_url, url(&feed));
assert_eq!(hit.card.title, title);
assert_eq!(hit.publishers, vec![web.service.endpoint_id()]);
// The private podcast is not discoverable, by text or by id.
assert!(stranger.dht.search(&hidden, 10).await.unwrap().is_empty());
// The finder sees the card in the federated search, looks at its
// episodes and subscribes: its own private subscription with the
// card's id, fetched from the feed (never from the publisher).
let views = super::search::hit_views(&store, finder, &title, &[], vec![hit.clone()])
.await
.unwrap();
assert_eq!(views.len(), 1);
assert!(!views[0].subscribed && views[0].exact_match);
let found = json!({ "podcast_id": views[0].podcast_id, "feed_url": views[0].feed_url });
let fetcher = super::postgres_tests::local_fetcher();
let preview = api::preview_found(&store, &fetcher, finder, found.clone())
.await
.unwrap();
assert_eq!(preview.episodes.len(), 1);
let subscribed = api::subscribe_found(&store, &fetcher, finder, found, now_ms())
.await
.unwrap();
assert_eq!(subscribed.podcast.podcast_id, podcast_id.to_string());
assert!(subscribed.podcast.subscribed && !subscribed.podcast.shared);
// Idempotent; one account stopping keeps the card of the other.
done(share(&store, u1, &podcast_id, json!({"shared": false})).await);
let report = discovery::reconcile_once(&pool, &web.dht, Some(&users))
.await
.unwrap();
assert_eq!(
(
report.stats.published,
report.stats.unchanged,
report.stats.unpublished
),
(0, 1, 0)
);
// The last account unsubscribes: the card is revoked with a tombstone.
api::unsubscribe(&store, u2, &podcast_id).await.unwrap();
let report = discovery::reconcile_once(&pool, &web.dht, Some(&users))
.await
.unwrap();
assert_eq!(report.stats.unpublished, 1);
wait_for("the revocation", async || {
stranger
.dht
.lookup_podcast(&podcast_id)
.await
.unwrap()
.is_none()
.then_some(())
})
.await;
assert!(
stranger
.dht
.search(&title, 10)
.await
.unwrap()
.iter()
.all(|hit| hit.podcast_id != podcast_id)
);
let view = api::share_view(&store, u2, &podcast_id, Discovery::Running(&web.dht))
.await
.unwrap();
assert!(!view.shared && !view.published);
// The revocation does not touch the finder's subscription.
let kept = store.podcast(finder, &podcast_id).await.unwrap().unwrap();
assert!(kept.subscribed && !kept.shared);
cleanup(&pool, [finder]).await;
web.stop().await;
stranger.stop().await;
remove_accounts(&pool, &users).await;
let _ = std::fs::remove_dir_all(&directory);
}
/// Connects `stranger` to `web` and fills both overlay routing tables.
async fn meet(web: &Node, stranger: &Node) {
stranger
.service
.connect(web.service.ticket().await.unwrap())
.await
.unwrap();
wait_for("music DHT contacts", async || {
(!web.service.known_peers().is_empty() && !stranger.service.known_peers().is_empty())
.then_some(())
})
.await;
web.dht.bootstrap().await.unwrap();
stranger.dht.bootstrap().await.unwrap();
wait_for("overlay contacts", async || {
(web.dht.known_peers() >= 1 && stranger.dht.known_peers() >= 1).then_some(())
})
.await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_a_restarted_server_publishes_only_after_the_first_device_sync() {
tokio::time::timeout(Duration::from_secs(240), restart_scenario())
.await
.unwrap();
}
/// Interop scenario 7c with the web server as the device that was stopped:
/// a TUI of the account turned sharing off meanwhile, and the server comes
/// back with the old "shared" rows.
async fn restart_scenario() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let users = accounts::<1>(&pool).await;
let [user] = users;
let tag = random_hex(6);
let directory = std::env::temp_dir().join(format!("furumusic-podcast-restart-{tag}"));
let network = NetworkId::from_name(&format!("furumusic-podcast-restart-{tag}"));
let kept_title = format!("Kept Show {tag}");
let revoked_title = format!("Revoked Show {tag}");
let kept = subscribe(
&store,
user,
&format!("https://podcasts.example/kept-{tag}.xml"),
&kept_title,
)
.await;
let revoked = subscribe(
&store,
user,
&format!("https://podcasts.example/revoked-{tag}.xml"),
&revoked_title,
)
.await;
done(share(&store, user, &kept, json!({"shared": true})).await);
done(share(&store, user, &revoked, json!({"shared": true})).await);
let web = start_node(&directory.join("web"), network).await;
let report = discovery::reconcile_once(&pool, &web.dht, Some(&users))
.await
.unwrap();
assert_eq!(report.stats.published, 2);
web.stop().await;
// The server restarts from the same directory (same key, same overlay
// cache) and meets a stranger before its first device-sync round.
let web = start_node(&directory.join("web"), network).await;
let stranger = start_node(&directory.join("stranger"), network).await;
meet(&web, &stranger).await;
let (first_sync_done, first_sync) = tokio::sync::watch::channel(false);
let reconciler = tokio::spawn(discovery::run_rounds(
pool.clone(),
Arc::clone(&web.dht),
first_sync,
Duration::from_secs(600),
Some(users.to_vec()),
));
// Time for a round: nothing is announced from the stored decisions.
tokio::time::sleep(Duration::from_secs(5)).await;
for podcast_id in [&kept, &revoked] {
assert!(
stranger
.dht
.lookup_podcast(podcast_id)
.await
.unwrap()
.is_none(),
"a card was announced before the first device-sync round"
);
}
assert!(
stranger
.dht
.search(&revoked_title, 10)
.await
.unwrap()
.is_empty()
);
// The first device-sync round brings the share-off the TUI made while
// the server was stopped.
let share_off = share_op("tui_while_stopped", 1, &revoked, false, 100);
assert_eq!(
store.apply_remote_op(user, &share_off).await.unwrap(),
ApplyOutcome::Changed
);
first_sync_done.send_replace(true);
let hit = wait_for("the card still shared", async || {
stranger.dht.lookup_podcast(&kept).await.unwrap()
})
.await;
assert_eq!(hit.card.title, kept_title);
assert_eq!(hit.publishers, vec![web.service.endpoint_id()]);
assert!(
stranger
.dht
.lookup_podcast(&revoked)
.await
.unwrap()
.is_none()
);
assert!(
stranger
.dht
.search(&revoked_title, 10)
.await
.unwrap()
.is_empty()
);
let revoked_record = web
.dht
.published()
.await
.unwrap()
.into_iter()
.find(|publication| publication.podcast_id == revoked)
.unwrap();
assert!(!revoked_record.active, "the revoked card is tombstoned");
reconciler.abort();
web.stop().await;
stranger.stop().await;
remove_accounts(&pool, &users).await;
let _ = std::fs::remove_dir_all(&directory);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_an_unreadable_row_does_not_stop_other_revocations() {
tokio::time::timeout(Duration::from_secs(120), unreadable_row_scenario())
.await
.unwrap();
}
async fn unreadable_row_scenario() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let users = accounts::<2>(&pool).await;
let [a, b] = users;
let tag = random_hex(6);
let directory = std::env::temp_dir().join(format!("furumusic-podcast-unreadable-{tag}"));
let network = NetworkId::from_name(&format!("furumusic-podcast-unreadable-{tag}"));
let web = start_node(&directory.join("web"), network).await;
let a_feed = format!("https://podcasts.example/a-{tag}.xml");
let b_feed = format!("https://podcasts.example/b-{tag}.xml");
let a_id = subscribe(&store, a, &a_feed, "A Show").await;
let b_id = subscribe(&store, b, &b_feed, "B Show").await;
done(share(&store, a, &a_id, json!({"shared": true})).await);
done(share(&store, b, &b_id, json!({"shared": true})).await);
let report = discovery::reconcile_once(&pool, &web.dht, Some(&users))
.await
.unwrap();
assert_eq!(report.stats.published, 2);
let active = async || {
web.dht
.published()
.await
.unwrap()
.into_iter()
.filter(|publication| publication.active)
.map(|publication| publication.podcast_id)
.collect::<Vec<_>>()
};
assert_eq!(active().await.len(), 2);
// B's stored alias no longer decodes (an imported row, a stricter
// parser after an upgrade); A turns sharing off.
sqlx::query(
"INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url)
VALUES ($1, $2, 'not a feed url')",
)
.bind(b)
.bind(b_id.as_str())
.execute(&pool)
.await
.unwrap();
done(share(&store, a, &a_id, json!({"shared": false})).await);
// The round still runs: A's card is revoked, and B's card (which can no
// longer be read) fails closed instead of being republished forever.
let report = discovery::reconcile_once(&pool, &web.dht, Some(&users))
.await
.unwrap();
assert_eq!(report.aggregation.unreadable, 1);
assert!(report.aggregation.states.is_empty());
assert_eq!(report.stats.unpublished, 2);
assert!(active().await.is_empty());
// Repaired: B's card is published again; A's stays revoked.
sqlx::query(
"DELETE FROM furumusic__podcast_feed_alias WHERE user_id = $1 AND feed_url = 'not a feed url'",
)
.bind(b)
.execute(&pool)
.await
.unwrap();
let report = discovery::reconcile_once(&pool, &web.dht, Some(&users))
.await
.unwrap();
assert_eq!(report.aggregation.unreadable, 0);
assert_eq!(active().await, vec![b_id.clone()]);
web.stop().await;
remove_accounts(&pool, &users).await;
let _ = std::fs::remove_dir_all(&directory);
}
+322
View File
@@ -0,0 +1,322 @@
//! Conditional, size- and time-bounded feed downloads through [`SafeHttp`].
use std::fmt;
use std::time::Duration;
use reqwest::header::{
ACCEPT, ACCEPT_ENCODING, ETAG, HeaderMap, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH,
LAST_MODIFIED, RETRY_AFTER,
};
use reqwest::{StatusCode, Url};
use music_dht::podcast::FeedUrl;
use music_dht::podcast::feed::{ParsedFeed, parse_feed_with_base};
use super::net::{GuardError, REFUSED_MESSAGE, SafeHttp, SafeHttpConfig};
/// Hard cap on a feed document (large catalogs are a few MiB).
pub(crate) const MAX_FEED_BYTES: usize = 32 * 1024 * 1024;
/// Whole fetch, from connect to the last body byte.
pub(crate) const FEED_FETCH_TIMEOUT: Duration = Duration::from_secs(60);
/// Redirect hops followed for a feed.
pub(crate) const MAX_FEED_REDIRECTS: usize = 5;
/// `Retry-After` values above this are clamped.
const MAX_RETRY_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
const FEED_ACCEPT: &str = "application/rss+xml, application/atom+xml, application/xml;q=0.9, \
text/xml;q=0.9, */*;q=0.5";
/// HTTP client settings of the feed fetcher.
pub(crate) const FEED_HTTP: SafeHttpConfig = SafeHttpConfig {
connect_timeout: Duration::from_secs(10),
read_timeout: Duration::from_secs(30),
total_timeout: Some(FEED_FETCH_TIMEOUT),
max_redirects: MAX_FEED_REDIRECTS,
};
/// Validators of the previous successful fetch.
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
pub(crate) struct Validators {
pub(crate) etag: Option<String>,
pub(crate) last_modified: Option<String>,
}
/// A successfully fetched and parsed document.
#[derive(Debug, Clone)]
pub(crate) struct FetchedFeed {
pub(crate) feed: ParsedFeed,
/// Where the feed now lives: the final URL after a chain of permanent
/// redirects, otherwise the requested URL.
pub(crate) fetched_from: FeedUrl,
/// Whether the feed moved: at least one redirect, all permanent (see
/// [`super::net::Followed::moved_permanently`]), to a valid feed URL.
pub(crate) moved_permanently: bool,
/// New validators for the next conditional request.
pub(crate) validators: Validators,
}
/// Outcome of [`FeedFetcher::fetch`].
#[derive(Debug, Clone)]
pub(crate) enum FeedFetch {
/// `304 Not Modified`: the stored catalog is current.
NotModified,
/// A new document.
Fetched(Box<FetchedFeed>),
}
/// Kind of a failed fetch.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum FetchErrorKind {
/// Refused by the SSRF guard (blocked address or port, scheme,
/// credentials).
Refused,
/// Non-success HTTP status.
Status(u16),
/// Body over [`MAX_FEED_BYTES`].
TooLarge,
/// The document is not a feed.
Parse,
/// Timed out.
Timeout,
/// DNS, connect, TLS or a dropped connection.
Transport,
}
/// A failed fetch. The stored catalog is never touched on failure.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct FetchError {
pub(crate) kind: FetchErrorKind,
/// Details for the server logs (no URLs; see [`Self::public_message`]
/// for what users see).
pub(crate) message: String,
/// Server-requested delay (`Retry-After` on 429/503), clamped to a day.
pub(crate) retry_after: Option<Duration>,
}
impl FetchError {
fn new(kind: FetchErrorKind, message: impl Into<String>) -> Self {
Self {
kind,
message: message.into(),
retry_after: None,
}
}
/// What the signed-in user is told (API answers, the stored last
/// error): the kind of failure, never resolved addresses or transport
/// details, which would let users probe the server's network.
pub(crate) fn public_message(&self) -> String {
match &self.kind {
FetchErrorKind::Refused => REFUSED_MESSAGE.to_string(),
FetchErrorKind::Status(status) => format!("the server answered HTTP {status}"),
FetchErrorKind::TooLarge => format!(
"the feed is larger than {} MiB",
MAX_FEED_BYTES / (1024 * 1024)
),
FetchErrorKind::Parse => "the document is not a podcast feed".to_string(),
FetchErrorKind::Timeout => "the request timed out".to_string(),
FetchErrorKind::Transport => "could not connect to the server".to_string(),
}
}
}
impl fmt::Display for FetchError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for FetchError {}
impl From<GuardError> for FetchError {
fn from(error: GuardError) -> Self {
let kind = if error.is_refusal() {
FetchErrorKind::Refused
} else if error.is_timeout() {
FetchErrorKind::Timeout
} else {
FetchErrorKind::Transport
};
Self::new(kind, error.to_string())
}
}
/// Parses `Retry-After` (delta seconds or an HTTP date).
pub(crate) fn parse_retry_after(
value: &str,
now: chrono::DateTime<chrono::Utc>,
) -> Option<Duration> {
let value = value.trim();
let delay = if let Ok(seconds) = value.parse::<u64>() {
Duration::from_secs(seconds)
} else {
let at = chrono::DateTime::parse_from_rfc2822(value).ok()?;
(at.with_timezone(&chrono::Utc) - now)
.to_std()
.unwrap_or_default()
};
Some(delay.min(MAX_RETRY_AFTER))
}
/// Downloads feeds with the SSRF guard, conditional requests, redirects
/// and hard size/time limits; parses them with the shared parser.
#[derive(Debug, Clone)]
pub(crate) struct FeedFetcher {
http: SafeHttp,
max_bytes: usize,
}
impl FeedFetcher {
/// The production fetcher (public addresses only).
pub(crate) fn new() -> anyhow::Result<Self> {
Ok(Self::with_http(SafeHttp::public(FEED_HTTP)?))
}
pub(crate) fn with_http(http: SafeHttp) -> Self {
Self {
http,
max_bytes: MAX_FEED_BYTES,
}
}
#[cfg(test)]
pub(crate) fn with_max_bytes(mut self, max_bytes: usize) -> Self {
self.max_bytes = max_bytes;
self
}
/// Fetches `url`, conditionally when `validators` are known.
pub(crate) async fn fetch(
&self,
url: &FeedUrl,
validators: &Validators,
) -> Result<FeedFetch, FetchError> {
match tokio::time::timeout(FEED_FETCH_TIMEOUT, self.fetch_inner(url, validators)).await {
Ok(result) => result,
Err(_) => Err(FetchError::new(
FetchErrorKind::Timeout,
format!(
"feed fetch timed out after {} s",
FEED_FETCH_TIMEOUT.as_secs()
),
)),
}
}
async fn fetch_inner(
&self,
feed_url: &FeedUrl,
validators: &Validators,
) -> Result<FeedFetch, FetchError> {
let url = Url::parse(feed_url.as_str())
.map_err(|error| FetchError::new(FetchErrorKind::Refused, error.to_string()))?;
let mut headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static(FEED_ACCEPT));
// Compressed bodies are not negotiated (no decoder is enabled), so
// the size cap applies to what is parsed.
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
if let Some(etag) = validators
.etag
.as_deref()
.and_then(|value| HeaderValue::from_str(value).ok())
{
headers.insert(IF_NONE_MATCH, etag);
}
if let Some(date) = validators
.last_modified
.as_deref()
.and_then(|value| HeaderValue::from_str(value).ok())
{
headers.insert(IF_MODIFIED_SINCE, date);
}
let followed = self.http.get(url, &headers).await?;
let mut moved_permanently = followed.moved_permanently();
let final_url = followed.final_url;
let response = followed.response;
let status = response.status();
if status == StatusCode::NOT_MODIFIED {
return Ok(FeedFetch::NotModified);
}
if !status.is_success() {
let mut error = FetchError::new(
FetchErrorKind::Status(status.as_u16()),
format!("feed request failed: HTTP {status}"),
);
if matches!(status.as_u16(), 429 | 503) {
error.retry_after = response
.headers()
.get(RETRY_AFTER)
.and_then(|value| value.to_str().ok())
.and_then(|value| parse_retry_after(value, chrono::Utc::now()));
}
return Err(error);
}
let header = |name| {
response
.headers()
.get(name)
.and_then(|value: &HeaderValue| value.to_str().ok())
.map(str::to_string)
};
let validators = Validators {
etag: header(ETAG),
last_modified: header(LAST_MODIFIED),
};
let body = read_capped(response, self.max_bytes).await?;
// Relative enclosure URLs resolve against where the feed really is.
let base = final_url.to_string();
let feed = tokio::task::spawn_blocking(move || parse_feed_with_base(&body, Some(&base)))
.await
.map_err(|error| {
FetchError::new(
FetchErrorKind::Parse,
format!("feed parser task failed: {error}"),
)
})?
.map_err(|error| {
FetchError::new(
FetchErrorKind::Parse,
format!("not a podcast feed: {error}"),
)
})?;
// Only a permanent move changes the feed URL; a target that is not a
// valid feed URL (over-long, ...) is not learned.
let moved_to = moved_permanently
.then(|| FeedUrl::parse(final_url.as_str()).ok())
.flatten();
moved_permanently = moved_to.is_some();
let fetched_from = moved_to.unwrap_or_else(|| feed_url.clone());
Ok(FeedFetch::Fetched(Box::new(FetchedFeed {
feed,
fetched_from,
moved_permanently,
validators,
})))
}
}
async fn read_capped(mut response: reqwest::Response, cap: usize) -> Result<Vec<u8>, FetchError> {
let too_large = |detail: String| {
FetchError::new(
FetchErrorKind::TooLarge,
format!("feed is too large ({detail}, limit {cap} bytes)"),
)
};
if let Some(len) = response.content_length()
&& len > cap as u64
{
return Err(too_large(format!("{len} bytes")));
}
let mut body = Vec::new();
while let Some(chunk) = response
.chunk()
.await
.map_err(|error| FetchError::from(GuardError::from(error)))?
{
if body.len() + chunk.len() > cap {
return Err(too_large("body exceeds the limit".to_string()));
}
body.extend_from_slice(&chunk);
}
Ok(body)
}
+289
View File
@@ -0,0 +1,289 @@
//! Feed fetching (conditional requests, redirects, limits) and the refresh
//! schedule, against the local test server.
use std::time::Duration;
use music_dht::podcast::{FeedUrl, PodcastId};
use super::fetch::{FeedFetch, FeedFetcher, FetchErrorKind, Validators, parse_retry_after};
use super::runner::{
BACKOFF_BASE_MS, BACKOFF_MAX_MS, REFRESH_INTERVAL_MS, failure_backoff_ms, next_regular_check_ms,
};
use std::sync::Arc;
use super::test_server::{Route, ScriptedLookup, TestServer, test_http};
// ---------------------------------------------------------------------------
// Feed fetching
// ---------------------------------------------------------------------------
fn rss(items: &str) -> String {
format!(
r#"<?xml version="1.0"?><rss version="2.0"><channel><title>Show</title>{items}</channel></rss>"#
)
}
fn item(guid: &str) -> String {
format!(
r#"<item><title>{guid}</title><guid>{guid}</guid><enclosure url="/audio/{guid}.mp3" type="audio/mpeg"/></item>"#
)
}
fn fetcher(lookup: Arc<ScriptedLookup>) -> FeedFetcher {
FeedFetcher::with_http(test_http(lookup))
}
fn feed_url(text: &str) -> FeedUrl {
FeedUrl::parse(text).unwrap()
}
#[tokio::test]
async fn feeds_are_fetched_conditionally() {
let server = TestServer::start();
server.set(
"/feed.xml",
Route::new(rss(&item("a")), "application/rss+xml")
.etag("\"v1\"")
.last_modified("Wed, 21 Oct 2026 07:28:00 GMT"),
);
let fetcher = fetcher(ScriptedLookup::with(&[]));
let feed = feed_url(&server.url("/feed.xml"));
let FeedFetch::Fetched(first) = fetcher.fetch(&feed, &Validators::default()).await.unwrap()
else {
panic!("expected a document");
};
assert_eq!(first.feed.episodes.len(), 1);
assert_eq!(
first.feed.episodes[0].audio_url,
server.url("/audio/a.mp3"),
"relative enclosures resolve against the feed URL"
);
assert_eq!(first.fetched_from, feed);
assert!(!first.moved_permanently);
assert_eq!(first.validators.etag.as_deref(), Some("\"v1\""));
let second = fetcher.fetch(&feed, &first.validators).await.unwrap();
assert!(matches!(second, FeedFetch::NotModified));
// Only Last-Modified known.
let by_date = Validators {
etag: None,
last_modified: first.validators.last_modified.clone(),
};
assert!(matches!(
fetcher.fetch(&feed, &by_date).await.unwrap(),
FeedFetch::NotModified
));
let requests = server.requests("/feed.xml");
assert_eq!(requests.len(), 3);
assert!(!requests[0].contains_key("if-none-match"));
assert_eq!(requests[1]["if-none-match"], "\"v1\"");
assert_eq!(
requests[1]["if-modified-since"],
"Wed, 21 Oct 2026 07:28:00 GMT"
);
assert!(!requests[2].contains_key("if-none-match"));
assert_eq!(requests[2]["accept-encoding"], "identity");
}
#[tokio::test]
async fn only_permanent_redirect_chains_move_a_feed() {
let server = TestServer::start();
server.set(
"/new.xml",
Route::new(rss(&item("a")), "application/rss+xml"),
);
server.set("/old.xml", Route::redirect(301, "/new.xml"));
server.set("/moved.xml", Route::redirect(308, "/old.xml"));
server.set("/temp.xml", Route::redirect(302, "/new.xml"));
server.set("/mixed.xml", Route::redirect(307, "/old.xml"));
let fetcher = fetcher(ScriptedLookup::with(&[]));
for (path, moved) in [
("/old.xml", true),
("/moved.xml", true),
("/temp.xml", false),
("/mixed.xml", false),
] {
let requested = feed_url(&server.url(path));
let FeedFetch::Fetched(document) = fetcher
.fetch(&requested, &Validators::default())
.await
.unwrap()
else {
panic!("{path}");
};
assert_eq!(document.moved_permanently, moved, "{path}");
let expected = if moved {
feed_url(&server.url("/new.xml"))
} else {
requested
};
assert_eq!(document.fetched_from, expected, "{path}");
assert_eq!(
document.feed.episodes[0].audio_url,
server.url("/audio/a.mp3")
);
}
}
#[tokio::test]
async fn feed_fetch_failures_are_typed() {
let server = TestServer::start();
let big = rss(&item("x").repeat(200));
server.set("/big.xml", Route::new(big.clone(), "application/rss+xml"));
server.set(
"/big-chunked.xml",
Route::new(big.clone(), "application/rss+xml").no_length(),
);
server.set(
"/page.html",
Route::new("<html><body>hi</body></html>", "text/html"),
);
server.set(
"/busy.xml",
Route::new("busy", "text/plain")
.status(503)
.header("Retry-After", "120"),
);
server.set("/gone.xml", Route::new("", "text/plain").status(410));
server.set(
"/private.xml",
Route::redirect(301, "http://192.168.0.1/feed"),
);
let fetcher = fetcher(ScriptedLookup::with(&[])).with_max_bytes(big.len() - 1);
let fetch = |path: &str| {
let feed = feed_url(&server.url(path));
let fetcher = fetcher.clone();
async move {
fetcher
.fetch(&feed, &Validators::default())
.await
.unwrap_err()
}
};
assert_eq!(fetch("/big.xml").await.kind, FetchErrorKind::TooLarge);
assert_eq!(
fetch("/big-chunked.xml").await.kind,
FetchErrorKind::TooLarge
);
assert_eq!(fetch("/page.html").await.kind, FetchErrorKind::Parse);
let busy = fetch("/busy.xml").await;
assert_eq!(busy.kind, FetchErrorKind::Status(503));
assert_eq!(busy.retry_after, Some(Duration::from_secs(120)));
let gone = fetch("/gone.xml").await;
assert_eq!(gone.kind, FetchErrorKind::Status(410));
assert_eq!(gone.retry_after, None);
assert_eq!(fetch("/private.xml").await.kind, FetchErrorKind::Refused);
assert_eq!(
fetch("/missing.xml").await.kind,
FetchErrorKind::Status(404)
);
}
#[test]
fn retry_after_accepts_seconds_and_dates() {
let now = chrono::DateTime::parse_from_rfc3339("2026-11-19T10:00:00Z")
.unwrap()
.with_timezone(&chrono::Utc);
assert_eq!(parse_retry_after("90", now), Some(Duration::from_secs(90)));
assert_eq!(
parse_retry_after("Thu, 19 Nov 2026 10:05:00 GMT", now),
Some(Duration::from_secs(300))
);
assert_eq!(
parse_retry_after("Thu, 19 Nov 2026 09:00:00 GMT", now),
Some(Duration::ZERO)
);
assert_eq!(
parse_retry_after("999999999", now),
Some(Duration::from_secs(24 * 60 * 60))
);
assert_eq!(parse_retry_after("soon", now), None);
}
// ---------------------------------------------------------------------------
// Refresh schedule
// ---------------------------------------------------------------------------
#[test]
fn failures_back_off_exponentially_up_to_a_day() {
assert_eq!(failure_backoff_ms(1, None), BACKOFF_BASE_MS);
assert_eq!(failure_backoff_ms(2, None), 2 * BACKOFF_BASE_MS);
assert_eq!(failure_backoff_ms(3, None), 4 * BACKOFF_BASE_MS);
assert_eq!(failure_backoff_ms(7, None), 64 * BACKOFF_BASE_MS);
assert_eq!(failure_backoff_ms(8, None), BACKOFF_MAX_MS);
assert_eq!(failure_backoff_ms(u32::MAX, None), BACKOFF_MAX_MS);
// A longer Retry-After wins, a shorter one does not shorten the backoff.
assert_eq!(
failure_backoff_ms(1, Some(Duration::from_secs(3 * 60 * 60))),
3 * 60 * 60 * 1000
);
assert_eq!(
failure_backoff_ms(2, Some(Duration::from_secs(1))),
2 * BACKOFF_BASE_MS
);
assert_eq!(
failure_backoff_ms(1, Some(Duration::from_secs(u64::MAX))),
BACKOFF_MAX_MS
);
}
#[test]
fn regular_checks_are_hourly_with_a_stable_spread() {
let podcast = PodcastId::from_feed_url(&feed_url("https://feeds.example/a.xml"));
let other = PodcastId::from_feed_url(&feed_url("https://feeds.example/b.xml"));
let now = 1_000_000;
let next = next_regular_check_ms(now, 7, &podcast);
assert!(
(now + REFRESH_INTERVAL_MS..now + REFRESH_INTERVAL_MS + 10 * 60 * 1000).contains(&next)
);
assert_eq!(next, next_regular_check_ms(now, 7, &podcast));
let spread: std::collections::BTreeSet<i64> = (0..20)
.map(|user| next_regular_check_ms(now, user, &podcast))
.chain([next_regular_check_ms(now, 7, &other)])
.collect();
assert!(spread.len() > 10);
}
/// A port on which nothing listens.
fn closed_port() -> u16 {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.local_addr().unwrap().port()
}
#[tokio::test]
async fn internal_names_cannot_be_told_from_unreachable_ones() {
// A name that resolves to a private address, a name that does not
// resolve and a server that refuses the connection look the same to
// the user: otherwise "add RSS" would tell which names exist in the
// server's internal DNS. Only what the URL itself says (an IP literal,
// the port, the scheme) is refused as such.
let port = closed_port();
let lookup = ScriptedLookup::with(&[
("vault.internal.test", &[&["10.20.30.40"]]),
("closed.test", &[&["127.0.0.1"]]),
]);
let fetcher = fetcher(lookup);
let mut answers = Vec::new();
for text in [
format!("http://vault.internal.test:{port}/feed"),
format!("http://nowhere.internal.test:{port}/feed"),
format!("http://closed.test:{port}/feed"),
] {
let error = fetcher
.fetch(&feed_url(&text), &Validators::default())
.await
.unwrap_err();
assert!(!error.public_message().contains("internal"), "{text}");
answers.push((error.public_message(), error.kind));
}
assert_eq!(answers[0].1, FetchErrorKind::Transport, "{answers:?}");
assert!(
answers.iter().all(|answer| *answer == answers[0]),
"{answers:?}"
);
let literal = fetcher
.fetch(&feed_url("http://10.20.30.40/feed"), &Validators::default())
.await
.unwrap_err();
assert_eq!(literal.kind, FetchErrorKind::Refused);
assert_ne!(literal.public_message(), answers[0].0);
}
+980
View File
@@ -0,0 +1,980 @@
//! HTTP endpoints of the podcast runtime, mounted at `/api/podcasts`. The
//! request handling lives in [`super::api`]; this module authenticates (the
//! session or a bearer token of the signed-in account) and maps results to
//! JSON responses.
//!
//! Podcast UI:
//!
//! * `GET /feeds[?all=1]` — subscribed podcasts (with `all`, also
//! unsubscribed ones and podcasts only known from synced progress), with
//! counters and the local fetch state.
//! * `POST /feeds` `{url}` — "add RSS": fetch, then subscribe (private).
//! * `GET /feeds/{podcast_id}` — one podcast and its season groups.
//! * `GET /feeds/{podcast_id}/episodes?season=&played=&q=&sort=&limit=&offset=`
//! * `POST /feeds/{podcast_id}/subscribe`, `POST /feeds/{podcast_id}/unsubscribe`
//! (unsubscribing keeps progress and history and revokes sharing).
//! * `POST /feeds/{podcast_id}/share` `{shared, confirm?}` — "share in the
//! federation" on or off; 409 with `needs_confirmation` when the feed URLs
//! look private and the request does not confirm exactly them.
//! * `POST /discovery/preview` `{podcast_id, feed_url}` — seasons and
//! episodes of a podcast found in the public search, read from its feed
//! (nothing stored).
//! * `POST /discovery/subscribe` `{podcast_id, feed_url}` — subscribe
//! (private) to a found podcast with its origin feed URL; the id must hash
//! from the URL (422 otherwise).
//! * `POST /episodes/{episode_id}/restart|played|unplayed` — explicit
//! progress actions.
//! * `POST /{podcast_id}/refresh` — manual feed refresh of a subscribed
//! podcast (ignores the failure backoff, at most every 30 s).
//!
//! Web player:
//!
//! * `POST /episodes/{episode_id}/playback` — audio source, resume position
//! and (when playing starts) a new listening session.
//! * `POST /episodes/{episode_id}/progress` — checkpoints, pause, seek,
//! stop, handoff and end of stream of that session (also sent with
//! `navigator.sendBeacon` when the page is hidden).
//! * `GET /episodes/{episode_id}/source` — how the browser gets the audio:
//! the publisher URL (default, the browser streams it directly) and the
//! proxy URL when the administrator enabled the proxy.
//! * `GET /episodes/{episode_id}/audio` — the optional audio proxy
//! (`podcast_audio_proxy_enabled`). It takes an episode id, never a URL,
//! and only serves episodes of the signed-in account.
use std::sync::Arc;
use cot::db::Database;
use cot::http::StatusCode;
use cot::http::header::{
ACCEPT_RANGES, CACHE_CONTROL, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, RANGE, RETRY_AFTER,
};
use cot::json::Json;
use cot::request::extractors::{Path, UrlQuery};
use cot::response::IntoResponse;
use cot::router::method::{get, post};
use cot::router::{Route, Router};
use cot::session::Session;
use cot::{App, Body};
use futures_util::TryStreamExt;
use music_dht::podcast::{EpisodeId, PodcastId};
use serde::{Deserialize, Serialize};
use super::api::{self, ApiError, EpisodeAction, EpisodeListParams};
use super::discovery::Discovery;
use super::fetch::FeedFetcher;
use super::limits::{self, ConcurrencyLimit, Permit};
use super::proxy::AudioProxy;
use super::runner::{self, ManualRefreshError, TargetResult};
use super::{PodcastStore, now_ms};
use crate::auth;
use crate::config::AppConfig;
fn json_error(status: StatusCode, message: &str) -> cot::response::Response {
let body = serde_json::json!({ "error": message });
cot::http::Response::builder()
.status(status)
.header(CONTENT_TYPE, "application/json")
.header(CACHE_CONTROL, "private, no-store")
.body(Body::fixed(body.to_string()))
.expect("valid response")
}
fn internal(error: impl std::fmt::Display) -> cot::Error {
cot::Error::internal(error.to_string())
}
fn api_error(error: ApiError) -> cot::response::Response {
json_error(
StatusCode::from_u16(error.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
&error.message,
)
}
/// JSON response of an [`api`] result; personal data is never cached.
fn api_response<T: Serialize>(result: api::ApiResult<T>) -> cot::Result<cot::response::Response> {
match result {
Ok(value) => {
let mut response = Json(value).into_response()?;
response
.headers_mut()
.insert(CACHE_CONTROL, "private, no-store".parse().expect("static"));
Ok(response)
}
Err(error) => Ok(api_error(error)),
}
}
/// The signed-in account (session or bearer token).
async fn signed_in(auth_ctx: &auth::AuthContext, session: &Session, db: &Database) -> Option<i64> {
auth::get_request_user(auth_ctx, session, db)
.await
.map(|user| user.id)
}
fn unauthorized() -> cot::response::Response {
json_error(StatusCode::UNAUTHORIZED, "not authenticated")
}
/// 429 when an account or the server has too many requests of a kind in
/// flight (see [`limits`]).
fn too_busy(message: &str) -> cot::response::Response {
let mut response = json_error(StatusCode::TOO_MANY_REQUESTS, message);
response.headers_mut().insert(
RETRY_AFTER,
limits::RETRY_AFTER_SECS
.to_string()
.parse()
.expect("digits"),
);
response
}
const FEEDS_BUSY: &str = "too many feed requests in progress; try again in a few seconds";
/// Largest JSON body accepted (a playback request carries one episode reference).
const MAX_JSON_BODY: usize = 64 * 1024;
/// Whether a `Content-Type` value is `application/json` (parameters such
/// as `charset` allowed).
fn is_json_content_type(value: Option<&str>) -> bool {
value
.and_then(|value| value.split(';').next())
.is_some_and(|essence| essence.trim().eq_ignore_ascii_case("application/json"))
}
/// Request body as JSON (`{}` when empty); malformed JSON is a 400.
///
/// A body must be sent as `application/json` (415 otherwise), like cot's
/// `Json` extractor requires on the other player endpoints: a cross-origin
/// page can only send that type after a CORS preflight, while `text/plain`
/// and form bodies need none and would carry the session cookie from a
/// same-site page. `navigator.sendBeacon` sends a Blob typed
/// `application/json`.
async fn json_body(request: cot::request::Request) -> Result<serde_json::Value, ApiError> {
let json = is_json_content_type(
request
.headers()
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok()),
);
let bytes = request
.into_body()
.into_bytes_limited(MAX_JSON_BODY)
.await
.map_err(|error| ApiError::new(400, format!("unreadable body: {error}")))?;
if bytes.iter().all(u8::is_ascii_whitespace) {
return Ok(serde_json::Value::Object(Default::default()));
}
if !json {
return Err(ApiError::new(415, "the body must be application/json"));
}
serde_json::from_slice(&bytes).map_err(|error| ApiError::new(400, error.to_string()))
}
/// Shared state of the handlers.
#[derive(Clone)]
struct State {
config: Arc<AppConfig>,
pool: Arc<tokio::sync::OnceCell<sqlx::PgPool>>,
fetcher: Arc<tokio::sync::OnceCell<FeedFetcher>>,
proxy: Arc<tokio::sync::OnceCell<AudioProxy>>,
/// Feed downloads users started ("add RSS", manual refresh).
feed_fetches: ConcurrencyLimit,
/// Audio proxy responses being streamed.
proxy_streams: ConcurrencyLimit,
}
impl State {
fn feed_fetch(&self, user_id: i64) -> Option<Permit> {
self.feed_fetches.try_acquire(user_id)
}
async fn store(&self) -> cot::Result<PodcastStore> {
let pool = self
.pool
.get_or_try_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&self.config.database_url)
.await
})
.await
.map_err(internal)?;
Ok(PodcastStore::new(pool.clone()))
}
async fn fetcher(&self) -> cot::Result<&FeedFetcher> {
self.fetcher
.get_or_try_init(|| async { FeedFetcher::new() })
.await
.map_err(internal)
}
async fn proxy(&self) -> cot::Result<&AudioProxy> {
self.proxy
.get_or_try_init(|| async { AudioProxy::new() })
.await
.map_err(internal)
}
}
/// The podcast runtime endpoints (see the module documentation).
pub(crate) struct PodcastsApp {
state: State,
}
impl PodcastsApp {
pub(crate) fn new(config: Arc<AppConfig>) -> Self {
Self {
state: State {
config,
pool: Arc::new(tokio::sync::OnceCell::new()),
fetcher: Arc::new(tokio::sync::OnceCell::new()),
proxy: Arc::new(tokio::sync::OnceCell::new()),
feed_fetches: ConcurrencyLimit::new(
limits::FEED_FETCHES_PER_ACCOUNT,
limits::FEED_FETCHES_TOTAL,
),
proxy_streams: ConcurrencyLimit::new(
limits::PROXY_STREAMS_PER_ACCOUNT,
limits::PROXY_STREAMS_TOTAL,
),
},
}
}
}
#[derive(Debug, Deserialize)]
struct PodcastPath {
podcast_id: String,
}
#[derive(Debug, Deserialize)]
struct EpisodePath {
episode_id: String,
}
/// Path of the proxy endpoint for an episode.
pub(crate) fn proxy_path(episode_id: &EpisodeId) -> String {
format!("/api/podcasts/episodes/{episode_id}/audio")
}
#[derive(Debug, Serialize)]
struct RefreshResponse {
status: &'static str,
new_episodes: usize,
updated_episodes: usize,
unchanged_episodes: usize,
skipped_items: usize,
feed_url: Option<String>,
error: Option<String>,
next_check_at_ms: Option<i64>,
}
impl RefreshResponse {
fn from_result(result: TargetResult) -> Self {
let mut response = Self {
status: "",
new_episodes: 0,
updated_episodes: 0,
unchanged_episodes: 0,
skipped_items: 0,
feed_url: None,
error: None,
next_check_at_ms: None,
};
match result {
TargetResult::Updated(outcome) => {
response.status = "updated";
response.new_episodes = outcome.new_episodes;
response.updated_episodes = outcome.updated_episodes;
response.unchanged_episodes = outcome.unchanged_episodes;
response.skipped_items = outcome.skipped_items;
response.feed_url = outcome.feed_url.map(String::from);
}
TargetResult::NotModified => response.status = "not_modified",
TargetResult::Failed {
error,
next_check_at_ms,
..
} => {
response.status = "failed";
response.error = Some(error);
response.next_check_at_ms = Some(next_check_at_ms);
}
TargetResult::Skipped(reason) => {
response.status = "skipped";
response.error = Some(reason);
}
}
response
}
}
async fn refresh_handler(
state: State,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PodcastPath>,
) -> cot::Result<cot::response::Response> {
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
let Ok(podcast_id) = PodcastId::parse(&path.0.podcast_id) else {
return Ok(json_error(StatusCode::BAD_REQUEST, "invalid podcast id"));
};
let Some(_permit) = state.feed_fetch(user.id) else {
return Ok(too_busy(FEEDS_BUSY));
};
let store = state.store().await?;
let fetcher = state.fetcher().await?;
match runner::refresh_now(&store, fetcher, user.id, &podcast_id, now_ms()).await {
Ok(result) => Json(RefreshResponse::from_result(result)).into_response(),
Err(error) => match error.downcast_ref::<ManualRefreshError>() {
Some(ManualRefreshError::UnknownPodcast) => {
Ok(json_error(StatusCode::NOT_FOUND, "unknown podcast"))
}
Some(ManualRefreshError::NotSubscribed) => {
Ok(json_error(StatusCode::CONFLICT, &error.to_string()))
}
Some(ManualRefreshError::TooSoon { retry_after_ms }) => {
let mut response = json_error(StatusCode::TOO_MANY_REQUESTS, &error.to_string());
response.headers_mut().insert(
RETRY_AFTER,
((retry_after_ms + 999) / 1000)
.to_string()
.parse()
.expect("digits"),
);
Ok(response)
}
None => Err(internal(format!("{error:#}"))),
},
}
}
#[derive(Debug, Serialize)]
struct SourceResponse {
episode_id: String,
/// Publisher URL; the browser streams it directly by default.
direct_url: String,
/// Same-origin proxy path, only when the proxy is enabled.
proxy_url: Option<String>,
mime: Option<String>,
}
async fn source_handler(
state: State,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<EpisodePath>,
) -> cot::Result<cot::response::Response> {
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
let Ok(episode_id) = EpisodeId::parse(&path.0.episode_id) else {
return Ok(json_error(StatusCode::BAD_REQUEST, "invalid episode id"));
};
let store = state.store().await?;
let Some((audio_url, mime)) = store
.audio_source(user.id, &episode_id)
.await
.map_err(internal)?
else {
return Ok(json_error(StatusCode::NOT_FOUND, "unknown episode"));
};
let (config, _) = AppConfig::load_with_db(&db).await;
let mut response = Json(SourceResponse {
proxy_url: config
.podcast_audio_proxy_enabled
.then(|| proxy_path(&episode_id)),
episode_id: episode_id.to_string(),
direct_url: audio_url,
mime,
})
.into_response()?;
response
.headers_mut()
.insert(CACHE_CONTROL, "private, no-store".parse().expect("static"));
Ok(response)
}
async fn audio_handler(
state: State,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<EpisodePath>,
request: cot::request::Request,
) -> cot::Result<cot::response::Response> {
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
let (config, _) = AppConfig::load_with_db(&db).await;
if !config.podcast_audio_proxy_enabled {
return Ok(json_error(
StatusCode::NOT_FOUND,
"the podcast audio proxy is disabled; stream from the publisher URL",
));
}
let Ok(episode_id) = EpisodeId::parse(&path.0.episode_id) else {
return Ok(json_error(StatusCode::BAD_REQUEST, "invalid episode id"));
};
let store = state.store().await?;
// The URL comes from this account's catalog, never from the request.
let Some((audio_url, _)) = store
.audio_source(user.id, &episode_id)
.await
.map_err(internal)?
else {
return Ok(json_error(StatusCode::NOT_FOUND, "unknown episode"));
};
let range = request
.headers()
.get(RANGE)
.and_then(|value| value.to_str().ok());
// Held until the body is fully sent or the browser goes away.
let Some(permit) = state.proxy_streams.try_acquire(user.id) else {
return Ok(too_busy("too many podcast audio streams in progress"));
};
let proxy = state.proxy().await?;
let audio = match proxy.fetch(&audio_url, range).await {
Ok(audio) => audio,
Err(error) => {
// Neither the URL (access tokens) nor the account is logged.
tracing::info!(episode = %episode_id, "podcast proxy: {error}");
let status =
StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::BAD_GATEWAY);
return Ok(json_error(status, &error.public_message()));
}
};
let mut builder = cot::http::Response::builder()
.status(StatusCode::from_u16(audio.status).unwrap_or(StatusCode::BAD_GATEWAY))
.header(CACHE_CONTROL, "private, no-store")
.header("X-Content-Type-Options", "nosniff")
.header("Content-Security-Policy", "default-src 'none'; sandbox")
.header("Content-Disposition", "inline");
if let Some(content_type) = &audio.content_type {
builder = builder.header(CONTENT_TYPE, content_type.as_str());
}
if let Some(length) = audio.content_length {
builder = builder.header(CONTENT_LENGTH, length.to_string());
}
if let Some(content_range) = &audio.content_range {
builder = builder.header(CONTENT_RANGE, content_range.as_str());
}
if audio.accept_ranges {
builder = builder.header(ACCEPT_RANGES, "bytes");
}
let body = audio.body.map_err(move |error| {
let _held = &permit;
cot::Error::internal(error.to_string())
});
builder.body(Body::streaming(body)).map_err(internal)
}
#[derive(Debug, Deserialize)]
struct ListQuery {
#[serde(default)]
all: Option<String>,
}
async fn list_handler(
state: State,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
query: UrlQuery<ListQuery>,
) -> cot::Result<cot::response::Response> {
let Some(user_id) = signed_in(&auth_ctx, &session, &db).await else {
return Ok(unauthorized());
};
let all = matches!(query.0.all.as_deref(), Some("1" | "true"));
let store = state.store().await?;
api_response(api::list_podcasts(&store, user_id, all).await)
}
async fn subscribe_url_handler(
state: State,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
request: cot::request::Request,
) -> cot::Result<cot::response::Response> {
let Some(user_id) = signed_in(&auth_ctx, &session, &db).await else {
return Ok(unauthorized());
};
let body = match json_body(request).await {
Ok(body) => body,
Err(error) => return Ok(api_error(error)),
};
let Some(_permit) = state.feed_fetch(user_id) else {
return Ok(too_busy(FEEDS_BUSY));
};
let store = state.store().await?;
let fetcher = state.fetcher().await?;
api_response(api::subscribe_url(&store, fetcher, user_id, body, now_ms()).await)
}
/// What a `/discovery/...` route does with a found podcast.
#[derive(Debug, Clone, Copy)]
enum FoundRoute {
Preview,
Subscribe,
}
/// A podcast from a public search hit: read its feed for a preview, or
/// subscribe. Both fetch the feed, so they share the feed request limits.
async fn found_handler(
state: State,
route: FoundRoute,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
request: cot::request::Request,
) -> cot::Result<cot::response::Response> {
let Some(user_id) = signed_in(&auth_ctx, &session, &db).await else {
return Ok(unauthorized());
};
let body = match json_body(request).await {
Ok(body) => body,
Err(error) => return Ok(api_error(error)),
};
let Some(_permit) = state.feed_fetch(user_id) else {
return Ok(too_busy(FEEDS_BUSY));
};
let store = state.store().await?;
let fetcher = state.fetcher().await?;
match route {
FoundRoute::Preview => {
api_response(api::preview_found(&store, fetcher, user_id, body).await)
}
FoundRoute::Subscribe => {
api_response(api::subscribe_found(&store, fetcher, user_id, body, now_ms()).await)
}
}
}
/// Whether public discovery runs on this server's federation node.
fn discovery_state<'a>(
federation: &crate::federation::Federation,
dht: Option<&'a music_dht::podcast_dht::PodcastDht>,
) -> Discovery<'a> {
match dht {
Some(dht) => Discovery::Running(dht),
None if federation.podcast_discovery_unavailable() => Discovery::Unavailable,
None => Discovery::Off,
}
}
/// What a `/feeds/{podcast_id}...` route does.
#[derive(Debug, Clone, Copy)]
enum PodcastRoute {
Detail,
Subscribe,
Unsubscribe,
}
async fn podcast_handler(
state: State,
route: PodcastRoute,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PodcastPath>,
) -> cot::Result<cot::response::Response> {
let Some(user_id) = signed_in(&auth_ctx, &session, &db).await else {
return Ok(unauthorized());
};
let podcast_id = match api::parse_podcast_id(&path.0.podcast_id) {
Ok(id) => id,
Err(error) => return Ok(api_error(error)),
};
let store = state.store().await?;
match route {
PodcastRoute::Detail => {
let federation = crate::federation::handle();
let dht = federation.podcast_discovery();
let discovery = discovery_state(&federation, dht.as_deref());
api_response(api::podcast_detail(&store, user_id, &podcast_id, discovery).await)
}
PodcastRoute::Subscribe => {
api_response(api::resubscribe(&store, user_id, &podcast_id).await)
}
PodcastRoute::Unsubscribe => {
api_response(api::unsubscribe(&store, user_id, &podcast_id).await)
}
}
}
async fn share_handler(
state: State,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PodcastPath>,
request: cot::request::Request,
) -> cot::Result<cot::response::Response> {
let Some(user_id) = signed_in(&auth_ctx, &session, &db).await else {
return Ok(unauthorized());
};
let podcast_id = match api::parse_podcast_id(&path.0.podcast_id) {
Ok(id) => id,
Err(error) => return Ok(api_error(error)),
};
// A JSON body is required (415 otherwise): the toggle cannot be sent by
// a cross-origin form or `text/plain` request without a CORS preflight.
let body = match json_body(request).await {
Ok(serde_json::Value::Object(body)) if !body.is_empty() => serde_json::Value::Object(body),
Ok(_) => {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"expected {\"shared\": bool}",
));
}
Err(error) => return Ok(api_error(error)),
};
let store = state.store().await?;
let federation = crate::federation::handle();
let dht = federation.podcast_discovery();
match api::set_share(
&store,
user_id,
&podcast_id,
body,
discovery_state(&federation, dht.as_deref()),
now_ms(),
)
.await
{
Ok(api::ShareOutcome::Done(response)) => api_response(Ok(*response)),
Ok(api::ShareOutcome::NeedsConfirmation(share)) => {
let body = serde_json::json!({
"error": "the feed URL may contain private data; confirm publishing it",
"needs_confirmation": true,
"share": share,
});
Ok(cot::http::Response::builder()
.status(StatusCode::CONFLICT)
.header(CONTENT_TYPE, "application/json")
.header(CACHE_CONTROL, "private, no-store")
.body(Body::fixed(body.to_string()))
.map_err(internal)?)
}
Err(error) => Ok(api_error(error)),
}
}
async fn episodes_handler(
state: State,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PodcastPath>,
query: UrlQuery<EpisodeListParams>,
) -> cot::Result<cot::response::Response> {
let Some(user_id) = signed_in(&auth_ctx, &session, &db).await else {
return Ok(unauthorized());
};
let podcast_id = match api::parse_podcast_id(&path.0.podcast_id) {
Ok(id) => id,
Err(error) => return Ok(api_error(error)),
};
let store = state.store().await?;
api_response(api::episodes(&store, user_id, &podcast_id, &query.0).await)
}
/// What a `/episodes/{episode_id}/...` POST route does.
#[derive(Debug, Clone, Copy)]
enum EpisodeRoute {
Action(EpisodeAction),
Playback,
Progress,
}
async fn episode_handler(
state: State,
route: EpisodeRoute,
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<EpisodePath>,
request: cot::request::Request,
) -> cot::Result<cot::response::Response> {
let Some(user_id) = signed_in(&auth_ctx, &session, &db).await else {
return Ok(unauthorized());
};
let episode_id = match api::parse_episode_id(&path.0.episode_id) {
Ok(id) => id,
Err(error) => return Ok(api_error(error)),
};
let store = state.store().await?;
match route {
EpisodeRoute::Action(action) => {
api_response(api::episode_action(&store, user_id, &episode_id, action).await)
}
EpisodeRoute::Playback => {
let body = match json_body(request).await {
Ok(body) => body,
Err(error) => return Ok(api_error(error)),
};
let (config, _) = AppConfig::load_with_db(&db).await;
api_response(
api::playback(
&store,
user_id,
&episode_id,
body,
config.podcast_audio_proxy_enabled,
)
.await,
)
}
EpisodeRoute::Progress => {
let body = match json_body(request).await {
Ok(body) => body,
Err(error) => return Ok(api_error(error)),
};
api_response(api::progress(&store, user_id, &episode_id, body, now_ms()).await)
}
}
}
impl App for PodcastsApp {
fn name(&self) -> &'static str {
"podcasts"
}
fn router(&self) -> Router {
let refresh_state = self.state.clone();
let source_state = self.state.clone();
let audio_state = self.state.clone();
let list_state = self.state.clone();
let subscribe_state = self.state.clone();
let episodes_state = self.state.clone();
let share_state = self.state.clone();
let found_route = |route: FoundRoute| {
let state = self.state.clone();
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
request: cot::request::Request| {
found_handler(state.clone(), route, auth_ctx, session, db, request)
}
};
let podcast_route = |route: PodcastRoute| {
let state = self.state.clone();
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PodcastPath>| {
podcast_handler(state.clone(), route, auth_ctx, session, db, path)
}
};
let episode_route = |route: EpisodeRoute| {
let state = self.state.clone();
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<EpisodePath>,
request: cot::request::Request| {
episode_handler(state.clone(), route, auth_ctx, session, db, path, request)
}
};
// Routes are matched in order: literal prefixes come before
// `/{podcast_id}/refresh`.
Router::with_urls([
Route::with_handler_and_name(
"/feeds",
get(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
query: UrlQuery<ListQuery>| {
list_handler(list_state.clone(), auth_ctx, session, db, query)
},
)
.post(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
request: cot::request::Request| {
subscribe_url_handler(
subscribe_state.clone(),
auth_ctx,
session,
db,
request,
)
},
),
"podcasts_feeds",
),
Route::with_handler_and_name(
"/discovery/preview",
post(found_route(FoundRoute::Preview)),
"podcasts_discovery_preview",
),
Route::with_handler_and_name(
"/discovery/subscribe",
post(found_route(FoundRoute::Subscribe)),
"podcasts_discovery_subscribe",
),
Route::with_handler_and_name(
"/feeds/{podcast_id}/episodes",
get(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PodcastPath>,
query: UrlQuery<EpisodeListParams>| {
episodes_handler(episodes_state.clone(), auth_ctx, session, db, path, query)
},
),
"podcasts_episodes",
),
Route::with_handler_and_name(
"/feeds/{podcast_id}/subscribe",
post(podcast_route(PodcastRoute::Subscribe)),
"podcasts_subscribe",
),
Route::with_handler_and_name(
"/feeds/{podcast_id}/unsubscribe",
post(podcast_route(PodcastRoute::Unsubscribe)),
"podcasts_unsubscribe",
),
Route::with_handler_and_name(
"/feeds/{podcast_id}/share",
post(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PodcastPath>,
request: cot::request::Request| {
share_handler(share_state.clone(), auth_ctx, session, db, path, request)
},
),
"podcasts_share",
),
Route::with_handler_and_name(
"/feeds/{podcast_id}",
get(podcast_route(PodcastRoute::Detail)),
"podcasts_detail",
),
Route::with_handler_and_name(
"/episodes/{episode_id}/playback",
post(episode_route(EpisodeRoute::Playback)),
"podcasts_episode_playback",
),
Route::with_handler_and_name(
"/episodes/{episode_id}/progress",
post(episode_route(EpisodeRoute::Progress)),
"podcasts_episode_progress",
),
Route::with_handler_and_name(
"/episodes/{episode_id}/restart",
post(episode_route(EpisodeRoute::Action(EpisodeAction::Restart))),
"podcasts_episode_restart",
),
Route::with_handler_and_name(
"/episodes/{episode_id}/played",
post(episode_route(EpisodeRoute::Action(
EpisodeAction::MarkPlayed,
))),
"podcasts_episode_played",
),
Route::with_handler_and_name(
"/episodes/{episode_id}/unplayed",
post(episode_route(EpisodeRoute::Action(
EpisodeAction::MarkUnplayed,
))),
"podcasts_episode_unplayed",
),
Route::with_handler_and_name(
"/episodes/{episode_id}/source",
get(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<EpisodePath>| {
source_handler(source_state.clone(), auth_ctx, session, db, path)
},
),
"podcasts_episode_source",
),
Route::with_handler_and_name(
"/episodes/{episode_id}/audio",
get(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<EpisodePath>,
request: cot::request::Request| {
audio_handler(audio_state.clone(), auth_ctx, session, db, path, request)
},
),
"podcasts_episode_audio",
),
Route::with_handler_and_name(
"/{podcast_id}/refresh",
post(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PodcastPath>| {
refresh_handler(refresh_state.clone(), auth_ctx, session, db, path)
},
),
"podcasts_refresh",
),
])
}
}
#[cfg(test)]
mod tests {
use super::*;
fn request(content_type: Option<&str>, body: &'static str) -> cot::request::Request {
let mut builder = cot::http::Request::builder()
.method("POST")
.uri("/api/podcasts/feeds");
if let Some(value) = content_type {
builder = builder.header(CONTENT_TYPE, value);
}
builder.body(Body::fixed(body)).unwrap()
}
#[tokio::test]
async fn json_bodies_need_the_json_content_type() {
let url = r#"{"url": "https://podcasts.example/feed.xml"}"#;
for content_type in [
"application/json",
"application/json; charset=UTF-8",
"Application/JSON",
] {
let body = json_body(request(Some(content_type), url)).await.unwrap();
assert_eq!(body["url"], "https://podcasts.example/feed.xml");
}
// What a page can send cross-origin without a CORS preflight.
for content_type in [
Some("text/plain"),
Some("text/plain;charset=UTF-8"),
Some("application/x-www-form-urlencoded"),
Some("multipart/form-data; boundary=x"),
None,
] {
let error = json_body(request(content_type, url)).await.unwrap_err();
assert_eq!(error.status, 415, "{content_type:?}");
}
// An empty body (a plain POST) is an empty object whatever the type.
let empty = json_body(request(None, "")).await.unwrap();
assert_eq!(empty, serde_json::json!({}));
let malformed = json_body(request(Some("application/json"), "{"))
.await
.unwrap_err();
assert_eq!(malformed.status, 400);
}
}
+123
View File
@@ -0,0 +1,123 @@
//! Bounds on the outbound work signed-in users can start: feed downloads
//! they ask for ("add RSS", manual refresh) and audio proxy streams. Each
//! kind has a per-account and a server-wide limit on requests in flight, so
//! one account cannot tie up the server's memory, connections or bandwidth.
//! Scheduled refreshes are bounded by the job itself (`runner::RUN_LIMIT`,
//! `runner::RUN_CONCURRENCY`, accounts in turns).
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
/// Feed downloads one account may run at once ("add RSS", refresh).
pub(crate) const FEED_FETCHES_PER_ACCOUNT: usize = 2;
/// Feed downloads started by users, server-wide (each may buffer up to
/// `fetch::MAX_FEED_BYTES`).
pub(crate) const FEED_FETCHES_TOTAL: usize = 16;
/// Audio proxy responses one account may stream at once (a player opens a
/// new range request when seeking while the old one is still closing).
pub(crate) const PROXY_STREAMS_PER_ACCOUNT: usize = 4;
/// Audio proxy responses streamed at once, server-wide.
pub(crate) const PROXY_STREAMS_TOTAL: usize = 64;
/// Seconds a refused client is asked to wait (`Retry-After`).
pub(crate) const RETRY_AFTER_SECS: u64 = 5;
#[derive(Debug, Default)]
struct Counts {
total: usize,
accounts: HashMap<i64, usize>,
}
/// Requests in flight per account and in total.
#[derive(Debug, Clone)]
pub(crate) struct ConcurrencyLimit {
per_account: usize,
total: usize,
counts: Arc<Mutex<Counts>>,
}
impl ConcurrencyLimit {
pub(crate) fn new(per_account: usize, total: usize) -> Self {
Self {
per_account,
total,
counts: Arc::default(),
}
}
/// A slot for one request of `user_id`, released when the permit is
/// dropped; `None` when the account or the server is at its limit.
pub(crate) fn try_acquire(&self, user_id: i64) -> Option<Permit> {
let mut counts = self
.counts
.lock()
.unwrap_or_else(|error| error.into_inner());
let account = counts.accounts.get(&user_id).copied().unwrap_or(0);
if account >= self.per_account || counts.total >= self.total {
return None;
}
counts.total += 1;
counts.accounts.insert(user_id, account + 1);
Some(Permit {
user_id,
counts: Arc::clone(&self.counts),
})
}
#[cfg(test)]
fn in_flight(&self, user_id: i64) -> (usize, usize) {
let counts = self.counts.lock().unwrap();
(
counts.accounts.get(&user_id).copied().unwrap_or(0),
counts.total,
)
}
}
/// One request in flight (see [`ConcurrencyLimit::try_acquire`]).
#[derive(Debug)]
pub(crate) struct Permit {
user_id: i64,
counts: Arc<Mutex<Counts>>,
}
impl Drop for Permit {
fn drop(&mut self) {
let mut counts = self
.counts
.lock()
.unwrap_or_else(|error| error.into_inner());
counts.total = counts.total.saturating_sub(1);
if let Some(account) = counts.accounts.get_mut(&self.user_id) {
*account = account.saturating_sub(1);
if *account == 0 {
counts.accounts.remove(&self.user_id);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn accounts_and_the_server_are_bounded_until_permits_are_dropped() {
let limit = ConcurrencyLimit::new(2, 3);
let a1 = limit.try_acquire(1).unwrap();
let a2 = limit.try_acquire(1).unwrap();
assert!(limit.try_acquire(1).is_none(), "per account");
let b1 = limit.try_acquire(2).unwrap();
assert!(limit.try_acquire(3).is_none(), "server-wide");
assert_eq!(limit.in_flight(1), (2, 3));
drop(a1);
assert_eq!(limit.in_flight(1), (1, 2));
let a3 = limit.try_acquire(1).unwrap();
drop((a2, a3, b1));
assert_eq!(limit.in_flight(1), (0, 0));
assert!(limit.counts.lock().unwrap().accounts.is_empty());
// Clones share the counts.
let clone = limit.clone();
let _held = clone.try_acquire(4).unwrap();
assert_eq!(limit.in_flight(4), (1, 1));
}
}
+341
View File
@@ -0,0 +1,341 @@
//! PostgreSQL podcast store of the web server.
//!
//! [`PodcastStore`] implements the podcast contract of `music_dht::podcast`
//! (normative description: `PODCASTS.md` of frid) on PostgreSQL, with the
//! same invariants as the SQLite reference store of `furumi-library`. Each
//! web account is one virtual trusted device, so every table is keyed by
//! `user_id`. The store owns no HTTP, scheduling or UI: RSS fetching and the
//! player use it.
//!
//! # Tables
//!
//! | Table | Key | Content | Synced |
//! | --- | --- | --- | --- |
//! | `furumusic__podcast` | `podcast_id` | local channel metadata, `feed_url` = URL this account fetches | no (catalog) |
//! | `furumusic__podcast_feed_alias` | `(podcast_id, feed_url)` | bounded grow-only alias set | yes, inside subscription records and `podcast_feed_aliases_added` |
//! | `furumusic__podcast_episode` | `episode_id` | `EpisodeMeta`, `source` (`feed`/`sync`), `in_feed` | no (catalog) |
//! | `furumusic__podcast_season` (view) | `(podcast_id, season)` | derived from the episodes | no |
//! | `furumusic__podcast_subscription` | `podcast_id` | subscription register | yes |
//! | `furumusic__podcast_share` | `podcast_id` | share register | yes |
//! | `furumusic__podcast_progress` | `episode_id` | progress register | yes |
//! | `furumusic__podcast_listen_session` | `session_id` | grow-only set of sessions | yes |
//! | `furumusic__podcast_fetch_state` | `podcast_id` | ETag, Last-Modified, check times, errors, cover cache | **never** |
//! | `furumusic__podcast_share_confirmation` | `podcast_id` | privacy confirmation of a shared card (share decision, card URLs) | **never** |
//!
//! # One apply path
//!
//! Remote operations ([`PodcastStore::apply_remote_op`]), snapshot sections
//! ([`PodcastStore::apply_snapshot`]) and local actions all end in the same
//! functions: the `SyncOpWire` materializers of `music_dht::device_sync` and
//! the `merge_*` rules of `music_dht::podcast`. Applying is therefore
//! idempotent and independent of delivery order, and a snapshot never
//! replaces newer state.
//!
//! Local actions ([`PodcastStore::subscribe`], [`PodcastStore::seek`], ...)
//! run in one transaction: they read the registers, append the operation to
//! the account's device-sync log with its stamp
//! ([`crate::federation::devices::append_local_op`]: next `seq`, `hlc_ms`,
//! `op_id = <device>:<seq>`) and apply exactly that operation. Either both
//! the log and the registers change or neither does. Every write
//! transaction takes a per-account advisory lock, so concurrent writers of
//! one account serialize.
//!
//! RSS refreshes ([`PodcastStore::apply_feed`]) touch the catalog and the
//! alias set only; they never write a register, progress or a session.
//!
//! # Runtime
//!
//! * `net` — the SSRF-guarded HTTP client used for every feed and audio
//! request (public addresses only, checked per connection and per
//! redirect hop).
//! * `fetch` and [`runner`] — conditional feed downloads and their schedule
//! (hourly checks, exponential backoff, one request per feed URL), run by
//! the `podcast_refresh` job and the manual refresh endpoint.
//! * `proxy` and [`http`] — the optional audio proxy and the
//! `/api/podcasts` endpoints. Audio is streamed from the publisher, never
//! stored.
//! * [`api`] — what the endpoints of the podcast UI and the web player do
//! (subscribe by URL, lists, explicit progress actions, sharing, playback
//! sessions and progress reports), independent of HTTP.
//! * [`discovery`] — public discovery: which podcasts the server node
//! publishes to `music_dht::podcast_dht` for the accounts that explicitly
//! shared them, the local privacy confirmations and the reconciler that
//! calls `PodcastDht::sync_shares`.
//! * [`search`] — public podcasts found by the federated search, as seen by
//! the searching account ("already subscribed"), and the preview of a
//! found podcast's feed before subscribing.
mod actions;
pub(crate) mod api;
mod apply;
pub(crate) mod discovery;
mod fetch;
pub(crate) mod http;
mod limits;
mod net;
mod proxy;
mod queries;
mod refresh;
pub(crate) mod runner;
pub(crate) mod schema;
pub(crate) mod search;
#[cfg(test)]
mod discovery_tests;
#[cfg(test)]
mod fetch_tests;
#[cfg(test)]
mod net_tests;
#[cfg(test)]
pub(crate) mod postgres_tests;
#[cfg(test)]
mod proxy_tests;
#[cfg(test)]
mod search_tests;
#[cfg(test)]
pub(crate) mod test_server;
#[cfg(test)]
mod tests;
use std::fmt;
use anyhow::Result;
use music_dht::podcast::{
EpisodeId, FeedUrl, ListenSession, PodcastError, PodcastId, PodcastMeta, ProgressRecord,
ProgressStatus,
};
use sqlx::Row;
use sqlx::postgres::PgRow;
pub(crate) use actions::{CheckpointOutcome, ProgressSession};
pub(crate) use apply::ApplyOutcome;
pub(crate) use fetch::FeedFetcher;
pub(crate) use queries::{
EpisodeListItem, EpisodeQuery, EpisodeSort, PlayedFilter, PodcastListFilter, PodcastSummary,
SeasonFilter, SeasonSummary,
};
pub(crate) use refresh::{FeedRefresh, FetchState, RefreshOutcome};
/// Typed errors of [`PodcastStore`]; other failures are database errors.
///
/// They are returned inside `anyhow::Error`; use `downcast_ref` to match.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum PodcastStoreError {
/// The podcast is unknown to this account.
UnknownPodcast(PodcastId),
/// The action needs an active subscription.
NotSubscribed(PodcastId),
/// Input violates the podcast contract.
Invalid(String),
/// A new subscription would exceed [`MAX_SUBSCRIPTIONS_PER_ACCOUNT`].
SubscriptionLimit(usize),
/// A new episode reference from the browser would exceed
/// [`MAX_REFERENCE_EPISODES_PER_ACCOUNT`].
ReferenceLimit(usize),
}
/// Active subscriptions an account may add locally (subscriptions synced
/// from the account's own devices are always applied). Every subscription
/// is a feed the server downloads about once an hour.
pub(crate) const MAX_SUBSCRIPTIONS_PER_ACCOUNT: usize = 1000;
/// Episodes known only from references (`source = 'sync'`: not from one of
/// the account's feeds) beyond which the browser cannot add another one
/// through `POST /api/podcasts/episodes/{id}/playback`. References in
/// operations of the account's own devices are always applied; this bounds
/// what a signed-in user can insert directly, one request at a time.
pub(crate) const MAX_REFERENCE_EPISODES_PER_ACCOUNT: usize = 10_000;
impl fmt::Display for PodcastStoreError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnknownPodcast(id) => write!(f, "unknown podcast {id}"),
Self::NotSubscribed(id) => write!(f, "not subscribed to podcast {id}"),
Self::Invalid(reason) => write!(f, "invalid podcast data: {reason}"),
Self::SubscriptionLimit(limit) => {
write!(f, "subscription limit reached ({limit} podcasts)")
}
Self::ReferenceLimit(limit) => write!(
f,
"too many episodes from outside your feeds ({limit}); subscribe to the podcast to play it"
),
}
}
}
impl std::error::Error for PodcastStoreError {}
impl From<PodcastError> for PodcastStoreError {
fn from(error: PodcastError) -> Self {
Self::Invalid(error.to_string())
}
}
pub(crate) fn invalid(reason: impl ToString) -> anyhow::Error {
PodcastStoreError::Invalid(reason.to_string()).into()
}
/// PostgreSQL store of podcast catalog, personal state and fetch state.
///
/// Cheap to clone (the pool is shared). See the module documentation.
#[derive(Debug, Clone)]
pub(crate) struct PodcastStore {
pool: sqlx::PgPool,
}
impl PodcastStore {
pub(crate) fn new(pool: sqlx::PgPool) -> Self {
Self { pool }
}
/// The pool the store runs on.
pub(crate) fn pool(&self) -> &sqlx::PgPool {
&self.pool
}
/// Starts a write transaction holding the account's podcast lock.
async fn begin_write(
&self,
user_id: i64,
) -> Result<sqlx::Transaction<'static, sqlx::Postgres>> {
let mut tx = self.pool.begin().await?;
lock_account(&mut tx, user_id).await?;
Ok(tx)
}
}
/// Serializes podcast writers of one account until the transaction ends.
pub(crate) async fn lock_account(conn: &mut sqlx::PgConnection, user_id: i64) -> Result<()> {
sqlx::query(
"SELECT pg_advisory_xact_lock(hashtextextended('furumusic:podcasts:' || $1::text, 0))",
)
.bind(user_id)
.execute(conn)
.await?;
Ok(())
}
pub(crate) fn now_ms() -> i64 {
chrono::Utc::now().timestamp_millis()
}
/// Converts a contract generation to the `BIGINT` column value.
///
/// Generations above `i64::MAX` cannot be stored; records carrying one are
/// rejected (no conforming client produces them).
pub(crate) fn generation_to_sql(generation: u64) -> Result<i64, PodcastStoreError> {
i64::try_from(generation)
.map_err(|_| PodcastStoreError::Invalid("generation exceeds i64::MAX".to_string()))
}
pub(crate) fn generation_from_sql(value: i64) -> u64 {
u64::try_from(value).unwrap_or(0)
}
pub(crate) fn status_to_sql(status: ProgressStatus) -> &'static str {
match status {
ProgressStatus::NotStarted => "not_started",
ProgressStatus::InProgress => "in_progress",
ProgressStatus::Completed => "completed",
}
}
pub(crate) fn status_from_sql(value: &str) -> ProgressStatus {
match value {
"in_progress" => ProgressStatus::InProgress,
"completed" => ProgressStatus::Completed,
_ => ProgressStatus::NotStarted,
}
}
/// `u32` contract numbers (season, episode number) from a `BIGINT` column.
pub(crate) fn u32_from_sql(value: Option<i64>) -> Option<u32> {
value.and_then(|value| u32::try_from(value).ok())
}
fn decode_error(message: impl ToString) -> sqlx::Error {
sqlx::Error::Decode(Box::new(PodcastStoreError::Invalid(message.to_string())))
}
/// Parses a stored id; stored ids are validated on write.
pub(crate) fn podcast_id_col(row: &PgRow, column: &str) -> sqlx::Result<PodcastId> {
PodcastId::try_from(row.try_get::<String, _>(column)?).map_err(decode_error)
}
pub(crate) fn episode_id_col(row: &PgRow, column: &str) -> sqlx::Result<EpisodeId> {
EpisodeId::try_from(row.try_get::<String, _>(column)?).map_err(decode_error)
}
pub(crate) fn feed_url_col(row: &PgRow, column: &str) -> sqlx::Result<FeedUrl> {
FeedUrl::try_from(row.try_get::<String, _>(column)?).map_err(decode_error)
}
pub(crate) fn podcast_meta_col(row: &PgRow, column: &str) -> sqlx::Result<PodcastMeta> {
serde_json::from_value(row.try_get::<serde_json::Value, _>(column)?).map_err(decode_error)
}
/// Progress register columns of table alias `alias`, named `{prefix}...`.
pub(crate) fn progress_columns(alias: &str, prefix: &str) -> String {
[
"episode_id",
"podcast_id",
"position_ms",
"status",
"generation",
"session_id",
"hlc_ms",
"device_id",
"op_id",
]
.iter()
.map(|column| format!("{alias}.{column} AS {prefix}{column}"))
.collect::<Vec<_>>()
.join(", ")
}
/// Progress register selected with [`progress_columns`].
pub(crate) fn progress_from_row(row: &PgRow, prefix: &str) -> sqlx::Result<ProgressRecord> {
let column = |name: &str| format!("{prefix}{name}");
Ok(ProgressRecord {
episode_id: episode_id_col(row, &column("episode_id"))?,
podcast_id: podcast_id_col(row, &column("podcast_id"))?,
position_ms: row.try_get(column("position_ms").as_str())?,
status: status_from_sql(&row.try_get::<String, _>(column("status").as_str())?),
generation: generation_from_sql(row.try_get(column("generation").as_str())?),
session_id: row.try_get(column("session_id").as_str())?,
hlc_ms: row.try_get(column("hlc_ms").as_str())?,
device_id: row.try_get(column("device_id").as_str())?,
op_id: row.try_get(column("op_id").as_str())?,
})
}
/// Listening session columns of table alias `s` (plain names).
pub(crate) const SESSION_COLUMNS: &str = "s.session_id, s.episode_id, s.podcast_id, s.device_id, \
s.started_at_ms, s.ended_at_ms, s.start_position_ms, s.end_position_ms, s.listened_ms, \
s.completed";
pub(crate) fn session_from_row(row: &PgRow) -> sqlx::Result<ListenSession> {
Ok(ListenSession {
session_id: row.try_get("session_id")?,
episode_id: episode_id_col(row, "episode_id")?,
podcast_id: podcast_id_col(row, "podcast_id")?,
device_id: row.try_get("device_id")?,
started_at_ms: row.try_get("started_at_ms")?,
ended_at_ms: row.try_get("ended_at_ms")?,
start_position_ms: row.try_get("start_position_ms")?,
end_position_ms: row.try_get("end_position_ms")?,
listened_ms: row.try_get("listened_ms")?,
completed: row.try_get("completed")?,
})
}
/// Normalized search text of an episode: `normalize_name` of the title and
/// of the description, separated by a newline (the normalization never
/// produces one, so a needle cannot match across the two).
pub(crate) fn episode_search_norm(title: &str, description: Option<&str>) -> String {
format!(
"{}\n{}",
music_dht::normalize_name(title),
music_dht::normalize_name(description.unwrap_or_default())
)
}
+629
View File
@@ -0,0 +1,629 @@
//! Outbound HTTP for podcast feeds and the audio proxy, guarded against SSRF.
//!
//! Feed URLs and enclosure URLs come from users and from third-party feed
//! documents, so the server must never be tricked into requesting its own
//! network. Every request made through [`SafeHttp`]:
//!
//! * accepts only `http` and `https` URLs with a host and without userinfo;
//! * resolves host names with [`GuardedResolver`], which rejects the whole
//! answer when **any** address is not public (loopback, private, shared
//! CGNAT, link-local, cloud metadata, multicast, unspecified, documentation,
//! reserved; IPv4 and IPv6, including IPv4-mapped, NAT64 and 6to4 forms).
//! The connection is made to exactly the addresses that were checked: the
//! resolver *is* the connector's resolver, there is no separate pre-check
//! that a second DNS answer could bypass (DNS rebinding);
//! * checks IP-literal hosts the same way (the connector does not resolve
//! them);
//! * connects only to the web ports [`ALLOWED_PORTS`] (80, 443, 8080, 8443):
//! a public address can still be the server's own, and its other services
//! (databases, admin ports) are often reachable from the host itself;
//! * follows redirects itself, at most `max_redirects` hops, checking every
//! hop like the first URL;
//! * never uses a system proxy (it would resolve the target instead of the
//! guarded resolver), never stores or sends cookies and never forwards
//! credentials: callers pass the few headers they need explicitly.
//!
//! Loopback can be allowed only in tests (`AddressPolicy::test_allow_loopback`
//! is `#[cfg(test)]`), so the local test servers work and nothing else does.
use std::fmt;
use std::future::Future;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use reqwest::header::{HeaderMap, LOCATION};
use reqwest::{StatusCode, Url};
/// Why an address is not reachable through [`SafeHttp`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum AddressClass {
/// `0.0.0.0/8`, `::`.
Unspecified,
/// `127.0.0.0/8`, `::1`.
Loopback,
/// RFC 1918 and the deprecated IPv6 site-local range.
Private,
/// `100.64.0.0/10` (carrier-grade NAT).
SharedCgnat,
/// `169.254.0.0/16`, `fe80::/10`.
LinkLocal,
/// Cloud instance metadata endpoints.
CloudMetadata,
/// `fc00::/7`.
UniqueLocal,
/// `224.0.0.0/4`, `ff00::/8`.
Multicast,
/// `255.255.255.255`.
Broadcast,
/// Documentation ranges (`192.0.2.0/24`, `2001:db8::/32`, ...).
Documentation,
/// `198.18.0.0/15`, `2001:2::/48`.
Benchmarking,
/// Everything else outside global unicast (IETF protocol assignments,
/// `240.0.0.0/4`, IPv4-compatible IPv6, Teredo, local NAT64, ...).
Reserved,
}
impl fmt::Display for AddressClass {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Unspecified => "unspecified",
Self::Loopback => "loopback",
Self::Private => "private",
Self::SharedCgnat => "shared (CGNAT)",
Self::LinkLocal => "link-local",
Self::CloudMetadata => "cloud metadata",
Self::UniqueLocal => "unique local",
Self::Multicast => "multicast",
Self::Broadcast => "broadcast",
Self::Documentation => "documentation",
Self::Benchmarking => "benchmarking",
Self::Reserved => "reserved",
})
}
}
/// Well-known cloud metadata endpoints. They also fall into blocked ranges
/// (link-local, CGNAT, unique local); naming them makes errors clearer.
const METADATA_V4: [Ipv4Addr; 3] = [
Ipv4Addr::new(169, 254, 169, 254),
Ipv4Addr::new(169, 254, 170, 2),
Ipv4Addr::new(100, 100, 100, 200),
];
const METADATA_V6: [Ipv6Addr; 1] = [Ipv6Addr::new(0xfd00, 0x0ec2, 0, 0, 0, 0, 0, 0x0254)];
fn in_v4(ip: Ipv4Addr, net: [u8; 4], prefix: u32) -> bool {
let mask = if prefix == 0 {
0
} else {
u32::MAX << (32 - prefix)
};
u32::from(ip) & mask == u32::from(Ipv4Addr::from(net)) & mask
}
fn in_v6(ip: Ipv6Addr, net: [u16; 8], prefix: u32) -> bool {
let mask = if prefix == 0 {
0
} else {
u128::MAX << (128 - prefix)
};
let [a, b, c, d, e, f, g, h] = net;
u128::from(ip) & mask == u128::from(Ipv6Addr::new(a, b, c, d, e, f, g, h)) & mask
}
/// Class of a non-public IPv4 address; `None` for a public one.
fn classify_v4(ip: Ipv4Addr) -> Option<AddressClass> {
use AddressClass::*;
if METADATA_V4.contains(&ip) {
return Some(CloudMetadata);
}
const RULES: &[([u8; 4], u32, AddressClass)] = &[
([0, 0, 0, 0], 8, Unspecified),
([10, 0, 0, 0], 8, Private),
([100, 64, 0, 0], 10, SharedCgnat),
([127, 0, 0, 0], 8, Loopback),
([169, 254, 0, 0], 16, LinkLocal),
([172, 16, 0, 0], 12, Private),
([192, 0, 0, 0], 24, Reserved),
([192, 0, 2, 0], 24, Documentation),
([192, 88, 99, 0], 24, Reserved),
([192, 168, 0, 0], 16, Private),
([198, 18, 0, 0], 15, Benchmarking),
([198, 51, 100, 0], 24, Documentation),
([203, 0, 113, 0], 24, Documentation),
([224, 0, 0, 0], 4, Multicast),
([255, 255, 255, 255], 32, Broadcast),
([240, 0, 0, 0], 4, Reserved),
];
RULES
.iter()
.find(|(net, prefix, _)| in_v4(ip, *net, *prefix))
.map(|(_, _, class)| *class)
}
/// Class of a non-public IPv6 address; `None` for a public one. Forms that
/// embed an IPv4 address (mapped, NAT64, 6to4) are judged by that address.
fn classify_v6(ip: Ipv6Addr) -> Option<AddressClass> {
use AddressClass::*;
if METADATA_V6.contains(&ip) {
return Some(CloudMetadata);
}
if ip.is_unspecified() {
return Some(Unspecified);
}
if ip.is_loopback() {
return Some(Loopback);
}
let bits = u128::from(ip);
let embedded_v4 = |shift: u32| Ipv4Addr::from((bits >> shift) as u32);
// ::ffff:a.b.c.d
if let Some(v4) = ip.to_ipv4_mapped() {
return classify_v4(v4);
}
// 64:ff9b::/96, the well-known NAT64 prefix.
if in_v6(ip, [0x64, 0xff9b, 0, 0, 0, 0, 0, 0], 96) {
return classify_v4(embedded_v4(0));
}
// 2002::/16, 6to4: the IPv4 address follows the prefix.
if in_v6(ip, [0x2002, 0, 0, 0, 0, 0, 0, 0], 16) {
return classify_v4(embedded_v4(80));
}
const RULES: &[([u16; 8], u32, AddressClass)] = &[
// IPv4-compatible (deprecated) and the rest of ::/96.
([0, 0, 0, 0, 0, 0, 0, 0], 96, Reserved),
([0x64, 0xff9b, 1, 0, 0, 0, 0, 0], 48, Reserved),
([0x100, 0, 0, 0, 0, 0, 0, 0], 64, Reserved),
([0x2001, 0x2, 0, 0, 0, 0, 0, 0], 48, Benchmarking),
([0x2001, 0xdb8, 0, 0, 0, 0, 0, 0], 32, Documentation),
// IETF protocol assignments, including Teredo (2001::/32).
([0x2001, 0, 0, 0, 0, 0, 0, 0], 23, Reserved),
([0x3fff, 0, 0, 0, 0, 0, 0, 0], 20, Documentation),
([0xfc00, 0, 0, 0, 0, 0, 0, 0], 7, UniqueLocal),
([0xfe80, 0, 0, 0, 0, 0, 0, 0], 10, LinkLocal),
([0xfec0, 0, 0, 0, 0, 0, 0, 0], 10, Private),
([0xff00, 0, 0, 0, 0, 0, 0, 0], 8, Multicast),
];
if let Some((_, _, class)) = RULES
.iter()
.find(|(net, prefix, _)| in_v6(ip, *net, *prefix))
{
return Some(*class);
}
// Only global unicast (2000::/3) is public.
(!in_v6(ip, [0x2000, 0, 0, 0, 0, 0, 0, 0], 3)).then_some(Reserved)
}
/// Class of a non-public address; `None` when it is public.
pub(crate) fn classify(ip: IpAddr) -> Option<AddressClass> {
match ip {
IpAddr::V4(ip) => classify_v4(ip),
IpAddr::V6(ip) => classify_v6(ip),
}
}
/// Ports feeds and enclosures may be fetched from.
pub(crate) const ALLOWED_PORTS: [u16; 4] = [80, 443, 8080, 8443];
/// Which addresses [`SafeHttp`] may connect to.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct AddressPolicy {
allow_loopback: bool,
any_port: bool,
}
impl AddressPolicy {
/// Public addresses on [`ALLOWED_PORTS`] only. The only policy
/// available outside tests.
pub(crate) const PUBLIC_ONLY: Self = Self {
allow_loopback: false,
any_port: false,
};
/// Public addresses plus loopback on any port, for the local test
/// servers.
#[cfg(test)]
pub(crate) const fn test_allow_loopback() -> Self {
Self {
allow_loopback: true,
any_port: true,
}
}
/// Whether a connection to `port` is allowed.
pub(crate) fn allows_port(&self, port: u16) -> bool {
self.any_port || ALLOWED_PORTS.contains(&port)
}
/// Checks one address.
pub(crate) fn check(&self, ip: IpAddr) -> Result<(), AddressClass> {
match classify(ip) {
None => Ok(()),
Some(AddressClass::Loopback) if self.allow_loopback => Ok(()),
Some(class) => Err(class),
}
}
}
/// A host name or literal resolved to an address that is not allowed.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct BlockedAddress {
/// Host as written in the URL.
pub(crate) host: String,
/// The offending address.
pub(crate) address: IpAddr,
/// Why it is blocked.
pub(crate) class: AddressClass,
}
impl BlockedAddress {
/// Whether the URL named the address itself (an IP literal) rather
/// than a host name that resolved to it.
pub(crate) fn is_literal(&self) -> bool {
let literal = self.host.trim_start_matches('[').trim_end_matches(']');
literal.parse::<IpAddr>().is_ok()
}
}
impl fmt::Display for BlockedAddress {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.is_literal() {
write!(f, "{} is a {} address", self.host, self.class)?;
} else {
write!(
f,
"{} resolves to a {} address ({})",
self.host, self.class, self.address
)?;
}
f.write_str("; only public addresses are allowed")
}
}
impl std::error::Error for BlockedAddress {}
/// Name resolution used by [`GuardedResolver`].
pub(crate) trait HostLookup: Send + Sync + 'static {
/// Every address of `host` (a DNS name, never an IP literal).
fn lookup(&self, host: &str) -> LookupFuture;
}
pub(crate) type LookupFuture =
Pin<Box<dyn Future<Output = std::io::Result<Vec<IpAddr>>> + Send + 'static>>;
/// The operating system resolver (`getaddrinfo` via tokio).
#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct SystemLookup;
impl HostLookup for SystemLookup {
fn lookup(&self, host: &str) -> LookupFuture {
let host = host.to_string();
Box::pin(async move {
Ok(tokio::net::lookup_host((host.as_str(), 0))
.await?
.map(|addr| addr.ip())
.collect())
})
}
}
const LOOKUP_TIMEOUT: Duration = Duration::from_secs(10);
/// DNS resolver of the guarded client: resolves, then rejects the answer
/// unless every address is allowed. The connector connects to exactly the
/// returned addresses, so a checked answer cannot be swapped afterwards.
pub(crate) struct GuardedResolver {
policy: AddressPolicy,
lookup: Arc<dyn HostLookup>,
}
impl GuardedResolver {
pub(crate) fn new(policy: AddressPolicy, lookup: Arc<dyn HostLookup>) -> Self {
Self { policy, lookup }
}
/// Resolves and checks `host`; the addresses a connection may use.
pub(crate) async fn resolve_checked(
&self,
host: &str,
) -> Result<Vec<IpAddr>, Box<dyn std::error::Error + Send + Sync>> {
let ips = tokio::time::timeout(LOOKUP_TIMEOUT, self.lookup.lookup(host))
.await
.map_err(|_| format!("resolving {host} timed out"))??;
if ips.is_empty() {
return Err(format!("{host} has no addresses").into());
}
for ip in &ips {
if let Err(class) = self.policy.check(*ip) {
return Err(Box::new(BlockedAddress {
host: host.to_string(),
address: *ip,
class,
}));
}
}
Ok(ips)
}
}
impl reqwest::dns::Resolve for GuardedResolver {
fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
let resolver = Self {
policy: self.policy,
lookup: Arc::clone(&self.lookup),
};
let host = name.as_str().to_string();
Box::pin(async move {
let ips = resolver.resolve_checked(&host).await?;
// Port 0: the connector uses the URL's port.
let addrs: Vec<SocketAddr> = ips.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect();
Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs)
})
}
}
/// Errors of [`SafeHttp`].
#[derive(Debug)]
pub(crate) enum GuardError {
/// Not `http`/`https`.
UnsupportedScheme(String),
/// The URL carries `user:password@`.
Credentials,
/// The URL has no host.
MissingHost,
/// The host is (or resolves to) a blocked address.
Blocked(BlockedAddress),
/// The port is not one of [`ALLOWED_PORTS`].
Port(u16),
/// More redirects than allowed.
TooManyRedirects(usize),
/// A redirect without a usable `Location`.
InvalidRedirect(String),
/// Any other transport failure (DNS, connect, TLS, timeout, ...).
Http(reqwest::Error),
}
impl GuardError {
pub(crate) fn is_timeout(&self) -> bool {
matches!(self, Self::Http(error) if error.is_timeout())
}
/// Whether the guard refused the URL for what it says, before any name
/// resolution: scheme, credentials, port, a blocked IP literal.
///
/// A host *name* that resolves to a blocked address is not a refusal in
/// this sense: users are told the same as for a name that does not
/// resolve or a server that does not answer, so the answers cannot be
/// used to learn which names exist in the server's internal DNS. The
/// details stay in the server log.
pub(crate) fn is_refusal(&self) -> bool {
match self {
Self::UnsupportedScheme(_) | Self::Credentials | Self::MissingHost | Self::Port(_) => {
true
}
Self::Blocked(blocked) => blocked.is_literal(),
_ => false,
}
}
}
/// What a signed-in user is told about a refused or failed request. The
/// details ([`GuardError`]'s `Display`: resolved addresses, transport error
/// chains) stay in the server logs; showing them would turn feed and audio
/// requests into a scanner of the server's network.
pub(crate) const REFUSED_MESSAGE: &str =
"only public http and https addresses on ports 80, 443, 8080 and 8443 are allowed";
impl fmt::Display for GuardError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnsupportedScheme(scheme) => {
write!(f, "unsupported URL scheme {scheme:?} (only http and https)")
}
Self::Credentials => f.write_str("URLs with credentials are not allowed"),
Self::MissingHost => f.write_str("URL has no host"),
Self::Blocked(blocked) => blocked.fmt(f),
Self::Port(port) => write!(f, "port {port} is not allowed"),
Self::TooManyRedirects(limit) => write!(f, "more than {limit} redirects"),
Self::InvalidRedirect(reason) => write!(f, "invalid redirect: {reason}"),
Self::Http(error) => {
write!(f, "{error}")?;
let mut source = std::error::Error::source(error);
while let Some(inner) = source {
write!(f, ": {inner}")?;
source = inner.source();
}
Ok(())
}
}
}
}
impl std::error::Error for GuardError {}
impl From<reqwest::Error> for GuardError {
/// Surfaces a resolver rejection buried in the transport error chain.
/// The URL is dropped: feed and enclosure URLs often carry access
/// tokens, and these errors end up in logs.
fn from(error: reqwest::Error) -> Self {
let error = error.without_url();
let mut source = std::error::Error::source(&error);
while let Some(inner) = source {
if let Some(blocked) = inner.downcast_ref::<BlockedAddress>() {
return Self::Blocked(blocked.clone());
}
source = inner.source();
}
Self::Http(error)
}
}
/// Timeouts and limits of a [`SafeHttp`] client.
#[derive(Debug, Clone, Copy)]
pub(crate) struct SafeHttpConfig {
/// TCP + TLS connect timeout (name resolution has its own 10 s bound).
pub(crate) connect_timeout: Duration,
/// Maximum time between two reads of a response.
pub(crate) read_timeout: Duration,
/// Whole-request timeout including the body; `None` for streams.
pub(crate) total_timeout: Option<Duration>,
/// Redirect hops followed before giving up.
pub(crate) max_redirects: usize,
}
/// A response after following redirects.
#[derive(Debug)]
pub(crate) struct Followed {
/// The final response (never a followed redirect).
pub(crate) response: reqwest::Response,
/// URL of the final response.
pub(crate) final_url: Url,
/// Status of every redirect followed, in order.
pub(crate) redirects: Vec<StatusCode>,
}
impl Followed {
/// Whether at least one redirect was followed and all of them were
/// permanent (301/308): only then has the resource moved.
pub(crate) fn moved_permanently(&self) -> bool {
!self.redirects.is_empty()
&& self
.redirects
.iter()
.all(|status| matches!(status.as_u16(), 301 | 308))
}
}
/// SSRF-guarded HTTP client (see the module documentation).
#[derive(Clone)]
pub(crate) struct SafeHttp {
client: reqwest::Client,
policy: AddressPolicy,
max_redirects: usize,
}
impl fmt::Debug for SafeHttp {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SafeHttp")
.field("policy", &self.policy)
.field("max_redirects", &self.max_redirects)
.finish_non_exhaustive()
}
}
impl SafeHttp {
/// A production client: public addresses only, system DNS.
pub(crate) fn public(config: SafeHttpConfig) -> anyhow::Result<Self> {
Self::with_lookup(config, AddressPolicy::PUBLIC_ONLY, Arc::new(SystemLookup))
}
/// A client with an explicit policy and resolver (tests inject both).
pub(crate) fn with_lookup(
config: SafeHttpConfig,
policy: AddressPolicy,
lookup: Arc<dyn HostLookup>,
) -> anyhow::Result<Self> {
let mut builder = reqwest::Client::builder()
.dns_resolver(Arc::new(GuardedResolver::new(policy, lookup)))
// A proxy would resolve the target itself, bypassing the guard.
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.referer(false)
.connect_timeout(config.connect_timeout)
.read_timeout(config.read_timeout)
.user_agent(concat!(
"furumusic/",
env!("CARGO_PKG_VERSION"),
" (podcasts)"
));
if let Some(timeout) = config.total_timeout {
builder = builder.timeout(timeout);
}
Ok(Self {
client: builder.build()?,
policy,
max_redirects: config.max_redirects,
})
}
/// Checks a URL before any connection: scheme, credentials, host, and
/// IP-literal hosts against the policy (names are checked by the
/// resolver at connect time).
pub(crate) fn check_url(&self, url: &Url) -> Result<(), GuardError> {
check_url(self.policy, url)
}
/// `GET url` with `headers`, following redirects (each hop checked, the
/// same headers re-sent; nothing else is added).
pub(crate) async fn get(&self, url: Url, headers: &HeaderMap) -> Result<Followed, GuardError> {
let mut url = url;
let mut redirects = Vec::new();
loop {
self.check_url(&url)?;
let response = self
.client
.get(url.clone())
.headers(headers.clone())
.send()
.await?;
let status = response.status();
if !matches!(status.as_u16(), 301 | 302 | 303 | 307 | 308) {
return Ok(Followed {
response,
final_url: url,
redirects,
});
}
if redirects.len() >= self.max_redirects {
return Err(GuardError::TooManyRedirects(self.max_redirects));
}
let location = response
.headers()
.get(LOCATION)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| {
GuardError::InvalidRedirect(format!("HTTP {status} without Location"))
})?;
let next = url.join(location).map_err(|error| {
GuardError::InvalidRedirect(format!("unusable Location: {error}"))
})?;
redirects.push(status);
url = next;
}
}
}
/// See [`SafeHttp::check_url`].
pub(crate) fn check_url(policy: AddressPolicy, url: &Url) -> Result<(), GuardError> {
if !matches!(url.scheme(), "http" | "https") {
return Err(GuardError::UnsupportedScheme(url.scheme().to_string()));
}
if !url.username().is_empty() || url.password().is_some() {
return Err(GuardError::Credentials);
}
let host = url
.host_str()
.filter(|host| !host.is_empty())
.ok_or(GuardError::MissingHost)?;
// The URL parser normalizes every IPv4 spelling (decimal, octal, hex,
// trailing dot) to dotted quads and keeps IPv6 literals in brackets.
let bare = host
.strip_prefix('[')
.and_then(|host| host.strip_suffix(']'))
.unwrap_or(host);
if let Ok(ip) = bare.parse::<IpAddr>() {
policy.check(ip).map_err(|class| {
GuardError::Blocked(BlockedAddress {
host: host.to_string(),
address: ip,
class,
})
})?;
}
let port = url.port_or_known_default().ok_or(GuardError::MissingHost)?;
if !policy.allows_port(port) {
return Err(GuardError::Port(port));
}
Ok(())
}
+438
View File
@@ -0,0 +1,438 @@
//! SSRF guard: address classes, URL checks, the resolver, DNS rebinding,
//! redirects and headers. Network tests use the local test server; loopback
//! is allowed only through the test-only `AddressPolicy::test_allow_loopback`.
use std::net::IpAddr;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use reqwest::Url;
use reqwest::header::HeaderMap;
use super::net::{
AddressClass, AddressPolicy, BlockedAddress, GuardError, GuardedResolver, SafeHttp, check_url,
classify,
};
use super::test_server::{Route, ScriptedLookup, TEST_HTTP, TestServer, test_http};
// ---------------------------------------------------------------------------
// Address classes
// ---------------------------------------------------------------------------
fn ip(text: &str) -> IpAddr {
text.parse().unwrap()
}
#[test]
fn public_addresses_are_allowed() {
for address in [
"8.8.8.8",
"1.1.1.1",
"93.184.215.14",
"172.32.0.1",
"100.128.0.1",
"192.0.3.1",
"2606:4700:4700::1111",
"2a00:1450:4001:80b::200e",
"::ffff:8.8.8.8",
"64:ff9b::808:808",
"2002:808:808::1",
] {
assert_eq!(classify(ip(address)), None, "{address}");
assert_eq!(AddressPolicy::PUBLIC_ONLY.check(ip(address)), Ok(()));
}
}
#[test]
fn non_public_addresses_are_classified() {
use AddressClass::*;
let cases = [
// IPv4
("0.0.0.0", Unspecified),
("0.1.2.3", Unspecified),
("10.1.2.3", Private),
("100.64.0.1", SharedCgnat),
("100.127.255.254", SharedCgnat),
("127.0.0.1", Loopback),
("127.255.0.9", Loopback),
("169.254.1.1", LinkLocal),
("169.254.169.254", CloudMetadata),
("169.254.170.2", CloudMetadata),
("100.100.100.200", CloudMetadata),
("172.16.0.1", Private),
("172.31.255.255", Private),
("192.0.0.8", Reserved),
("192.0.2.5", Documentation),
("192.88.99.1", Reserved),
("192.168.1.1", Private),
("198.18.0.1", Benchmarking),
("198.19.255.255", Benchmarking),
("198.51.100.1", Documentation),
("203.0.113.1", Documentation),
("224.0.0.1", Multicast),
("239.255.255.250", Multicast),
("240.0.0.1", Reserved),
("255.255.255.255", Broadcast),
// IPv6
("::", Unspecified),
("::1", Loopback),
("::ffff:127.0.0.1", Loopback),
("::ffff:10.0.0.1", Private),
("::ffff:169.254.169.254", CloudMetadata),
("::ffff:0.0.0.0", Unspecified),
("::127.0.0.1", Reserved),
("::8.8.8.8", Reserved),
("64:ff9b::a00:1", Private),
("64:ff9b::7f00:1", Loopback),
("64:ff9b:1::1", Reserved),
("2002:7f00:1::1", Loopback),
("2002:a9fe:a9fe::1", CloudMetadata),
("2002:c0a8:101::1", Private),
("100::1", Reserved),
("2001::1", Reserved),
("2001:2::1", Benchmarking),
("2001:db8::1", Documentation),
("3fff::1", Documentation),
("fc00::1", UniqueLocal),
("fd12:3456::1", UniqueLocal),
("fd00:ec2::254", CloudMetadata),
("fe80::1", LinkLocal),
("febf::1", LinkLocal),
("fec0::1", Private),
("ff02::1", Multicast),
("4000::1", Reserved),
("::2", Reserved),
];
for (address, class) in cases {
assert_eq!(classify(ip(address)), Some(class), "{address}");
assert_eq!(
AddressPolicy::PUBLIC_ONLY.check(ip(address)),
Err(class),
"{address}"
);
}
}
#[test]
fn only_the_test_policy_allows_loopback_and_nothing_else() {
let test = AddressPolicy::test_allow_loopback();
assert_eq!(test.check(ip("127.0.0.1")), Ok(()));
assert_eq!(test.check(ip("::1")), Ok(()));
assert_eq!(test.check(ip("::ffff:127.0.0.1")), Ok(()));
assert_eq!(test.check(ip("10.0.0.1")), Err(AddressClass::Private));
assert_eq!(
test.check(ip("169.254.169.254")),
Err(AddressClass::CloudMetadata)
);
assert_eq!(
AddressPolicy::PUBLIC_ONLY.check(ip("127.0.0.1")),
Err(AddressClass::Loopback)
);
}
fn url(text: &str) -> Url {
Url::parse(text).unwrap()
}
#[test]
fn urls_are_checked_before_connecting() {
let policy = AddressPolicy::PUBLIC_ONLY;
for (text, class) in [
("http://127.0.0.1/feed", AddressClass::Loopback),
// Every IPv4 spelling is normalized by the URL parser.
("http://2130706433/feed", AddressClass::Loopback),
("http://0x7f.1/feed", AddressClass::Loopback),
("http://127.1/feed", AddressClass::Loopback),
("http://0177.0.0.1/feed", AddressClass::Loopback),
("http://127.0.0.1./feed", AddressClass::Loopback),
("http://0/feed", AddressClass::Unspecified),
("http://[::1]:8080/feed", AddressClass::Loopback),
("http://[::ffff:7f00:1]/feed", AddressClass::Loopback),
("http://[::ffff:a9fe:a9fe]/", AddressClass::CloudMetadata),
(
"http://169.254.169.254/latest/meta-data/",
AddressClass::CloudMetadata,
),
("https://10.0.0.5/audio.mp3", AddressClass::Private),
("https://[fd00::1]/audio.mp3", AddressClass::UniqueLocal),
] {
match check_url(policy, &url(text)) {
Err(GuardError::Blocked(blocked)) => assert_eq!(blocked.class, class, "{text}"),
other => panic!("{text}: {other:?}"),
}
}
assert!(matches!(
check_url(policy, &url("ftp://example.com/feed")),
Err(GuardError::UnsupportedScheme(_))
));
assert!(matches!(
check_url(policy, &url("file:///etc/passwd")),
Err(GuardError::UnsupportedScheme(_))
));
assert!(matches!(
check_url(policy, &url("https://user:secret@example.com/feed")),
Err(GuardError::Credentials)
));
assert!(matches!(
check_url(policy, &url("https://user@example.com/feed")),
Err(GuardError::Credentials)
));
// Names are checked by the resolver at connect time.
assert!(check_url(policy, &url("https://feeds.example.com/rss")).is_ok());
assert!(check_url(policy, &url("http://localhost/rss")).is_ok());
}
#[test]
fn only_web_ports_are_allowed() {
// A public address can be the server's own; its databases and admin
// ports must not be reachable through feed or audio requests.
let policy = AddressPolicy::PUBLIC_ONLY;
for text in [
"https://feeds.example.com/rss",
"http://feeds.example.com/rss",
"https://feeds.example.com:443/rss",
"http://feeds.example.com:8080/rss",
"https://feeds.example.com:8443/rss",
"https://8.8.8.8/rss",
] {
assert!(check_url(policy, &url(text)).is_ok(), "{text}");
}
for (text, port) in [
("http://feeds.example.com:5432/", 5432),
("https://feeds.example.com:22/", 22),
("http://8.8.8.8:6379/feed", 6379),
("https://feeds.example.com:3000/", 3000),
] {
match check_url(policy, &url(text)) {
Err(error @ GuardError::Port(refused)) => {
assert_eq!(refused, port, "{text}");
assert!(error.is_refusal());
}
other => panic!("{text}: {other:?}"),
}
}
// The local test servers listen on random ports.
assert!(
check_url(
AddressPolicy::test_allow_loopback(),
&url("http://127.0.0.1:5432/")
)
.is_ok()
);
}
// ---------------------------------------------------------------------------
// Resolver
// ---------------------------------------------------------------------------
fn blocked_class(error: &(dyn std::error::Error + Send + Sync + 'static)) -> Option<AddressClass> {
error
.downcast_ref::<BlockedAddress>()
.map(|blocked| blocked.class)
}
#[tokio::test]
async fn resolver_rejects_the_whole_answer_when_any_address_is_not_public() {
let lookup = ScriptedLookup::with(&[
("public.test", &[&["8.8.8.8", "2606:4700:4700::1111"]]),
("mixed.test", &[&["8.8.8.8", "10.0.0.1"]]),
("mapped.test", &[&["::ffff:192.168.0.10"]]),
("localhost", &[&["127.0.0.1", "::1"]]),
("empty.test", &[&[]]),
]);
let resolver = GuardedResolver::new(AddressPolicy::PUBLIC_ONLY, lookup);
assert_eq!(
resolver.resolve_checked("public.test").await.unwrap(),
vec![ip("8.8.8.8"), ip("2606:4700:4700::1111")]
);
let error = resolver.resolve_checked("mixed.test").await.unwrap_err();
assert_eq!(blocked_class(error.as_ref()), Some(AddressClass::Private));
let error = resolver.resolve_checked("mapped.test").await.unwrap_err();
assert_eq!(blocked_class(error.as_ref()), Some(AddressClass::Private));
let error = resolver.resolve_checked("localhost").await.unwrap_err();
assert_eq!(blocked_class(error.as_ref()), Some(AddressClass::Loopback));
assert!(resolver.resolve_checked("empty.test").await.is_err());
assert!(resolver.resolve_checked("nowhere.test").await.is_err());
}
fn public_http(lookup: Arc<ScriptedLookup>) -> SafeHttp {
SafeHttp::with_lookup(TEST_HTTP, AddressPolicy::PUBLIC_ONLY, lookup).unwrap()
}
#[tokio::test]
async fn production_policy_never_reaches_a_local_server() {
let server = TestServer::start();
server.set("/feed", Route::new("ok", "text/plain"));
let lookup = ScriptedLookup::with(&[("loop.test", &[&["127.0.0.1"]])]);
let http = public_http(Arc::clone(&lookup));
match http.get(url(&server.url("/feed")), &HeaderMap::new()).await {
Err(GuardError::Blocked(blocked)) => assert_eq!(blocked.class, AddressClass::Loopback),
other => panic!("{other:?}"),
}
// The test server's random port is refused before any lookup.
let named = url(&server.url_on("loop.test", "/feed"));
match http.get(named.clone(), &HeaderMap::new()).await {
Err(GuardError::Port(port)) => assert_eq!(Some(port), named.port()),
other => panic!("{other:?}"),
}
assert_eq!(lookup.calls.load(Ordering::SeqCst), 0);
// On an allowed port the resolver refuses the loopback answer.
let resolver = GuardedResolver::new(AddressPolicy::PUBLIC_ONLY, lookup);
let error = resolver.resolve_checked("loop.test").await.unwrap_err();
assert_eq!(blocked_class(error.as_ref()), Some(AddressClass::Loopback));
assert_eq!(server.request_count(), 0);
}
#[tokio::test]
async fn every_connection_uses_the_addresses_it_checked() {
// DNS rebinding: the first answer is allowed, the next one points to a
// private address. Each connection resolves and checks once and
// connects to exactly those addresses, so the rebound answer is refused
// instead of being used after an earlier successful check.
let server = TestServer::start();
server.set("/feed", Route::new("first", "text/plain"));
let lookup = ScriptedLookup::with(&[("rebind.test", &[&["127.0.0.1"], &["10.0.0.7"]])]);
let http = test_http(Arc::clone(&lookup));
let target = url(&server.url_on("rebind.test", "/feed"));
let first = http.get(target.clone(), &HeaderMap::new()).await.unwrap();
assert_eq!(first.response.text().await.unwrap(), "first");
match http.get(target, &HeaderMap::new()).await {
Err(GuardError::Blocked(blocked)) => {
assert_eq!(blocked.class, AddressClass::Private);
assert_eq!(blocked.address, ip("10.0.0.7"));
}
other => panic!("{other:?}"),
}
assert_eq!(lookup.calls.load(Ordering::SeqCst), 2);
assert_eq!(server.requests("/feed").len(), 1);
}
#[tokio::test]
async fn every_redirect_hop_is_checked() {
let server = TestServer::start();
let lookup = ScriptedLookup::with(&[("intranet.test", &[&["192.168.1.20"]])]);
let http = test_http(lookup);
let cases = [
(
"/to-private",
"http://10.0.0.1/feed".to_string(),
AddressClass::Private,
),
(
"/to-metadata",
"http://169.254.169.254/latest/meta-data/".to_string(),
AddressClass::CloudMetadata,
),
(
"/to-mapped",
"http://[::ffff:10.1.1.1]/".to_string(),
AddressClass::Private,
),
(
"/to-name",
server.url_on("intranet.test", "/feed"),
AddressClass::Private,
),
];
for (path, location, class) in &cases {
server.set(path, Route::redirect(302, location));
match http.get(url(&server.url(path)), &HeaderMap::new()).await {
Err(GuardError::Blocked(blocked)) => assert_eq!(blocked.class, *class, "{path}"),
other => panic!("{path}: {other:?}"),
}
}
server.set("/to-file", Route::redirect(301, "file:///etc/passwd"));
assert!(matches!(
http.get(url(&server.url("/to-file")), &HeaderMap::new())
.await,
Err(GuardError::UnsupportedScheme(_))
));
server.set("/no-location", Route::new("", "text/plain").status(302));
assert!(matches!(
http.get(url(&server.url("/no-location")), &HeaderMap::new())
.await,
Err(GuardError::InvalidRedirect(_))
));
// Hop limit (3 in the test client): /hop0 -> /hop1 -> ... -> /hop4.
for hop in 0..4 {
server.set(
&format!("/hop{hop}"),
Route::redirect(307, &format!("/hop{}", hop + 1)),
);
}
server.set("/hop4", Route::new("end", "text/plain"));
assert!(matches!(
http.get(url(&server.url("/hop0")), &HeaderMap::new()).await,
Err(GuardError::TooManyRedirects(3))
));
let followed = http
.get(url(&server.url("/hop1")), &HeaderMap::new())
.await
.unwrap();
assert_eq!(followed.redirects.len(), 3);
assert_eq!(followed.final_url.path(), "/hop4");
}
#[tokio::test]
async fn transport_errors_never_carry_the_url() {
// Feed and enclosure URLs often carry access tokens; errors are logged.
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
drop(listener);
let http = test_http(ScriptedLookup::with(&[]));
let error = http
.get(
url(&format!(
"http://127.0.0.1:{port}/private/feed.xml?token=secret"
)),
&HeaderMap::new(),
)
.await
.unwrap_err();
assert!(matches!(error, GuardError::Http(_)), "{error:?}");
let text = error.to_string();
assert!(
!text.contains("secret") && !text.contains("feed.xml"),
"{text}"
);
assert!(!format!("{error:?}").contains("secret"));
// A redirect to an unusable location does not echo it either.
let server = TestServer::start();
server.set("/bad", Route::redirect(302, "http://[::1/?token=secret"));
let error = http
.get(url(&server.url("/bad")), &HeaderMap::new())
.await
.unwrap_err();
assert!(matches!(error, GuardError::InvalidRedirect(_)), "{error:?}");
assert!(!error.to_string().contains("secret"), "{error}");
}
#[tokio::test]
async fn no_cookies_or_credentials_are_sent() {
let server = TestServer::start();
server.set(
"/start",
Route::redirect(302, "/next").header("Set-Cookie", "session=leak; Path=/"),
);
server.set(
"/next",
Route::new("ok", "text/plain").header("Set-Cookie", "other=leak; Path=/"),
);
let http = test_http(ScriptedLookup::with(&[]));
for _ in 0..2 {
let followed = http
.get(url(&server.url("/start")), &HeaderMap::new())
.await
.unwrap();
assert_eq!(followed.response.text().await.unwrap(), "ok");
}
for request in server
.requests("/start")
.iter()
.chain(&server.requests("/next"))
{
for header in ["cookie", "authorization", "referer", "proxy-authorization"] {
assert!(!request.contains_key(header), "{header} sent: {request:?}");
}
}
}
File diff suppressed because it is too large. Load diff
+537
View File
@@ -0,0 +1,537 @@
//! Optional audio proxy: streams a stored episode enclosure from the
//! publisher for browsers that cannot load it directly (CORS, mixed
//! content). Audio is passed through, never stored or imported.
//!
//! The caller resolves the URL from the database for an episode of the
//! authenticated account; a client never supplies a URL. Requests go
//! through [`SafeHttp`] (SSRF guard, every redirect hop checked), carry
//! only `Range`, `Accept`, `Accept-Encoding: identity` and the user agent,
//! and responses are limited in size, time and content type. The content
//! type is decided by the publisher's response (and, for generic binary
//! types, the URL extension or the file's first bytes), never by the
//! enclosure type stored with the episode.
use std::fmt;
use std::time::Duration;
use bytes::Bytes;
use futures_util::StreamExt;
use futures_util::stream::BoxStream;
use reqwest::header::{
ACCEPT, ACCEPT_ENCODING, ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, HeaderMap,
HeaderValue, RANGE,
};
use reqwest::{StatusCode, Url};
use super::net::{GuardError, REFUSED_MESSAGE, SafeHttp, SafeHttpConfig};
/// Largest response body passed through (a long episode at a high bitrate
/// stays far below).
pub(crate) const MAX_PROXY_BYTES: u64 = 1024 * 1024 * 1024;
/// Upstream must answer with headers within this time.
pub(crate) const PROXY_HEADER_TIMEOUT: Duration = Duration::from_secs(20);
/// Longest a single proxied response may stream.
pub(crate) const PROXY_MAX_DURATION: Duration = Duration::from_secs(4 * 60 * 60);
/// Redirect hops followed (enclosures often go through tracking redirects).
pub(crate) const MAX_PROXY_REDIRECTS: usize = 5;
/// HTTP client settings of the proxy (no whole-request timeout: bodies are
/// streamed; the idle read timeout and [`PROXY_MAX_DURATION`] bound them).
pub(crate) const PROXY_HTTP: SafeHttpConfig = SafeHttpConfig {
connect_timeout: Duration::from_secs(10),
read_timeout: Duration::from_secs(30),
total_timeout: None,
max_redirects: MAX_PROXY_REDIRECTS,
};
/// Why a proxied request failed.
#[derive(Debug)]
pub(crate) enum ProxyError {
/// The stored URL is not a valid URL.
InvalidUrl(String),
/// Refused or failed in transport (see [`GuardError`]).
Upstream(GuardError),
/// The publisher answered with an unusable status.
Status(u16),
/// The publisher's content type is not audio.
ContentType(String),
/// Declared body larger than [`MAX_PROXY_BYTES`].
TooLarge(u64),
/// No response headers within [`PROXY_HEADER_TIMEOUT`].
Timeout,
/// The body stream stopped (limit reached or upstream failure).
Body(String),
}
impl ProxyError {
/// Status the proxy answers the browser with.
pub(crate) fn http_status(&self) -> u16 {
match self {
Self::Timeout => 504,
Self::Upstream(error) if error.is_timeout() => 504,
_ => 502,
}
}
/// What the browser is told; the details (`Display`) stay in the
/// server log.
pub(crate) fn public_message(&self) -> String {
match self {
Self::InvalidUrl(_) => "the episode's audio URL is invalid".to_string(),
Self::Upstream(error) if error.is_refusal() => REFUSED_MESSAGE.to_string(),
Self::Upstream(error) if error.is_timeout() => {
"the publisher did not answer in time".to_string()
}
Self::Upstream(_) => "could not connect to the publisher".to_string(),
Self::Status(status) => format!("the publisher answered HTTP {status}"),
Self::ContentType(_) => "the publisher's file is not audio".to_string(),
Self::TooLarge(_) => "the publisher's file is too large".to_string(),
Self::Timeout => "the publisher did not answer in time".to_string(),
Self::Body(_) => "the publisher stream failed".to_string(),
}
}
}
impl fmt::Display for ProxyError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidUrl(reason) => write!(f, "stored audio URL is invalid: {reason}"),
Self::Upstream(error) => write!(f, "publisher request failed: {error}"),
Self::Status(status) => write!(f, "publisher answered HTTP {status}"),
Self::ContentType(value) => write!(f, "publisher content type {value:?} is not audio"),
Self::TooLarge(len) => write!(
f,
"publisher response is too large ({len} bytes, limit {MAX_PROXY_BYTES})"
),
Self::Timeout => write!(
f,
"publisher did not answer within {} s",
PROXY_HEADER_TIMEOUT.as_secs()
),
Self::Body(reason) => write!(f, "publisher stream failed: {reason}"),
}
}
}
impl std::error::Error for ProxyError {}
impl From<GuardError> for ProxyError {
fn from(error: GuardError) -> Self {
Self::Upstream(error)
}
}
/// A proxied response, ready to be sent to the browser.
pub(crate) struct ProxiedAudio {
/// 200, 206 or 416.
pub(crate) status: u16,
/// Allow-listed audio type (absent on 416).
pub(crate) content_type: Option<String>,
pub(crate) content_length: Option<u64>,
/// `bytes a-b/n` (206) or `bytes */n` (416).
pub(crate) content_range: Option<String>,
/// Whether the publisher advertised byte ranges.
pub(crate) accept_ranges: bool,
/// The body, cut off at [`MAX_PROXY_BYTES`] and [`PROXY_MAX_DURATION`].
pub(crate) body: BoxStream<'static, Result<Bytes, ProxyError>>,
}
impl fmt::Debug for ProxiedAudio {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ProxiedAudio")
.field("status", &self.status)
.field("content_type", &self.content_type)
.field("content_length", &self.content_length)
.field("content_range", &self.content_range)
.field("accept_ranges", &self.accept_ranges)
.finish_non_exhaustive()
}
}
/// Keeps a browser `Range` header only when it is one well-formed byte
/// range (`bytes=a-`, `bytes=a-b`, `bytes=-n`); anything else is dropped
/// and the full resource is requested.
pub(crate) fn sanitize_range(value: &str) -> Option<String> {
let spec = value.trim().strip_prefix("bytes=")?.trim();
if spec.len() > 64 {
return None;
}
let (start, end) = spec.split_once('-')?;
let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit());
let ok = match (start.is_empty(), end.is_empty()) {
(false, true) => digits(start) && start.parse::<u64>().is_ok(),
(false, false) => {
digits(start)
&& digits(end)
&& matches!((start.parse::<u64>(), end.parse::<u64>()), (Ok(a), Ok(b)) if a <= b)
}
(true, false) => digits(end) && end.parse::<u64>().is_ok_and(|n| n > 0),
(true, true) => false,
};
ok.then(|| format!("bytes={start}-{end}"))
}
/// `Content-Range` of a 206/416 passed through only when well-formed.
fn sanitize_content_range(value: &str) -> Option<String> {
let rest = value.trim().strip_prefix("bytes ")?;
let (range, total) = rest.split_once('/')?;
let total_ok = total == "*" || total.parse::<u64>().is_ok();
let range_ok = range == "*"
|| range.split_once('-').is_some_and(
|(a, b)| matches!((a.parse::<u64>(), b.parse::<u64>()), (Ok(a), Ok(b)) if a <= b),
);
(total_ok && range_ok && !(range == "*" && total == "*")).then(|| value.trim().to_string())
}
fn is_audio_type(mime: &str) -> bool {
let mime = mime.trim().to_ascii_lowercase();
let essence = mime.split(';').next().unwrap_or_default().trim();
(essence.starts_with("audio/") && essence.len() > "audio/".len())
|| matches!(essence, "application/ogg" | "application/x-ogg")
}
/// Audio type implied by the URL path's extension.
fn audio_type_from_url(url: &Url) -> Option<&'static str> {
let ext = url.path().rsplit('/').next()?.rsplit_once('.')?.1;
Some(match ext.to_ascii_lowercase().as_str() {
"mp3" => "audio/mpeg",
"m4a" | "m4b" | "mp4" => "audio/mp4",
"aac" => "audio/aac",
"ogg" | "oga" => "audio/ogg",
"opus" => "audio/ogg",
"flac" => "audio/flac",
"wav" => "audio/wav",
_ => return None,
})
}
/// Bytes read from the start of a file to recognize its format.
const SNIFF_BYTES: usize = 16;
/// What the declared type and the URL say about a response.
#[derive(Debug, PartialEq, Eq)]
enum TypeCheck {
/// Audio: the publisher's type, or a generic type with an audio file
/// extension.
Audio(String),
/// A generic binary type (or none) without an audio extension: only the
/// file's first bytes can tell.
Content,
}
/// The content type sent to the browser: the publisher's when it is audio;
/// for a generic binary type, the type implied by the URL extension
/// ([`TypeCheck::Content`] without one); anything else is refused.
///
/// The enclosure type from the feed (or from an episode reference a client
/// sent) is never trusted here: it is chosen by whoever wrote the feed, so
/// it would turn the proxy into a relay for any file served as
/// `application/octet-stream`.
fn audio_content_type(upstream: Option<&str>, url: &Url) -> Result<TypeCheck, ProxyError> {
let upstream = upstream.map(str::trim).filter(|value| !value.is_empty());
if let Some(value) = upstream
&& is_audio_type(value)
{
return Ok(TypeCheck::Audio(value.to_string()));
}
let generic = upstream.is_none_or(|value| {
let essence = value.split(';').next().unwrap_or_default().trim();
essence.eq_ignore_ascii_case("application/octet-stream")
|| essence.eq_ignore_ascii_case("binary/octet-stream")
});
if !generic {
return Err(ProxyError::ContentType(
upstream.unwrap_or("(none)").to_string(),
));
}
Ok(match audio_type_from_url(url) {
Some(guess) => TypeCheck::Audio(guess.to_string()),
None => TypeCheck::Content,
})
}
/// The audio format a file starts with (magic numbers of the formats
/// podcasts use), or `None`.
pub(crate) fn sniff_audio(head: &[u8]) -> Option<&'static str> {
let at = |range: std::ops::Range<usize>| head.get(range);
if head.starts_with(b"ID3") {
return Some("audio/mpeg");
}
if head.starts_with(b"OggS") {
return Some("audio/ogg");
}
if head.starts_with(b"fLaC") {
return Some("audio/flac");
}
if at(0..4) == Some(b"RIFF") && at(8..12) == Some(b"WAVE") {
return Some("audio/wav");
}
// MPEG-4 audio brands only: video files are not passed through.
if at(4..8) == Some(b"ftyp")
&& matches!(
at(8..12),
Some(b"M4A " | b"M4B " | b"M4P " | b"F4A " | b"F4B ")
)
{
return Some("audio/mp4");
}
// A bare frame header has few fixed bits, so the whole header must be
// valid. Layer I (`FF FE`, `FF FF`: the UTF-16 byte order mark of text
// files) is never used by podcasts and is not accepted.
if let [0xFF, second, third, ..] = *head {
// ADTS AAC: sync, layer 00, a defined sampling frequency.
if second & 0xF6 == 0xF0 && (third >> 2) & 0b1111 < 13 {
return Some("audio/aac");
}
// MPEG audio frame: sync, a defined version, layer II or III, a
// bitrate index other than free (0) and bad (15), and a defined
// sampling rate.
let version = (second >> 3) & 0b11;
let layer = (second >> 1) & 0b11;
let bitrate = third >> 4;
let sample_rate = (third >> 2) & 0b11;
if second & 0xE0 == 0xE0
&& version != 0b01
&& matches!(layer, 0b01 | 0b10)
&& (1..15).contains(&bitrate)
&& sample_rate != 0b11
{
return Some("audio/mpeg");
}
}
None
}
type Upstream = BoxStream<'static, reqwest::Result<Bytes>>;
/// Reads at least `want` bytes (fewer at the end of the body) from
/// `upstream`; the chunks are sent to the browser afterwards.
async fn read_head(upstream: &mut Upstream, want: usize) -> Result<Vec<Bytes>, ProxyError> {
let mut chunks = Vec::new();
let mut have = 0;
while have < want {
match tokio::time::timeout(PROXY_HEADER_TIMEOUT, upstream.next()).await {
Err(_) => return Err(ProxyError::Timeout),
Ok(None) => break,
Ok(Some(Ok(chunk))) => {
have += chunk.len();
chunks.push(chunk);
}
Ok(Some(Err(error))) => {
return Err(ProxyError::Body(GuardError::from(error).to_string()));
}
}
}
Ok(chunks)
}
fn head_bytes(chunks: &[Bytes]) -> Vec<u8> {
let mut head = Vec::with_capacity(SNIFF_BYTES);
for chunk in chunks {
let take = (SNIFF_BYTES - head.len()).min(chunk.len());
head.extend_from_slice(&chunk[..take]);
if head.len() == SNIFF_BYTES {
break;
}
}
head
}
/// Streams episode enclosures from publishers.
#[derive(Debug, Clone)]
pub(crate) struct AudioProxy {
http: SafeHttp,
max_bytes: u64,
max_duration: Duration,
}
impl AudioProxy {
/// The production proxy (public addresses only).
pub(crate) fn new() -> anyhow::Result<Self> {
Ok(Self::with_http(SafeHttp::public(PROXY_HTTP)?))
}
pub(crate) fn with_http(http: SafeHttp) -> Self {
Self {
http,
max_bytes: MAX_PROXY_BYTES,
max_duration: PROXY_MAX_DURATION,
}
}
#[cfg(test)]
pub(crate) fn with_limits(mut self, max_bytes: u64, max_duration: Duration) -> Self {
self.max_bytes = max_bytes;
self.max_duration = max_duration;
self
}
/// Requests `audio_url` (from the database) with the browser's `Range`
/// (sanitized). A response with a generic binary type and no audio file
/// extension is passed through only when the file starts like audio
/// (read from this response when it starts at byte 0 and is long
/// enough, otherwise from a separate 16-byte range request).
pub(crate) async fn fetch(
&self,
audio_url: &str,
range: Option<&str>,
) -> Result<ProxiedAudio, ProxyError> {
let url =
Url::parse(audio_url).map_err(|error| ProxyError::InvalidUrl(error.to_string()))?;
let mut headers = HeaderMap::new();
headers.insert(ACCEPT, HeaderValue::from_static("audio/*, */*;q=0.1"));
// Byte ranges must address the stored bytes, not a compressed form.
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
let range = range.and_then(sanitize_range);
if let Some(range) = &range {
headers.insert(
RANGE,
HeaderValue::from_str(range).expect("sanitized range is a valid header"),
);
}
let followed = tokio::time::timeout(PROXY_HEADER_TIMEOUT, self.http.get(url, &headers))
.await
.map_err(|_| ProxyError::Timeout)??;
let response = followed.response;
let final_url = followed.final_url;
let status = response.status();
let header = |name| {
response
.headers()
.get(name)
.and_then(|value: &HeaderValue| value.to_str().ok())
.map(str::to_string)
};
let content_range = header(CONTENT_RANGE)
.as_deref()
.and_then(sanitize_content_range);
if status == StatusCode::RANGE_NOT_SATISFIABLE {
return Ok(ProxiedAudio {
status: 416,
content_type: None,
content_length: Some(0),
content_range,
accept_ranges: true,
body: futures_util::stream::empty().boxed(),
});
}
let partial = status == StatusCode::PARTIAL_CONTENT;
// 206 only answers a range we asked for, and must say which bytes.
if !(status == StatusCode::OK || (partial && range.is_some() && content_range.is_some())) {
return Err(ProxyError::Status(status.as_u16()));
}
let declared = header(CONTENT_TYPE);
let check = audio_content_type(declared.as_deref(), &final_url)?;
let content_length = header(CONTENT_LENGTH).and_then(|value| value.parse::<u64>().ok());
if let Some(len) = content_length
&& len > self.max_bytes
{
return Err(ProxyError::TooLarge(len));
}
let accept_ranges = partial
|| header(ACCEPT_RANGES)
.is_some_and(|value| value.trim().eq_ignore_ascii_case("bytes"));
let starts_at_zero = !partial
|| content_range
.as_deref()
.is_some_and(|value| value.starts_with("bytes 0-"));
let mut upstream: Upstream = response.bytes_stream().boxed();
let mut head = Vec::new();
let content_type = match check {
TypeCheck::Audio(content_type) => content_type,
TypeCheck::Content => {
let mut start = Vec::new();
if starts_at_zero {
head = read_head(&mut upstream, SNIFF_BYTES).await?;
start = head_bytes(&head);
}
// A range past the start, or one that ends before the
// sniffed bytes (Safari and AVFoundation probe with
// `bytes=0-1`): the file's head comes from its own request.
if partial && start.len() < SNIFF_BYTES {
start = self.file_head(&final_url).await?;
}
sniff_audio(&start)
.ok_or_else(|| {
ProxyError::ContentType(format!(
"{} (not audio content)",
declared.as_deref().unwrap_or("(none)")
))
})?
.to_string()
}
};
Ok(ProxiedAudio {
status: status.as_u16(),
content_type: Some(content_type),
content_length,
content_range: if partial { content_range } else { None },
accept_ranges,
body: limited_body(head, upstream, self.max_bytes, self.max_duration),
})
}
/// The first [`SNIFF_BYTES`] of the file at `url` (a range request; a
/// server without ranges answers with the whole file, of which only the
/// start is read before the connection is dropped).
async fn file_head(&self, url: &Url) -> Result<Vec<u8>, ProxyError> {
let mut headers = HeaderMap::new();
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
headers.insert(
RANGE,
HeaderValue::from_str(&format!("bytes=0-{}", SNIFF_BYTES - 1)).expect("valid range"),
);
let followed =
tokio::time::timeout(PROXY_HEADER_TIMEOUT, self.http.get(url.clone(), &headers))
.await
.map_err(|_| ProxyError::Timeout)??;
let status = followed.response.status();
if !(status == StatusCode::OK || status == StatusCode::PARTIAL_CONTENT) {
return Err(ProxyError::Status(status.as_u16()));
}
let mut upstream: Upstream = followed.response.bytes_stream().boxed();
Ok(head_bytes(&read_head(&mut upstream, SNIFF_BYTES).await?))
}
}
/// The upstream body (`head`, already read, first), ending with an error
/// once `max_bytes` would be exceeded or `max_duration` has passed.
/// Dropping it (browser gone or seeking elsewhere) closes the upstream
/// connection.
fn limited_body(
head: Vec<Bytes>,
upstream: Upstream,
max_bytes: u64,
max_duration: Duration,
) -> BoxStream<'static, Result<Bytes, ProxyError>> {
let deadline = tokio::time::Instant::now() + max_duration;
let mut upstream = futures_util::stream::iter(head.into_iter().map(Ok)).chain(upstream);
async_stream::stream! {
let mut sent: u64 = 0;
loop {
let next = match tokio::time::timeout_at(deadline, upstream.next()).await {
Ok(next) => next,
Err(_) => {
yield Err(ProxyError::Body("time limit reached".to_string()));
break;
}
};
match next {
None => break,
Some(Ok(chunk)) => {
sent += chunk.len() as u64;
if sent > max_bytes {
yield Err(ProxyError::Body(format!("size limit of {max_bytes} bytes reached")));
break;
}
yield Ok(chunk);
}
Some(Err(error)) => {
yield Err(ProxyError::Body(GuardError::from(error).to_string()));
break;
}
}
}
}
.boxed()
}
+449
View File
@@ -0,0 +1,449 @@
//! Audio proxy: Range pass-through, servers without ranges, redirects,
//! content types and limits, against the local test server.
use std::time::Duration;
use futures_util::StreamExt;
use futures_util::stream::BoxStream;
use super::net::{AddressClass, GuardError, REFUSED_MESSAGE};
use super::proxy::{AudioProxy, ProxyError, sanitize_range, sniff_audio};
use std::sync::Arc;
use super::test_server::{Route, ScriptedLookup, TestServer, noise, test_http};
// ---------------------------------------------------------------------------
// Audio proxy
// ---------------------------------------------------------------------------
#[test]
fn only_single_byte_ranges_are_forwarded() {
assert_eq!(sanitize_range("bytes=0-").as_deref(), Some("bytes=0-"));
assert_eq!(
sanitize_range("bytes=100-199").as_deref(),
Some("bytes=100-199")
);
assert_eq!(
sanitize_range(" bytes=-500 ").as_deref(),
Some("bytes=-500")
);
for invalid in [
"bytes=1-2,5-6",
"items=0-10",
"bytes=5-1",
"bytes=-",
"bytes=-0",
"bytes=a-b",
"bytes=1-2\r\nCookie: x",
"bytes=99999999999999999999999-",
"",
] {
assert_eq!(sanitize_range(invalid), None, "{invalid:?}");
}
}
async fn collect(
mut body: BoxStream<'static, Result<bytes::Bytes, ProxyError>>,
) -> (Vec<u8>, Option<String>) {
let mut out = Vec::new();
while let Some(chunk) = body.next().await {
match chunk {
Ok(chunk) => out.extend_from_slice(&chunk),
Err(error) => return (out, Some(error.to_string())),
}
}
(out, None)
}
fn proxy(lookup: Arc<ScriptedLookup>) -> AudioProxy {
AudioProxy::with_http(test_http(lookup))
}
#[tokio::test]
async fn proxy_passes_ranges_through() {
let server = TestServer::start();
let audio = noise(300_000, 7);
server.set("/ep.mp3", Route::new(audio.clone(), "audio/mpeg"));
let proxy = proxy(ScriptedLookup::with(&[]));
let target = server.url("/ep.mp3");
let full = proxy.fetch(&target, None).await.unwrap();
assert_eq!(full.status, 200);
assert_eq!(full.content_type.as_deref(), Some("audio/mpeg"));
assert_eq!(full.content_length, Some(audio.len() as u64));
assert!(full.accept_ranges);
assert_eq!(full.content_range, None);
assert_eq!(collect(full.body).await, (audio.clone(), None));
for (range, start, end) in [
("bytes=1000-1999", 1000, 2000),
("bytes=299000-", 299_000, 300_000),
("bytes=-500", 299_500, 300_000),
("bytes=0-", 0, 300_000),
] {
let part = proxy.fetch(&target, Some(range)).await.unwrap();
assert_eq!(part.status, 206, "{range}");
assert_eq!(
part.content_range,
Some(format!("bytes {start}-{}/{}", end - 1, audio.len())),
"{range}"
);
assert_eq!(part.content_length, Some((end - start) as u64));
assert_eq!(collect(part.body).await, (audio[start..end].to_vec(), None));
}
let unsatisfiable = proxy.fetch(&target, Some("bytes=400000-")).await.unwrap();
assert_eq!(unsatisfiable.status, 416);
assert_eq!(
unsatisfiable.content_range,
Some(format!("bytes */{}", audio.len()))
);
assert_eq!(collect(unsatisfiable.body).await, (Vec::new(), None));
// Malformed or multi-range requests fetch the whole file.
let whole = proxy.fetch(&target, Some("bytes=5-6,9-10")).await.unwrap();
assert_eq!(whole.status, 200);
let requests = server.requests("/ep.mp3");
let ranges: Vec<Option<&str>> = requests
.iter()
.map(|request| request.get("range").map(String::as_str))
.collect();
assert_eq!(
ranges,
vec![
None,
Some("bytes=1000-1999"),
Some("bytes=299000-"),
Some("bytes=-500"),
Some("bytes=0-"),
Some("bytes=400000-"),
None,
]
);
for request in &requests {
assert_eq!(request["accept-encoding"], "identity");
assert!(!request.contains_key("cookie"));
assert!(!request.contains_key("authorization"));
}
}
#[tokio::test]
async fn proxy_handles_servers_without_ranges_and_redirects() {
let server = TestServer::start();
let audio = noise(50_000, 3);
server.set(
"/plain.mp3",
Route::new(audio.clone(), "audio/mpeg").no_ranges(),
);
server.set("/track/ep.mp3", Route::redirect(302, "/cdn/ep.mp3"));
server.set("/cdn/ep.mp3", Route::new(audio.clone(), "audio/mpeg"));
server.set(
"/track/private.mp3",
Route::redirect(302, "http://172.20.0.3/ep.mp3"),
);
let proxy = proxy(ScriptedLookup::with(&[]));
// No range support: the whole file with 200 and no Accept-Ranges, so the
// browser knows it cannot seek by range.
let plain = proxy
.fetch(&server.url("/plain.mp3"), Some("bytes=100-"))
.await
.unwrap();
assert_eq!(plain.status, 200);
assert!(!plain.accept_ranges);
assert_eq!(plain.content_range, None);
assert_eq!(collect(plain.body).await.0, audio);
// Tracking redirects keep the range on every hop.
let part = proxy
.fetch(&server.url("/track/ep.mp3"), Some("bytes=10-19"))
.await
.unwrap();
assert_eq!(part.status, 206);
assert_eq!(collect(part.body).await.0, audio[10..20].to_vec());
assert_eq!(server.requests("/track/ep.mp3")[0]["range"], "bytes=10-19");
assert_eq!(server.requests("/cdn/ep.mp3")[0]["range"], "bytes=10-19");
let Err(error) = proxy.fetch(&server.url("/track/private.mp3"), None).await else {
panic!("redirect to a private address was followed");
};
assert!(
matches!(
&error,
ProxyError::Upstream(GuardError::Blocked(blocked))
if blocked.class == AddressClass::Private
),
"{error:?}"
);
// An IP literal is refused for what the URL says.
assert_eq!(error.public_message(), REFUSED_MESSAGE);
}
#[tokio::test]
async fn proxy_answers_alike_for_internal_and_unreachable_names() {
// Episode URLs can come from the user's own clients: the answer must
// not tell a name with a private address from one that does not exist
// or a server that refuses the connection.
let port = std::net::TcpListener::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap()
.port();
let proxy = proxy(ScriptedLookup::with(&[
("db.corp.test", &[&["192.168.7.7"]]),
("closed.test", &[&["127.0.0.1"]]),
]));
let mut answers = Vec::new();
for host in ["db.corp.test", "nowhere.corp.test", "closed.test"] {
let target = format!("http://{host}:{port}/episode.mp3");
let error = proxy.fetch(&target, None).await.unwrap_err();
assert!(matches!(error, ProxyError::Upstream(_)), "{error:?}");
answers.push((error.http_status(), error.public_message()));
}
assert_eq!(answers[0].1, "could not connect to the publisher");
assert!(
answers.iter().all(|answer| *answer == answers[0]),
"{answers:?}"
);
}
/// `noise` that starts like a file of the given format.
fn audio_file(magic: &[u8], len: usize) -> Vec<u8> {
let mut file = magic.to_vec();
file.extend(noise(len - magic.len(), 3));
file
}
#[tokio::test]
async fn proxy_only_serves_audio() {
let server = TestServer::start();
let audio = noise(1000, 5);
let mp3 = audio_file(b"ID3\x04\0\0", 1000);
server.set(
"/page.mp3",
Route::new("<script>alert(1)</script>", "text/html"),
);
server.set("/video.mp4", Route::new(audio.clone(), "video/mp4"));
// Generic types: the URL extension, or else the file's first bytes.
server.set(
"/binary",
Route::new(audio.clone(), "application/octet-stream"),
);
server.set(
"/zip",
Route::new(audio_file(b"PK\x03\x04", 1000), "application/octet-stream"),
);
server.set(
"/episode",
Route::new(mp3.clone(), "application/octet-stream"),
);
server.set("/untyped", Route::new(mp3.clone(), "x").no_content_type());
server.set(
"/binary.m4a",
Route::new(audio.clone(), "binary/octet-stream"),
);
server.set(
"/untyped.ogg",
Route::new(audio.clone(), "x").no_content_type(),
);
server.set("/ogg", Route::new(audio.clone(), "application/ogg"));
server.set(
"/typed",
Route::new(audio.clone(), "audio/mp4; codecs=\"mp4a.40.2\""),
);
server.set("/missing", Route::new("", "text/plain").status(404));
server.set("/error", Route::new("", "audio/mpeg").status(500));
let proxy = proxy(ScriptedLookup::with(&[]));
let content_type = |path: &'static str, range: Option<&'static str>| {
let proxy = proxy.clone();
let target = server.url(path);
async move {
proxy
.fetch(&target, range)
.await
.map(|audio| audio.content_type.unwrap())
}
};
for refused in ["/page.mp3", "/video.mp4", "/binary", "/zip"] {
for range in [None, Some("bytes=10-")] {
assert!(
matches!(
content_type(refused, range).await,
Err(ProxyError::ContentType(_))
),
"{refused} {range:?}"
);
}
}
assert_eq!(content_type("/episode", None).await.unwrap(), "audio/mpeg");
assert_eq!(content_type("/untyped", None).await.unwrap(), "audio/mpeg");
// The sniffed bytes are still sent to the browser.
let whole = proxy.fetch(&server.url("/episode"), None).await.unwrap();
assert_eq!(collect(whole.body).await, (mp3.clone(), None));
// A range past the start: the file's head is read with its own request.
let before = server.requests("/episode").len();
let part = proxy
.fetch(&server.url("/episode"), Some("bytes=100-199"))
.await
.unwrap();
assert_eq!(part.content_type.as_deref(), Some("audio/mpeg"));
assert_eq!(collect(part.body).await, (mp3[100..200].to_vec(), None));
let ranges: Vec<String> = server.requests("/episode")[before..]
.iter()
.map(|request| request["range"].clone())
.collect();
assert_eq!(ranges, ["bytes=100-199", "bytes=0-15"]);
// Safari and AVFoundation probe with a two-byte range from the start:
// too short to recognize the file, so its head is read separately.
let before = server.requests("/episode").len();
let probe = proxy
.fetch(&server.url("/episode"), Some("bytes=0-1"))
.await
.unwrap();
assert_eq!(probe.status, 206);
assert_eq!(probe.content_type.as_deref(), Some("audio/mpeg"));
assert_eq!(collect(probe.body).await, (mp3[..2].to_vec(), None));
let ranges: Vec<String> = server.requests("/episode")[before..]
.iter()
.map(|request| request["range"].clone())
.collect();
assert_eq!(ranges, ["bytes=0-1", "bytes=0-15"]);
// A long enough range from the start needs no second request.
let before = server.requests("/episode").len();
let start = proxy
.fetch(&server.url("/episode"), Some("bytes=0-99"))
.await
.unwrap();
assert_eq!(start.content_type.as_deref(), Some("audio/mpeg"));
assert_eq!(collect(start.body).await, (mp3[..100].to_vec(), None));
assert_eq!(server.requests("/episode").len(), before + 1);
// A probe of a file that is not audio is still refused.
assert!(matches!(
proxy.fetch(&server.url("/zip"), Some("bytes=0-1")).await,
Err(ProxyError::ContentType(_))
));
assert_eq!(
content_type("/binary.m4a", None).await.unwrap(),
"audio/mp4"
);
assert_eq!(
content_type("/untyped.ogg", None).await.unwrap(),
"audio/ogg"
);
assert_eq!(content_type("/ogg", None).await.unwrap(), "application/ogg");
assert_eq!(
content_type("/typed", None).await.unwrap(),
"audio/mp4; codecs=\"mp4a.40.2\""
);
let missing = content_type("/missing", None).await.unwrap_err();
assert!(matches!(missing, ProxyError::Status(404)));
assert_eq!(missing.http_status(), 502);
assert!(matches!(
content_type("/error", None).await,
Err(ProxyError::Status(500))
));
assert!(matches!(
proxy.fetch("not a url", None).await,
Err(ProxyError::InvalidUrl(_))
));
}
#[test]
fn audio_formats_are_recognized_by_their_first_bytes() {
for (head, format) in [
(&b"ID3\x04\0\0\0\0"[..], "audio/mpeg"),
(&[0xFF, 0xFB, 0x90, 0x64][..], "audio/mpeg"),
(&[0xFF, 0xF3, 0x48, 0xC4][..], "audio/mpeg"),
(&[0xFF, 0xFD, 0x94, 0x00][..], "audio/mpeg"),
(&[0xFF, 0xE3, 0x18, 0xC4][..], "audio/mpeg"),
(&[0xFF, 0xF9, 0x4C, 0x80][..], "audio/aac"),
(&[0xFF, 0xF1, 0x50, 0x80][..], "audio/aac"),
(&b"OggS\0\x02"[..], "audio/ogg"),
(&b"fLaC\0\0\0\x22"[..], "audio/flac"),
(&b"RIFF\x24\x08\0\0WAVEfmt "[..], "audio/wav"),
(&b"\0\0\0\x20ftypM4A \0\0\0\0"[..], "audio/mp4"),
] {
assert_eq!(sniff_audio(head), Some(format), "{head:?}");
}
for head in [
&b"\0\0\0\x20ftypisom\0\0\x02\0"[..],
&b"\0\0\0\x18ftypmp42"[..],
&b"PK\x03\x04"[..],
&b"<!DOCTYPE html>"[..],
&b"%PDF-1.7"[..],
&b"RIFF\x24\x08\0\0AVI "[..],
&[0xFF, 0xD8, 0xFF, 0xE0][..],
// UTF-16LE text (byte order mark FF FE) and FF FF are not frames.
&[0xFF, 0xFE, 0x3C, 0x00, 0x68, 0x00][..],
&[0xFF, 0xFE, 0x41, 0x00][..],
&[0xFF, 0xFF, 0x90, 0x00][..],
// Frame sync with an invalid bitrate, free bitrate or sample rate.
&[0xFF, 0xFB, 0xF0, 0x00][..],
&[0xFF, 0xFB, 0x00, 0x00][..],
&[0xFF, 0xFB, 0x9C, 0x00][..],
// ADTS sync with a reserved sampling frequency index.
&[0xFF, 0xF1, 0x3C, 0x80][..],
&[0xFF, 0xFB][..],
&[0xFF][..],
&[][..],
] {
assert_eq!(sniff_audio(head), None, "{head:?}");
}
}
#[tokio::test]
async fn proxy_limits_size_and_reports_dropped_streams() {
let server = TestServer::start();
let audio = noise(100_000, 9);
server.set("/sized.mp3", Route::new(audio.clone(), "audio/mpeg"));
server.set(
"/chunked.mp3",
Route::new(audio.clone(), "audio/mpeg").no_length(),
);
server.set(
"/cut.mp3",
Route::new(audio.clone(), "audio/mpeg").cut(40_000),
);
let limited = proxy(ScriptedLookup::with(&[])).with_limits(60_000, Duration::from_secs(60));
assert!(matches!(
limited.fetch(&server.url("/sized.mp3"), None).await,
Err(ProxyError::TooLarge(100_000))
));
// A range below the limit is fine.
let part = limited
.fetch(&server.url("/sized.mp3"), Some("bytes=0-49999"))
.await
.unwrap();
assert_eq!(collect(part.body).await, (audio[..50_000].to_vec(), None));
// Without a declared length the stream stops at the limit.
let chunked = limited
.fetch(&server.url("/chunked.mp3"), None)
.await
.unwrap();
let (received, error) = collect(chunked.body).await;
assert!(received.len() <= 60_000);
assert!(error.unwrap().contains("size limit"));
// A connection dropped mid-body ends the stream with an error instead
// of a silently truncated success.
let cut = proxy(ScriptedLookup::with(&[]))
.fetch(&server.url("/cut.mp3"), None)
.await
.unwrap();
let (received, error) = collect(cut.body).await;
assert!(received.len() <= 40_000);
assert!(error.is_some());
}
#[tokio::test]
async fn proxy_stream_has_a_time_limit() {
let server = TestServer::start();
server.set(
"/slow.mp3",
Route::new(noise(10, 1), "audio/mpeg").delay_body(Duration::from_secs(3)),
);
let limited = proxy(ScriptedLookup::with(&[])).with_limits(1 << 20, Duration::from_millis(300));
let audio = limited.fetch(&server.url("/slow.mp3"), None).await.unwrap();
let (_, error) = collect(audio.body).await;
assert!(error.unwrap().contains("time limit"));
}
+723
View File
@@ -0,0 +1,723 @@
//! Read-only queries for the podcast UI and the player.
use anyhow::Result;
use music_dht::podcast::{
EpisodeId, EpisodeMeta, EpisodeType, ListenSession, PodcastEpisodeRef, PodcastId, PodcastMeta,
ProgressRecord, ProgressStatus, SubscriptionRecord, effective_share,
};
use sqlx::postgres::PgRow;
use sqlx::{PgConnection, Postgres, QueryBuilder, Row};
use super::apply;
use super::{
PodcastStore, SESSION_COLUMNS, episode_id_col, episode_search_norm, feed_url_col,
podcast_id_col, progress_columns, progress_from_row, session_from_row, u32_from_sql,
};
/// Which podcasts [`PodcastStore::list_podcasts`] returns.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum PodcastListFilter {
/// Podcasts with an active subscription.
#[default]
Subscribed,
/// Every podcast in the catalog, including unsubscribed ones and
/// podcasts only known from synced progress or queue items.
All,
}
/// A podcast with its personal state and episode counters.
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct PodcastSummary {
/// Podcast id.
pub(crate) podcast_id: PodcastId,
/// Local channel metadata; `feed_url` is the URL this account fetches.
pub(crate) meta: PodcastMeta,
/// Whether the subscription register says subscribed.
pub(crate) subscribed: bool,
/// Whether the podcast must currently be published (`effective_share`).
pub(crate) shared: bool,
/// Episodes in the catalog.
pub(crate) episode_count: u64,
/// Episodes not completed (no progress, not started or in progress).
pub(crate) unplayed_count: u64,
/// Episodes in progress.
pub(crate) in_progress_count: u64,
/// Newest publication time.
pub(crate) latest_published_at_ms: Option<i64>,
/// Numbered seasons (episodes without a season are not counted).
pub(crate) season_count: u64,
}
/// One season group of a podcast. `season == None` groups the episodes
/// without `itunes:season`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct SeasonSummary {
/// Season number; `None` for episodes without a season.
pub(crate) season: Option<u32>,
/// Episodes in the group.
pub(crate) episode_count: u64,
/// Episodes not completed.
pub(crate) unplayed_count: u64,
/// Oldest publication time.
pub(crate) first_published_at_ms: Option<i64>,
/// Newest publication time.
pub(crate) last_published_at_ms: Option<i64>,
}
/// Season filter of an [`EpisodeQuery`].
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum SeasonFilter {
/// All episodes.
#[default]
Any,
/// Episodes of one season.
Season(u32),
/// Episodes without a season.
NoSeason,
}
/// Listening-state filter of an [`EpisodeQuery`].
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum PlayedFilter {
/// All episodes.
#[default]
Any,
/// Not completed (not started or in progress).
Unplayed,
/// Never started or marked unplayed.
NotStarted,
/// In progress.
InProgress,
/// Completed.
Played,
}
/// Episode ordering of an [`EpisodeQuery`].
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) enum EpisodeSort {
/// Newest publication first; undated episodes last.
#[default]
NewestFirst,
/// Oldest publication first; undated episodes last.
OldestFirst,
/// By season, then episode number, ascending; unnumbered last.
EpisodeNumber,
/// By season, then episode number, descending; unnumbered last.
EpisodeNumberDesc,
}
impl EpisodeSort {
fn order_by(self) -> &'static str {
match self {
Self::NewestFirst => "e.published_at_ms IS NULL, e.published_at_ms DESC, e.episode_id",
Self::OldestFirst => "e.published_at_ms IS NULL, e.published_at_ms, e.episode_id",
Self::EpisodeNumber => {
"e.season IS NULL, e.season, e.episode_number IS NULL, e.episode_number, \
e.published_at_ms IS NULL, e.published_at_ms, e.episode_id"
}
Self::EpisodeNumberDesc => {
"e.season IS NULL, e.season DESC, e.episode_number IS NULL, \
e.episode_number DESC, e.published_at_ms IS NULL, e.published_at_ms DESC, \
e.episode_id"
}
}
}
}
#[cfg(test)]
pub(crate) fn order_by_for_tests(sort: EpisodeSort) -> &'static str {
sort.order_by()
}
/// Parameters of [`PodcastStore::episodes`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct EpisodeQuery {
/// Podcast.
pub(crate) podcast_id: PodcastId,
/// Season filter.
pub(crate) season: SeasonFilter,
/// Listening-state filter.
pub(crate) played: PlayedFilter,
/// Text searched in titles and descriptions (`normalize_name`: Unicode
/// NFKC, case-insensitive substring match). Blank means no filter.
pub(crate) search: Option<String>,
/// Ordering.
pub(crate) sort: EpisodeSort,
/// Maximum rows (`None`: all).
pub(crate) limit: Option<usize>,
/// Rows to skip.
pub(crate) offset: usize,
}
impl EpisodeQuery {
/// All episodes of a podcast, newest first.
pub(crate) fn new(podcast_id: PodcastId) -> Self {
Self {
podcast_id,
season: SeasonFilter::Any,
played: PlayedFilter::Any,
search: None,
sort: EpisodeSort::NewestFirst,
limit: None,
offset: 0,
}
}
}
/// An episode with its personal state.
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct EpisodeListItem {
/// Episode metadata (feed or synced reference).
pub(crate) episode: EpisodeMeta,
/// Progress register, if any.
pub(crate) progress: Option<ProgressRecord>,
/// Distinct completed listening sessions (`completed_listen_count`).
pub(crate) completed_count: u64,
/// Whether the latest refresh of the feed listed the episode.
pub(crate) in_feed: bool,
}
impl EpisodeListItem {
/// Listening status (`NotStarted` without a register).
pub(crate) fn status(&self) -> ProgressStatus {
self.progress
.as_ref()
.map_or(ProgressStatus::NotStarted, |progress| progress.status)
}
/// Position ordinary Play resumes from.
pub(crate) fn resume_position_ms(&self) -> i64 {
self.progress
.as_ref()
.map_or(0, ProgressRecord::resume_position_ms)
}
}
/// A listening session in history, with display titles when known.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct PodcastListenEntry {
/// The session.
pub(crate) session: ListenSession,
/// Episode title from the catalog.
pub(crate) episode_title: Option<String>,
/// Podcast title from the catalog.
pub(crate) podcast_title: Option<String>,
}
const EPISODE_COLUMNS: &str = "e.episode_id, e.podcast_id, e.guid, e.title, e.description, \
e.season, e.episode_number, e.episode_type, e.published_at_ms, e.duration_ms, \
e.audio_url, e.mime, e.byte_len, e.image_url, e.link";
const NOT_COMPLETED: &str = "COALESCE(g.status, 'not_started') <> 'completed'";
fn episode_from_row(row: &PgRow) -> sqlx::Result<EpisodeMeta> {
Ok(EpisodeMeta {
episode_id: episode_id_col(row, "episode_id")?,
podcast_id: podcast_id_col(row, "podcast_id")?,
guid: row.try_get("guid")?,
title: row.try_get("title")?,
description: row.try_get("description")?,
season: u32_from_sql(row.try_get("season")?),
episode_number: u32_from_sql(row.try_get("episode_number")?),
episode_type: EpisodeType::parse(&row.try_get::<String, _>("episode_type")?),
published_at_ms: row.try_get("published_at_ms")?,
duration_ms: row.try_get("duration_ms")?,
audio_url: row.try_get("audio_url")?,
mime: row.try_get("mime")?,
byte_len: row
.try_get::<Option<i64>, _>("byte_len")?
.and_then(|len| u64::try_from(len).ok()),
image_url: row.try_get("image_url")?,
link: row.try_get("link")?,
})
}
fn list_item_from_row(row: &PgRow) -> sqlx::Result<EpisodeListItem> {
let progress = if row.try_get::<Option<String>, _>("g_episode_id")?.is_some() {
Some(progress_from_row(row, "g_")?)
} else {
None
};
Ok(EpisodeListItem {
episode: episode_from_row(row)?,
in_feed: row.try_get("in_feed")?,
progress,
completed_count: count(row.try_get("completed_count")?),
})
}
fn count(value: i64) -> u64 {
u64::try_from(value).unwrap_or(0)
}
pub(crate) async fn episode_meta_in(
conn: &mut PgConnection,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<EpisodeMeta>> {
let row = sqlx::query(&format!(
"SELECT {EPISODE_COLUMNS} FROM furumusic__podcast_episode e
WHERE e.user_id = $1 AND e.episode_id = $2"
))
.bind(user_id)
.bind(episode_id.as_str())
.fetch_optional(&mut *conn)
.await?;
Ok(row.as_ref().map(episode_from_row).transpose()?)
}
pub(crate) async fn podcast_meta_in(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<PodcastMeta>> {
let row = sqlx::query(
"SELECT feed_url, title, author, description, link, image_url
FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
let Some(row) = row else {
return Ok(None);
};
Ok(Some(PodcastMeta {
feed_url: feed_url_col(&row, "feed_url")?,
title: row.try_get("title")?,
author: row.try_get("author")?,
description: row.try_get("description")?,
link: row.try_get("link")?,
image_url: row.try_get("image_url")?,
}))
}
/// Queue reference of a catalog episode, with `progress_generation` set to
/// the generation of its progress register (handoff hint).
pub(crate) async fn episode_ref_in(
conn: &mut PgConnection,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<PodcastEpisodeRef>> {
let Some(episode) = episode_meta_in(conn, user_id, episode_id).await? else {
return Ok(None);
};
let Some(podcast) = podcast_meta_in(conn, user_id, &episode.podcast_id).await? else {
return Ok(None);
};
let mut reference = episode.to_ref(&podcast);
reference.progress_generation = apply::load_progress(conn, user_id, episode_id)
.await?
.map(|progress| progress.generation);
Ok(Some(reference))
}
async fn summary_in(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<PodcastSummary>> {
let Some(meta) = podcast_meta_in(conn, user_id, podcast_id).await? else {
return Ok(None);
};
let row = sqlx::query(&format!(
"SELECT COUNT(*) AS episode_count,
COUNT(*) FILTER (WHERE {NOT_COMPLETED}) AS unplayed_count,
COUNT(*) FILTER (WHERE g.status = 'in_progress') AS in_progress_count,
MAX(e.published_at_ms) AS latest,
(SELECT COUNT(DISTINCT e2.season) FROM furumusic__podcast_episode e2
WHERE e2.user_id = $1 AND e2.podcast_id = $2) AS season_count
FROM furumusic__podcast_episode e
LEFT JOIN furumusic__podcast_progress g
ON g.user_id = e.user_id AND g.episode_id = e.episode_id
WHERE e.user_id = $1 AND e.podcast_id = $2"
))
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_one(&mut *conn)
.await?;
let subscription = apply::load_subscription_register(conn, user_id, podcast_id).await?;
let share = apply::load_share(conn, user_id, podcast_id).await?;
Ok(Some(PodcastSummary {
podcast_id: podcast_id.clone(),
meta,
subscribed: subscription
.as_ref()
.is_some_and(|record| record.subscribed),
shared: effective_share(subscription.as_ref(), share.as_ref()),
episode_count: count(row.try_get("episode_count")?),
unplayed_count: count(row.try_get("unplayed_count")?),
in_progress_count: count(row.try_get("in_progress_count")?),
latest_published_at_ms: row.try_get("latest")?,
season_count: count(row.try_get("season_count")?),
}))
}
/// Computes the cached search text of episodes that do not have it yet
/// (rows imported by the `m_0057` migration).
async fn backfill_search_norm(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<()> {
let rows = sqlx::query(
"SELECT episode_id, title, description FROM furumusic__podcast_episode
WHERE user_id = $1 AND podcast_id = $2 AND search_norm IS NULL",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_all(&mut *conn)
.await?;
for row in rows {
let title: String = row.try_get("title")?;
let description: Option<String> = row.try_get("description")?;
sqlx::query(
"UPDATE furumusic__podcast_episode SET search_norm = $3
WHERE user_id = $1 AND episode_id = $2 AND search_norm IS NULL",
)
.bind(user_id)
.bind(row.try_get::<String, _>("episode_id")?)
.bind(episode_search_norm(&title, description.as_deref()))
.execute(&mut *conn)
.await?;
}
Ok(())
}
impl PodcastStore {
/// Podcasts with subscription state and counters, sorted by title.
pub(crate) async fn list_podcasts(
&self,
user_id: i64,
filter: PodcastListFilter,
) -> Result<Vec<PodcastSummary>> {
let mut conn = self.pool.acquire().await?;
let join = match filter {
PodcastListFilter::Subscribed => {
"JOIN furumusic__podcast_subscription r
ON r.user_id = p.user_id AND r.podcast_id = p.podcast_id AND r.subscribed"
}
PodcastListFilter::All => "",
};
let rows = sqlx::query(&format!(
"SELECT p.podcast_id FROM furumusic__podcast p {join}
WHERE p.user_id = $1
ORDER BY lower(p.title), p.podcast_id"
))
.bind(user_id)
.fetch_all(&mut *conn)
.await?;
let ids = rows
.iter()
.map(|row| podcast_id_col(row, "podcast_id"))
.collect::<sqlx::Result<Vec<_>>>()?;
let mut out = Vec::with_capacity(ids.len());
for id in ids {
if let Some(summary) = summary_in(&mut conn, user_id, &id).await? {
out.push(summary);
}
}
Ok(out)
}
/// One podcast with subscription state and counters.
pub(crate) async fn podcast(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<PodcastSummary>> {
let mut conn = self.pool.acquire().await?;
summary_in(&mut conn, user_id, podcast_id).await
}
/// The subscription register (with the stored alias set), if any.
pub(crate) async fn subscription(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<SubscriptionRecord>> {
let mut conn = self.pool.acquire().await?;
apply::load_subscription_register(&mut conn, user_id, podcast_id).await
}
/// The share register, if any.
#[cfg(test)]
pub(crate) async fn share(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<music_dht::podcast::ShareRecord>> {
let mut conn = self.pool.acquire().await?;
apply::load_share(&mut conn, user_id, podcast_id).await
}
/// The bounded, synced feed alias set of a podcast.
#[cfg(test)]
pub(crate) async fn feed_aliases(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<music_dht::podcast::FeedUrl>> {
let mut conn = self.pool.acquire().await?;
apply::load_aliases(&mut conn, user_id, podcast_id).await
}
/// Season groups of a podcast: numbered seasons ascending, then the
/// group of episodes without a season.
pub(crate) async fn seasons(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Vec<SeasonSummary>> {
let rows = sqlx::query(&format!(
"SELECT s.season, s.episode_count, s.first_published_at_ms, s.last_published_at_ms,
(SELECT COUNT(*) FROM furumusic__podcast_episode e
LEFT JOIN furumusic__podcast_progress g
ON g.user_id = e.user_id AND g.episode_id = e.episode_id
WHERE e.user_id = s.user_id AND e.podcast_id = s.podcast_id
AND e.season IS NOT DISTINCT FROM s.season
AND {NOT_COMPLETED}) AS unplayed_count
FROM furumusic__podcast_season s
WHERE s.user_id = $1 AND s.podcast_id = $2
ORDER BY s.season IS NULL, s.season"
))
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_all(&self.pool)
.await?;
let seasons = rows
.iter()
.map(|row| {
Ok(SeasonSummary {
season: u32_from_sql(row.try_get("season")?),
episode_count: count(row.try_get("episode_count")?),
first_published_at_ms: row.try_get("first_published_at_ms")?,
last_published_at_ms: row.try_get("last_published_at_ms")?,
unplayed_count: count(row.try_get("unplayed_count")?),
})
})
.collect::<sqlx::Result<Vec<_>>>()?;
Ok(seasons)
}
/// Episodes of a podcast or season with their progress.
pub(crate) async fn episodes(
&self,
user_id: i64,
query: &EpisodeQuery,
) -> Result<Vec<EpisodeListItem>> {
let mut conn = self.pool.acquire().await?;
let needle = query
.search
.as_deref()
.map(music_dht::normalize_name)
.filter(|needle| !needle.is_empty());
if needle.is_some() {
backfill_search_norm(&mut conn, user_id, &query.podcast_id).await?;
}
let mut sql: QueryBuilder<'_, Postgres> = QueryBuilder::new(format!(
"SELECT {EPISODE_COLUMNS}, e.in_feed, {},
(SELECT COUNT(*) FROM furumusic__podcast_listen_session s
WHERE s.user_id = e.user_id AND s.episode_id = e.episode_id
AND s.completed) AS completed_count
FROM furumusic__podcast_episode e
LEFT JOIN furumusic__podcast_progress g
ON g.user_id = e.user_id AND g.episode_id = e.episode_id
WHERE e.user_id = ",
progress_columns("g", "g_")
));
sql.push_bind(user_id);
sql.push(" AND e.podcast_id = ");
sql.push_bind(query.podcast_id.as_str().to_string());
match query.season {
SeasonFilter::Any => {}
SeasonFilter::Season(season) => {
sql.push(" AND e.season = ");
sql.push_bind(i64::from(season));
}
SeasonFilter::NoSeason => {
sql.push(" AND e.season IS NULL");
}
}
match query.played {
PlayedFilter::Any => {}
PlayedFilter::Unplayed => {
sql.push(format!(" AND {NOT_COMPLETED}"));
}
PlayedFilter::NotStarted => {
sql.push(" AND COALESCE(g.status, 'not_started') = 'not_started'");
}
PlayedFilter::InProgress => {
sql.push(" AND g.status = 'in_progress'");
}
PlayedFilter::Played => {
sql.push(" AND g.status = 'completed'");
}
}
if let Some(needle) = needle {
sql.push(" AND strpos(e.search_norm, ");
sql.push_bind(needle);
sql.push(") > 0");
}
sql.push(format!(" ORDER BY {}", query.sort.order_by()));
if let Some(limit) = query.limit {
sql.push(" LIMIT ");
sql.push_bind(i64::try_from(limit).unwrap_or(i64::MAX));
}
sql.push(" OFFSET ");
sql.push_bind(i64::try_from(query.offset).unwrap_or(i64::MAX));
let rows = sql.build().fetch_all(&mut *conn).await?;
Ok(rows
.iter()
.map(list_item_from_row)
.collect::<sqlx::Result<Vec<_>>>()?)
}
/// One episode with its progress.
pub(crate) async fn episode(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<EpisodeListItem>> {
let row = sqlx::query(&format!(
"SELECT {EPISODE_COLUMNS}, e.in_feed, {},
(SELECT COUNT(*) FROM furumusic__podcast_listen_session s
WHERE s.user_id = e.user_id AND s.episode_id = e.episode_id
AND s.completed) AS completed_count
FROM furumusic__podcast_episode e
LEFT JOIN furumusic__podcast_progress g
ON g.user_id = e.user_id AND g.episode_id = e.episode_id
WHERE e.user_id = $1 AND e.episode_id = $2",
progress_columns("g", "g_")
))
.bind(user_id)
.bind(episode_id.as_str())
.fetch_optional(&self.pool)
.await?;
Ok(row.as_ref().map(list_item_from_row).transpose()?)
}
/// Position ordinary Play resumes from
/// (`ProgressRecord::resume_position_ms`, 0 without progress).
#[cfg(test)]
pub(crate) async fn resume_position_ms(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<i64> {
Ok(self
.progress(user_id, episode_id)
.await?
.map_or(0, |progress| progress.resume_position_ms()))
}
/// Listening history (sessions that `should_record`), newest first.
pub(crate) async fn listen_history(
&self,
user_id: i64,
podcast_id: Option<&PodcastId>,
limit: usize,
) -> Result<Vec<PodcastListenEntry>> {
let rows = sqlx::query(&format!(
"SELECT {SESSION_COLUMNS}, e.title AS episode_title, p.title AS podcast_title
FROM furumusic__podcast_listen_session s
LEFT JOIN furumusic__podcast_episode e
ON e.user_id = s.user_id AND e.episode_id = s.episode_id
LEFT JOIN furumusic__podcast p
ON p.user_id = s.user_id AND p.podcast_id = s.podcast_id
WHERE s.user_id = $1 AND (s.completed OR s.listened_ms >= $2)
AND ($3::text IS NULL OR s.podcast_id = $3)
ORDER BY s.ended_at_ms DESC, s.session_id DESC
LIMIT $4"
))
.bind(user_id)
.bind(music_dht::podcast::MIN_RECORDED_EPISODE_LISTEN_MS)
.bind(podcast_id.map(PodcastId::as_str))
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await?;
Ok(rows
.iter()
.map(|row| {
Ok(PodcastListenEntry {
session: session_from_row(row)?,
episode_title: row.try_get("episode_title")?,
podcast_title: row.try_get("podcast_title")?,
})
})
.collect::<sqlx::Result<Vec<_>>>()?)
}
/// Number of distinct completed sessions of an episode.
#[cfg(test)]
pub(crate) async fn completed_listen_count(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<u64> {
let total: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM furumusic__podcast_listen_session
WHERE user_id = $1 AND episode_id = $2 AND completed",
)
.bind(user_id)
.bind(episode_id.as_str())
.fetch_one(&self.pool)
.await?;
Ok(count(total))
}
/// Generations of the progress registers of `episode_ids` (episodes
/// without a register are missing from the map).
///
/// Outgoing playback states carry them as
/// `PodcastEpisodeRef::progress_generation`, so the device that takes
/// over writes its explicit action after everything this account saw.
pub(crate) async fn progress_generations(
&self,
user_id: i64,
episode_ids: &[EpisodeId],
) -> Result<std::collections::HashMap<EpisodeId, u64>> {
if episode_ids.is_empty() {
return Ok(Default::default());
}
let ids: Vec<&str> = episode_ids.iter().map(EpisodeId::as_str).collect();
let rows = sqlx::query(
"SELECT episode_id, generation FROM furumusic__podcast_progress
WHERE user_id = $1 AND episode_id = ANY($2)",
)
.bind(user_id)
.bind(&ids)
.fetch_all(&self.pool)
.await?;
rows.iter()
.map(|row| {
Ok((
episode_id_col(row, "episode_id")?,
super::generation_from_sql(row.try_get("generation")?),
))
})
.collect::<sqlx::Result<_>>()
.map_err(Into::into)
}
/// Audio URL and enclosure type of an episode in this account's catalog
/// (`None` when the account does not know the episode). The audio proxy
/// and the playback source endpoint take URLs only from here.
pub(crate) async fn audio_source(
&self,
user_id: i64,
episode_id: &EpisodeId,
) -> Result<Option<(String, Option<String>)>> {
let row = sqlx::query(
"SELECT audio_url, mime FROM furumusic__podcast_episode
WHERE user_id = $1 AND episode_id = $2",
)
.bind(user_id)
.bind(episode_id.as_str())
.fetch_optional(&self.pool)
.await?;
Ok(row
.map(|row| Ok::<_, sqlx::Error>((row.try_get("audio_url")?, row.try_get("mime")?)))
.transpose()?)
}
}
+498
View File
@@ -0,0 +1,498 @@
//! Feed refresh ingestion and the local-only fetch state.
use std::collections::BTreeSet;
use anyhow::Result;
use music_dht::device_sync::{SyncOpPayload, SyncOpWire};
use music_dht::podcast::feed::ParsedFeed;
use music_dht::podcast::{EpisodeMeta, FeedUrl, PodcastId, merge_feed_aliases};
use sqlx::{PgConnection, Row};
use super::apply::{self, EpisodeSource};
use super::{PodcastStore, PodcastStoreError, episode_search_norm, now_ms};
use crate::federation::devices;
/// A fetched and parsed feed document for [`PodcastStore::apply_feed`].
#[derive(Debug, Clone, Copy)]
pub(crate) struct FeedRefresh<'a> {
/// Podcast the feed belongs to.
pub(crate) podcast_id: &'a PodcastId,
/// The parsed document (`music_dht::podcast::feed::parse_feed_with_base`
/// with `fetched_from` as base).
pub(crate) feed: &'a ParsedFeed,
/// URL the document was finally served from, after redirects.
pub(crate) fetched_from: &'a FeedUrl,
/// Whether every redirect leading to `fetched_from` was permanent
/// (301/308). Temporary redirects do not move the feed.
pub(crate) moved_permanently: bool,
}
/// Result of [`PodcastStore::apply_feed`].
#[derive(Debug, Clone, Default, PartialEq)]
pub(crate) struct RefreshOutcome {
/// Episodes seen for the first time.
pub(crate) new_episodes: usize,
/// Known episodes whose metadata changed (title, audio URL, ...).
pub(crate) updated_episodes: usize,
/// Known episodes without metadata changes.
pub(crate) unchanged_episodes: usize,
/// Items without identity or with invalid metadata.
pub(crate) skipped_items: usize,
/// Feed URLs learned from a permanent redirect or `itunes:new-feed-url`.
pub(crate) learned_aliases: Vec<FeedUrl>,
/// URL the podcast is fetched from after this refresh.
pub(crate) feed_url: Option<FeedUrl>,
/// `podcast_feed_aliases_added`, only when the learned URLs changed the
/// stored alias set (already appended to the sync log).
pub(crate) ops: Vec<SyncOpWire>,
}
/// Local-only fetch state of a podcast. Never part of sync output.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct FetchState {
/// HTTP `ETag` validator of the last successful fetch.
pub(crate) etag: Option<String>,
/// HTTP `Last-Modified` validator of the last successful fetch.
pub(crate) last_modified: Option<String>,
/// Last fetch attempt, unix milliseconds.
pub(crate) checked_at_ms: Option<i64>,
/// Last successful fetch (including "not modified").
pub(crate) last_success_at_ms: Option<i64>,
/// When the scheduler should check again (`None`: due now).
pub(crate) next_check_at_ms: Option<i64>,
/// Error of the last attempt; `None` after a success.
pub(crate) last_error: Option<String>,
/// Failed attempts since the last success.
pub(crate) consecutive_failures: u32,
/// Image URL the cached cover was downloaded from.
pub(crate) cover_url: Option<String>,
/// Local path of the cached cover.
pub(crate) cover_path: Option<String>,
}
async fn podcast_fetch_url(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<Option<FeedUrl>> {
let row = sqlx::query(
"SELECT feed_url FROM furumusic__podcast WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
Ok(row
.as_ref()
.map(|row| super::feed_url_col(row, "feed_url"))
.transpose()?)
}
async fn fetch_state_in(
conn: &mut PgConnection,
user_id: i64,
podcast_id: &PodcastId,
lock: bool,
) -> Result<FetchState> {
let row = sqlx::query(&format!(
"SELECT etag, last_modified, checked_at_ms, last_success_at_ms, next_check_at_ms,
last_error, consecutive_failures, cover_url, cover_path
FROM furumusic__podcast_fetch_state WHERE user_id = $1 AND podcast_id = $2{}",
if lock { " FOR UPDATE" } else { "" }
))
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *conn)
.await?;
let Some(row) = row else {
return Ok(FetchState::default());
};
Ok(FetchState {
etag: row.try_get("etag")?,
last_modified: row.try_get("last_modified")?,
checked_at_ms: row.try_get("checked_at_ms")?,
last_success_at_ms: row.try_get("last_success_at_ms")?,
next_check_at_ms: row.try_get("next_check_at_ms")?,
last_error: row.try_get("last_error")?,
consecutive_failures: u32::try_from(row.try_get::<i64, _>("consecutive_failures")?)
.unwrap_or(u32::MAX),
cover_url: row.try_get("cover_url")?,
cover_path: row.try_get("cover_path")?,
})
}
/// Inserts or updates a feed episode; `None` when it is new, otherwise
/// whether its metadata changed.
async fn upsert_feed_episode(
conn: &mut PgConnection,
user_id: i64,
meta: &EpisodeMeta,
now: i64,
) -> Result<Option<bool>> {
let Some(stored) = super::queries::episode_meta_in(conn, user_id, &meta.episode_id).await?
else {
apply::insert_episode(conn, user_id, meta, EpisodeSource::Feed, true, now).await?;
return Ok(None);
};
let changed = stored != *meta;
sqlx::query(
"UPDATE furumusic__podcast_episode
SET podcast_id = $3, guid = $4, title = $5, description = $6, season = $7,
episode_number = $8, episode_type = $9, published_at_ms = $10,
duration_ms = $11, audio_url = $12, mime = $13, byte_len = $14,
image_url = $15, link = $16, source = 'feed', in_feed = TRUE,
search_norm = $17,
updated_at_ms = CASE WHEN $18 THEN $19 ELSE updated_at_ms END
WHERE user_id = $1 AND episode_id = $2",
)
.bind(user_id)
.bind(meta.episode_id.as_str())
.bind(meta.podcast_id.as_str())
.bind(&meta.guid)
.bind(&meta.title)
.bind(&meta.description)
.bind(meta.season.map(i64::from))
.bind(meta.episode_number.map(i64::from))
.bind(apply::episode_type_to_sql(meta.episode_type))
.bind(meta.published_at_ms)
.bind(meta.duration_ms)
.bind(&meta.audio_url)
.bind(&meta.mime)
.bind(meta.byte_len.and_then(|len| i64::try_from(len).ok()))
.bind(&meta.image_url)
.bind(&meta.link)
.bind(episode_search_norm(
&meta.title,
meta.description.as_deref(),
))
.bind(changed)
.bind(now)
.execute(&mut *conn)
.await?;
Ok(Some(changed))
}
impl PodcastStore {
/// Ingests a fetched feed document for a podcast of the account.
///
/// * Channel metadata of the catalog row is replaced by the feed's.
/// * Episode ids are resolved with the contract identity rule
/// (`ParsedFeed::resolve_episode_ids`) against every stored episode of
/// the podcast, including episodes learned through sync, so a changed
/// title, audio URL or a GUID-less item keeps its id. Known episodes
/// get the new metadata (including the audio URL); new ones are added;
/// episodes missing from the document are kept (`in_feed = false`).
/// * A permanent redirect target and `itunes:new-feed-url` become feed
/// aliases and the new fetch URL. When they change the stored alias
/// set, one `podcast_feed_aliases_added` operation is appended.
/// * Subscription, share and progress registers and listening sessions
/// are never touched, so a refresh cannot resurrect an unsubscribed
/// podcast or move anyone's position.
///
/// Record the HTTP outcome separately with
/// [`PodcastStore::record_fetch_success`] and friends.
pub(crate) async fn apply_feed(
&self,
user_id: i64,
refresh: &FeedRefresh<'_>,
) -> Result<RefreshOutcome> {
let podcast_id = refresh.podcast_id;
let feed = refresh.feed;
let now = now_ms();
let device_id = devices::local_device_id(&self.pool, user_id).await?;
let mut tx = self.begin_write(user_id).await?;
let current_url = match podcast_fetch_url(&mut tx, user_id, podcast_id).await? {
Some(url) => url,
None => {
let Some(register) =
apply::load_subscription_register(&mut tx, user_id, podcast_id).await?
else {
return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into());
};
apply::apply_subscription_record(&mut tx, user_id, &register, now).await?;
register.podcast.feed_url
}
};
let known = apply::known_episodes(&mut tx, user_id, podcast_id).await?;
let ids = feed.resolve_episode_ids(podcast_id, &known);
sqlx::query(
"UPDATE furumusic__podcast_episode SET in_feed = FALSE
WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.execute(&mut *tx)
.await?;
let mut outcome = RefreshOutcome::default();
for (item, id) in feed.episodes.iter().zip(ids) {
let Some(id) = id else {
outcome.skipped_items += 1;
continue;
};
let meta = item.to_meta(podcast_id, id);
if meta.validate().is_err() {
outcome.skipped_items += 1;
continue;
}
match upsert_feed_episode(&mut tx, user_id, &meta, now).await? {
None => outcome.new_episodes += 1,
Some(true) => outcome.updated_episodes += 1,
Some(false) => outcome.unchanged_episodes += 1,
}
}
let mut fetch_url = current_url.clone();
let mut learned = BTreeSet::new();
if refresh.moved_permanently && *refresh.fetched_from != fetch_url {
learned.insert(fetch_url.clone());
learned.insert(refresh.fetched_from.clone());
fetch_url = refresh.fetched_from.clone();
}
if let Some(new_url) = feed
.new_feed_url
.as_deref()
.and_then(|url| FeedUrl::parse(url).ok())
&& new_url != fetch_url
{
learned.insert(fetch_url.clone());
learned.insert(new_url.clone());
fetch_url = new_url;
}
let meta = feed.to_podcast_meta(fetch_url.clone());
sqlx::query(
"UPDATE furumusic__podcast
SET feed_url = $3, title = $4, author = $5, description = $6, link = $7,
image_url = $8, meta_from_feed = TRUE, updated_at_ms = $9
WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(meta.feed_url.as_str())
.bind(&meta.title)
.bind(&meta.author)
.bind(&meta.description)
.bind(&meta.link)
.bind(&meta.image_url)
.bind(now)
.execute(&mut *tx)
.await?;
if !learned.is_empty() {
let learned: Vec<FeedUrl> = learned.into_iter().collect();
let current = apply::load_aliases(&mut tx, user_id, podcast_id).await?;
if merge_feed_aliases(podcast_id, &current, &learned) != current {
let op = super::actions::emit(&mut tx, user_id, &device_id, |_| {
SyncOpPayload::PodcastFeedAliasesAdded {
podcast_id: podcast_id.clone(),
feed_aliases: learned.clone(),
}
})
.await?;
outcome.ops.push(op);
}
outcome.learned_aliases = learned;
}
// The card of a shared podcast carries the channel metadata, the
// fetch URL and the episode counters of the feed.
let shared: Option<bool> = sqlx::query_scalar(
"SELECT shared FROM furumusic__podcast_share WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user_id)
.bind(podcast_id.as_str())
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
if shared == Some(true) {
super::discovery::request_reconcile();
}
outcome.feed_url = Some(fetch_url);
Ok(outcome)
}
/// Local fetch state of a podcast (defaults when never fetched).
pub(crate) async fn fetch_state(
&self,
user_id: i64,
podcast_id: &PodcastId,
) -> Result<FetchState> {
let mut conn = self.pool.acquire().await?;
fetch_state_in(&mut conn, user_id, podcast_id, false).await
}
/// Read-modify-write of the fetch state in one transaction with the
/// account lock and the row locked, so concurrent `record_*` /
/// `set_cover_cache` calls never lose each other's columns or failure
/// increments.
async fn update_fetch_state(
&self,
user_id: i64,
podcast_id: &PodcastId,
update: impl FnOnce(&mut FetchState),
) -> Result<FetchState> {
let mut tx = self.begin_write(user_id).await?;
if podcast_fetch_url(&mut tx, user_id, podcast_id)
.await?
.is_none()
{
return Err(PodcastStoreError::UnknownPodcast(podcast_id.clone()).into());
}
let mut state = fetch_state_in(&mut tx, user_id, podcast_id, true).await?;
update(&mut state);
sqlx::query(
"INSERT INTO furumusic__podcast_fetch_state
(user_id, podcast_id, etag, last_modified, checked_at_ms, last_success_at_ms,
next_check_at_ms, last_error, consecutive_failures, cover_url, cover_path)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (user_id, podcast_id) DO UPDATE SET
etag = EXCLUDED.etag, last_modified = EXCLUDED.last_modified,
checked_at_ms = EXCLUDED.checked_at_ms,
last_success_at_ms = EXCLUDED.last_success_at_ms,
next_check_at_ms = EXCLUDED.next_check_at_ms,
last_error = EXCLUDED.last_error,
consecutive_failures = EXCLUDED.consecutive_failures,
cover_url = EXCLUDED.cover_url, cover_path = EXCLUDED.cover_path",
)
.bind(user_id)
.bind(podcast_id.as_str())
.bind(&state.etag)
.bind(&state.last_modified)
.bind(state.checked_at_ms)
.bind(state.last_success_at_ms)
.bind(state.next_check_at_ms)
.bind(&state.last_error)
.bind(i64::from(state.consecutive_failures))
.bind(&state.cover_url)
.bind(&state.cover_path)
.execute(&mut *tx)
.await?;
tx.commit().await?;
Ok(state)
}
/// Records a successful fetch (HTTP 200) with its new validators.
pub(crate) async fn record_fetch_success(
&self,
user_id: i64,
podcast_id: &PodcastId,
now_ms: i64,
etag: Option<&str>,
last_modified: Option<&str>,
next_check_at_ms: Option<i64>,
) -> Result<()> {
self.update_fetch_state(user_id, podcast_id, |state| {
state.etag = etag.map(str::to_string);
state.last_modified = last_modified.map(str::to_string);
state.checked_at_ms = Some(now_ms);
state.last_success_at_ms = Some(now_ms);
state.next_check_at_ms = next_check_at_ms;
state.last_error = None;
state.consecutive_failures = 0;
})
.await
.map(|_| ())
}
/// Records a conditional fetch answered with "304 Not Modified".
pub(crate) async fn record_fetch_not_modified(
&self,
user_id: i64,
podcast_id: &PodcastId,
now_ms: i64,
next_check_at_ms: Option<i64>,
) -> Result<()> {
self.update_fetch_state(user_id, podcast_id, |state| {
state.checked_at_ms = Some(now_ms);
state.last_success_at_ms = Some(now_ms);
state.next_check_at_ms = next_check_at_ms;
state.last_error = None;
state.consecutive_failures = 0;
})
.await
.map(|_| ())
}
/// Records a failed fetch; returns the consecutive failure count. The
/// stored catalog is kept.
pub(crate) async fn record_fetch_failure(
&self,
user_id: i64,
podcast_id: &PodcastId,
now_ms: i64,
error: &str,
next_check_at_ms: Option<i64>,
) -> Result<u32> {
self.update_fetch_state(user_id, podcast_id, |state| {
state.checked_at_ms = Some(now_ms);
state.next_check_at_ms = next_check_at_ms;
state.last_error = Some(error.to_string());
state.consecutive_failures = state.consecutive_failures.saturating_add(1);
})
.await
.map(|state| state.consecutive_failures)
}
/// Records the locally cached cover image of a podcast.
// Covers are not cached yet: the browser loads `image_url` from the
// publisher. The column and this setter stay for a later cache.
#[cfg(test)]
pub(crate) async fn set_cover_cache(
&self,
user_id: i64,
podcast_id: &PodcastId,
cover_url: Option<&str>,
cover_path: Option<&str>,
) -> Result<()> {
self.update_fetch_state(user_id, podcast_id, |state| {
state.cover_url = cover_url.map(str::to_string);
state.cover_path = cover_path.map(str::to_string);
})
.await
.map(|_| ())
}
/// Subscribed podcasts of every account due for a refresh at `now_ms`
/// (never checked or `next_check_at_ms <= now_ms`) with their fetch
/// URL: `(user_id, podcast_id, feed_url)`, at most `limit`. Accounts
/// take turns: first the most overdue podcast of every account, then the
/// second, and so on; within a turn the most overdue first. An account
/// with many due feeds therefore never delays the others, and still uses
/// whatever the run has left over.
pub(crate) async fn podcasts_due_for_refresh(
&self,
now_ms: i64,
limit: usize,
) -> Result<Vec<(i64, PodcastId, FeedUrl)>> {
let rows = sqlx::query(
"SELECT user_id, podcast_id, feed_url
FROM (SELECT p.user_id, p.podcast_id, p.feed_url,
COALESCE(f.next_check_at_ms, 0) AS due,
ROW_NUMBER() OVER (
PARTITION BY p.user_id
ORDER BY COALESCE(f.next_check_at_ms, 0), p.podcast_id
) AS turn
FROM furumusic__podcast p
JOIN furumusic__podcast_subscription s
ON s.user_id = p.user_id AND s.podcast_id = p.podcast_id
AND s.subscribed
LEFT JOIN furumusic__podcast_fetch_state f
ON f.user_id = p.user_id AND f.podcast_id = p.podcast_id
WHERE f.next_check_at_ms IS NULL OR f.next_check_at_ms <= $1) due
ORDER BY turn, due, user_id, podcast_id
LIMIT $2",
)
.bind(now_ms)
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await?;
Ok(rows
.iter()
.map(|row| {
Ok((
row.try_get("user_id")?,
super::podcast_id_col(row, "podcast_id")?,
super::feed_url_col(row, "feed_url")?,
))
})
.collect::<sqlx::Result<Vec<_>>>()?)
}
}
+567
View File
@@ -0,0 +1,567 @@
//! Scheduling of feed refreshes: which feeds are due, one request per feed
//! URL per run, ingestion through [`PodcastStore::apply_feed`] and the local
//! fetch state (validators, next check, backoff).
use std::collections::BTreeMap;
use std::fmt;
use std::time::Duration;
use anyhow::Result;
use futures_util::StreamExt;
use music_dht::podcast::{FeedUrl, PodcastId};
use super::fetch::{FeedFetch, FeedFetcher, FetchError, FetchedFeed, Validators};
use super::{FeedRefresh, PodcastStore, PodcastStoreError, RefreshOutcome};
/// Regular interval between two checks of a healthy feed.
pub(crate) const REFRESH_INTERVAL_MS: i64 = 60 * 60 * 1000;
/// Spread of the regular interval, so feeds added together drift apart.
const REFRESH_JITTER_MS: i64 = 10 * 60 * 1000;
/// First retry delay after a failure; doubles per consecutive failure.
pub(crate) const BACKOFF_BASE_MS: i64 = 15 * 60 * 1000;
/// Longest delay between two attempts of a failing feed.
pub(crate) const BACKOFF_MAX_MS: i64 = 24 * 60 * 60 * 1000;
/// A manual refresh is refused when the feed was checked this recently.
pub(crate) const MANUAL_MIN_INTERVAL_MS: i64 = 30 * 1000;
/// Podcasts considered per scheduled run. Due podcasts are taken in turns
/// across accounts (see [`PodcastStore::podcasts_due_for_refresh`]), so an
/// account with many (or slow) feeds cannot hold back everyone else's
/// refreshes, but can use a run nobody else needs.
pub(crate) const RUN_LIMIT: usize = 200;
/// Feed requests in flight at once.
pub(crate) const RUN_CONCURRENCY: usize = 4;
/// Delay before the next attempt after `consecutive_failures` failures in a
/// row (counting the one just recorded): 15 min, 30 min, 1 h, ... capped at a
/// day. A longer server-requested `Retry-After` wins (also capped).
pub(crate) fn failure_backoff_ms(consecutive_failures: u32, retry_after: Option<Duration>) -> i64 {
let exponent = consecutive_failures.saturating_sub(1).min(16);
let backoff = BACKOFF_BASE_MS
.saturating_mul(1_i64 << exponent)
.min(BACKOFF_MAX_MS);
let requested = retry_after
.map(|delay| i64::try_from(delay.as_millis()).unwrap_or(i64::MAX))
.unwrap_or(0);
backoff.max(requested).min(BACKOFF_MAX_MS)
}
/// Next regular check after a success: the interval plus a stable per
/// account and podcast jitter.
pub(crate) fn next_regular_check_ms(now_ms: i64, user_id: i64, podcast_id: &PodcastId) -> i64 {
let key = format!("{user_id}:{podcast_id}");
let hash = blake3::hash(key.as_bytes());
let spread = u64::from_le_bytes(hash.as_bytes()[..8].try_into().expect("8 bytes"));
let jitter = i64::try_from(spread % REFRESH_JITTER_MS as u64).unwrap_or(0);
now_ms + REFRESH_INTERVAL_MS + jitter
}
/// One podcast of one account to refresh.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RefreshTarget {
pub(crate) user_id: i64,
pub(crate) podcast_id: PodcastId,
}
/// Result of refreshing one [`RefreshTarget`].
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum TargetResult {
/// A new document was ingested.
Updated(RefreshOutcome),
/// `304 Not Modified`.
NotModified,
/// The fetch or the ingestion failed; the catalog is unchanged.
Failed {
/// What the user sees (stored as the last error).
error: String,
/// Details for the server log (no URLs).
detail: String,
consecutive_failures: u32,
next_check_at_ms: i64,
},
/// Nothing to do any more (unsubscribed or removed meanwhile).
Skipped(String),
}
/// Counters of a scheduled run.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct RunSummary {
/// Account podcasts that were due.
pub(crate) targets: usize,
/// Distinct feed URLs requested.
pub(crate) feeds: usize,
pub(crate) updated: usize,
pub(crate) not_modified: usize,
pub(crate) failed: usize,
pub(crate) skipped: usize,
pub(crate) new_episodes: usize,
pub(crate) updated_episodes: usize,
/// `(podcast_id, details)` of failed targets, for the job log (no
/// account ids or URLs: feed URLs often carry access tokens).
pub(crate) failures: Vec<(PodcastId, String)>,
}
impl RunSummary {
fn add(&mut self, target: &RefreshTarget, result: &TargetResult) {
match result {
TargetResult::Updated(outcome) => {
self.updated += 1;
self.new_episodes += outcome.new_episodes;
self.updated_episodes += outcome.updated_episodes;
}
TargetResult::NotModified => self.not_modified += 1,
TargetResult::Failed { detail, .. } => {
self.failed += 1;
self.failures
.push((target.podcast_id.clone(), detail.clone()));
}
TargetResult::Skipped(_) => self.skipped += 1,
}
}
}
impl fmt::Display for RunSummary {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{} due podcasts, {} feeds requested: {} updated ({} new, {} changed episodes), \
{} not modified, {} failed, {} skipped",
self.targets,
self.feeds,
self.updated,
self.new_episodes,
self.updated_episodes,
self.not_modified,
self.failed,
self.skipped
)
}
}
/// Due podcasts grouped by feed URL: accounts subscribed to the same URL
/// share one request.
pub(crate) type RefreshGroups = BTreeMap<FeedUrl, Vec<RefreshTarget>>;
/// Groups the rows of [`PodcastStore::podcasts_due_for_refresh`].
pub(crate) fn group_targets(due: Vec<(i64, PodcastId, FeedUrl)>) -> RefreshGroups {
let mut groups = RefreshGroups::new();
for (user_id, podcast_id, feed_url) in due {
groups.entry(feed_url).or_default().push(RefreshTarget {
user_id,
podcast_id,
});
}
groups
}
/// Refreshes podcasts due at `now_ms` (at most `limit`, accounts in
/// turns), with at most
/// `concurrency` feed requests in flight. Accounts subscribed to the same
/// feed URL share one request. A failing feed never stops the others.
pub(crate) async fn refresh_due(
store: &PodcastStore,
fetcher: &FeedFetcher,
now_ms: i64,
limit: usize,
concurrency: usize,
) -> Result<RunSummary> {
let due = store.podcasts_due_for_refresh(now_ms, limit).await?;
refresh_groups(store, fetcher, group_targets(due), now_ms, concurrency).await
}
/// Refreshes `groups`, one request per feed URL, at most `concurrency` at
/// a time.
pub(crate) async fn refresh_groups(
store: &PodcastStore,
fetcher: &FeedFetcher,
groups: RefreshGroups,
now_ms: i64,
concurrency: usize,
) -> Result<RunSummary> {
let mut summary = RunSummary {
targets: groups.values().map(Vec::len).sum(),
feeds: groups.len(),
..RunSummary::default()
};
let mut results = futures_util::stream::iter(groups)
.map(|(feed_url, targets)| async move {
refresh_group(store, fetcher, &feed_url, targets, now_ms).await
})
.buffer_unordered(concurrency.max(1));
while let Some(group) = results.next().await {
for (target, result) in group? {
summary.add(&target, &result);
}
}
Ok(summary)
}
/// Fetches `feed_url` once for all `targets` and records the outcome for
/// each. The request is conditional only when every target holds the same
/// validators (the usual case: one account).
pub(crate) async fn refresh_group(
store: &PodcastStore,
fetcher: &FeedFetcher,
feed_url: &FeedUrl,
targets: Vec<RefreshTarget>,
now_ms: i64,
) -> Result<Vec<(RefreshTarget, TargetResult)>> {
let mut states = Vec::with_capacity(targets.len());
for target in &targets {
states.push(
store
.fetch_state(target.user_id, &target.podcast_id)
.await?,
);
}
let validators = states
.iter()
.map(|state| Validators {
etag: state.etag.clone(),
last_modified: state.last_modified.clone(),
})
.reduce(|a, b| if a == b { a } else { Validators::default() })
.unwrap_or_default();
let fetched = fetcher.fetch(feed_url, &validators).await;
let mut out = Vec::with_capacity(targets.len());
for (target, state) in targets.into_iter().zip(states) {
let result =
record_result(store, &target, &fetched, state.consecutive_failures, now_ms).await?;
out.push((target, result));
}
Ok(out)
}
/// Ingests (or records the failure of) one fetch for one target.
async fn record_result(
store: &PodcastStore,
target: &RefreshTarget,
fetched: &Result<FeedFetch, FetchError>,
previous_failures: u32,
now_ms: i64,
) -> Result<TargetResult> {
let (user_id, podcast_id) = (target.user_id, &target.podcast_id);
let failure = |error: String, detail: String, retry_after: Option<Duration>| {
let consecutive_failures = previous_failures.saturating_add(1);
let next_check_at_ms = now_ms + failure_backoff_ms(consecutive_failures, retry_after);
(error, detail, consecutive_failures, next_check_at_ms)
};
let (error, detail, consecutive_failures, next_check_at_ms) = match fetched {
Ok(FeedFetch::NotModified) => {
let next = next_regular_check_ms(now_ms, user_id, podcast_id);
return match store
.record_fetch_not_modified(user_id, podcast_id, now_ms, Some(next))
.await
{
Ok(()) => Ok(TargetResult::NotModified),
Err(error) => skipped_or(error),
};
}
Ok(FeedFetch::Fetched(document)) => {
match ingest(store, target, document, now_ms).await {
Ok(outcome) => return Ok(TargetResult::Updated(outcome)),
Err(error) if is_gone(&error) => return skipped_or(error),
// A store failure is recorded like a fetch failure, so the
// feed backs off instead of being retried every run.
Err(error) => failure(
"the feed could not be stored".to_string(),
format!("storing the feed failed: {error:#}"),
None,
),
}
}
Err(error) => failure(
error.public_message(),
error.message.clone(),
error.retry_after,
),
};
match store
.record_fetch_failure(user_id, podcast_id, now_ms, &error, Some(next_check_at_ms))
.await
{
Ok(_) => Ok(TargetResult::Failed {
error,
detail,
consecutive_failures,
next_check_at_ms,
}),
Err(error) => skipped_or(error),
}
}
async fn ingest(
store: &PodcastStore,
target: &RefreshTarget,
document: &FetchedFeed,
now_ms: i64,
) -> Result<RefreshOutcome> {
let outcome = store
.apply_feed(
target.user_id,
&FeedRefresh {
podcast_id: &target.podcast_id,
feed: &document.feed,
fetched_from: &document.fetched_from,
moved_permanently: document.moved_permanently,
},
)
.await?;
store
.record_fetch_success(
target.user_id,
&target.podcast_id,
now_ms,
document.validators.etag.as_deref(),
document.validators.last_modified.as_deref(),
Some(next_regular_check_ms(
now_ms,
target.user_id,
&target.podcast_id,
)),
)
.await?;
Ok(outcome)
}
fn is_gone(error: &anyhow::Error) -> bool {
matches!(
error.downcast_ref::<PodcastStoreError>(),
Some(PodcastStoreError::UnknownPodcast(_))
)
}
/// A podcast removed from the account meanwhile is skipped; any other
/// store error is returned.
fn skipped_or(error: anyhow::Error) -> Result<TargetResult> {
if is_gone(&error) {
Ok(TargetResult::Skipped(error.to_string()))
} else {
Err(error)
}
}
/// Why a manual refresh was not started.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ManualRefreshError {
/// The account does not know the podcast.
UnknownPodcast,
/// The account is not subscribed (unsubscribing stops monitoring).
NotSubscribed,
/// Checked less than [`MANUAL_MIN_INTERVAL_MS`] ago.
TooSoon { retry_after_ms: i64 },
}
impl fmt::Display for ManualRefreshError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnknownPodcast => f.write_str("unknown podcast"),
Self::NotSubscribed => f.write_str("not subscribed to this podcast"),
Self::TooSoon { retry_after_ms } => write!(
f,
"the feed was checked moments ago; retry in {} s",
(retry_after_ms + 999) / 1000
),
}
}
}
impl std::error::Error for ManualRefreshError {}
/// Refreshes one subscribed podcast of one account now, ignoring the
/// failure backoff but not more often than every
/// [`MANUAL_MIN_INTERVAL_MS`]. Returns `Err` with a
/// [`ManualRefreshError`] when the refresh was not attempted.
pub(crate) async fn refresh_now(
store: &PodcastStore,
fetcher: &FeedFetcher,
user_id: i64,
podcast_id: &PodcastId,
now_ms: i64,
) -> Result<TargetResult> {
let feed_url = match store.podcast(user_id, podcast_id).await? {
Some(summary) if summary.subscribed => summary.meta.feed_url,
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
None => match store.subscription(user_id, podcast_id).await? {
Some(register) if register.subscribed => register.podcast.feed_url,
Some(_) => return Err(ManualRefreshError::NotSubscribed.into()),
None => return Err(ManualRefreshError::UnknownPodcast.into()),
},
};
if let Some(checked) = store.fetch_state(user_id, podcast_id).await?.checked_at_ms {
let elapsed = now_ms - checked;
if (0..MANUAL_MIN_INTERVAL_MS).contains(&elapsed) {
return Err(ManualRefreshError::TooSoon {
retry_after_ms: MANUAL_MIN_INTERVAL_MS - elapsed,
}
.into());
}
}
let target = RefreshTarget {
user_id,
podcast_id: podcast_id.clone(),
};
let mut results = refresh_group(store, fetcher, &feed_url, vec![target], now_ms).await?;
Ok(results.pop().map(|(_, result)| result).expect("one target"))
}
/// Result of [`subscribe_url`].
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct UrlSubscription {
/// Podcast the URL resolved to (an existing one for a known URL or alias).
pub(crate) podcast_id: PodcastId,
/// The account was already subscribed; no subscription was written.
pub(crate) already_subscribed: bool,
/// Ingestion of the document fetched for the subscription.
pub(crate) refresh: RefreshOutcome,
}
/// Why [`subscribe_url`] did not subscribe.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum SubscribeUrlError {
/// The feed could not be fetched or parsed; nothing was stored.
Fetch(FetchError),
}
impl fmt::Display for SubscribeUrlError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Fetch(error) => {
write!(
f,
"the feed could not be loaded: {}",
error.public_message()
)
}
}
}
}
impl std::error::Error for SubscribeUrlError {}
/// Subscribes an account to the feed at `url` (the "add RSS" action).
///
/// The feed is fetched first ([`fetch_to_subscribe`]): an address that is
/// not public, a document that is not a feed or a failed request
/// subscribes nothing ([`SubscribeUrlError`] inside the error). Then
/// [`subscribe_fetched`] writes the subscription.
pub(crate) async fn subscribe_url(
store: &PodcastStore,
fetcher: &FeedFetcher,
user_id: i64,
url: &FeedUrl,
now_ms: i64,
) -> Result<UrlSubscription> {
let document = fetch_to_subscribe(fetcher, url).await?;
subscribe_fetched(store, user_id, url, &document, now_ms).await
}
/// Fetches the feed at `url` unconditionally for a new subscription or a
/// preview; a failure is a [`SubscribeUrlError`].
pub(crate) async fn fetch_to_subscribe(
fetcher: &FeedFetcher,
url: &FeedUrl,
) -> Result<Box<FetchedFeed>> {
match fetcher.fetch(url, &Validators::default()).await {
Ok(FeedFetch::Fetched(document)) => Ok(document),
Ok(FeedFetch::NotModified) => {
// Unconditional requests are never answered with 304 by a
// conforming server; refuse instead of subscribing blindly.
Err(SubscribeUrlError::Fetch(FetchError {
kind: super::fetch::FetchErrorKind::Status(304),
message: "unexpected 304 Not Modified".to_string(),
retry_after: None,
})
.into())
}
Err(error) => Err(SubscribeUrlError::Fetch(error).into()),
}
}
/// The URLs a document fetched from `requested` shows for the same
/// podcast: `requested`, the end of a chain of permanent redirects and the
/// feed's own `itunes:new-feed-url` (the same URLs a refresh learns as
/// aliases and moves to), without duplicates.
///
/// Only the first two are where the document came from. The
/// `itunes:new-feed-url` is a claim of the document: following the feed
/// moves to it anyway (its first refresh fetches from there), so it may
/// name the account's podcast for the feed, but the document is never that
/// podcast's content ([`subscribe_fetched`] does not ingest it there).
pub(crate) fn document_urls(requested: &FeedUrl, document: &FetchedFeed) -> Vec<FeedUrl> {
let mut urls = vec![requested.clone()];
if document.moved_permanently {
urls.push(document.fetched_from.clone());
}
if let Some(new_url) = document
.feed
.new_feed_url
.as_deref()
.and_then(|url| FeedUrl::parse(url).ok())
{
urls.push(new_url);
}
let mut seen = std::collections::BTreeSet::new();
urls.retain(|url| seen.insert(url.clone()));
urls
}
/// Whether a document fetched from `requested` is the content of a podcast
/// fetched from `feed_url`: the request went to `feed_url` itself (what a
/// refresh does), or a permanent redirect chain ended there.
fn fetched_from_feed(requested: &FeedUrl, document: &FetchedFeed, feed_url: &FeedUrl) -> bool {
requested == feed_url || (document.moved_permanently && document.fetched_from == *feed_url)
}
/// Writes the subscription for a document fetched from `url`.
///
/// * The podcast is the account's podcast for the feed
/// ([`super::search::known_podcast`] over [`document_urls`]: a subscribed
/// one first), so a feed the account already follows under another
/// address is not subscribed a second time; otherwise a new podcast with
/// the id of `url`.
/// * The document is ingested like a scheduled refresh (which also stores
/// the validators and the next check) only for a new podcast, or when it
/// came from the existing podcast's own fetch URL
/// ([`fetched_from_feed`]). Otherwise (the match came through another
/// alias or the feed's `itunes:new-feed-url` claim) it may be a stale
/// copy or another feed altogether: the existing podcast is subscribed
/// again with its stored channel metadata, like "subscribe again", and
/// the refresh job fetches its own URL. An active subscription is never
/// written.
///
/// [`PodcastStore::subscribe`] reuses the existing podcast with its
/// progress and history.
pub(crate) async fn subscribe_fetched(
store: &PodcastStore,
user_id: i64,
url: &FeedUrl,
document: &FetchedFeed,
now_ms: i64,
) -> Result<UrlSubscription> {
let requested_id = store.resolve_podcast_id(user_id, url).await?;
let urls = document_urls(url, document);
let existing = super::search::known_podcast(store, user_id, &requested_id, &urls).await?;
let meta = match &existing {
Some(known) if !fetched_from_feed(url, document, &known.meta.feed_url) => {
let subscription = store.subscribe(user_id, &known.meta).await?;
return Ok(UrlSubscription {
podcast_id: subscription.podcast_id,
already_subscribed: subscription.already_subscribed,
refresh: RefreshOutcome::default(),
});
}
Some(known) => document.feed.to_podcast_meta(known.meta.feed_url.clone()),
None => document.feed.to_podcast_meta(url.clone()),
};
let subscription = store.subscribe(user_id, &meta).await?;
let target = RefreshTarget {
user_id,
podcast_id: subscription.podcast_id.clone(),
};
let refresh = ingest(store, &target, document, now_ms).await?;
Ok(UrlSubscription {
podcast_id: subscription.podcast_id,
already_subscribed: subscription.already_subscribed,
refresh,
})
}
+366
View File
@@ -0,0 +1,366 @@
//! PostgreSQL schema of the podcast store and the import of the relay state.
//!
//! The migrations themselves are declared in `crate::music::db_migrations`
//! (`m_0056_create_podcast_store`, `m_0057_import_podcast_relay`,
//! `m_0058_create_podcast_share_confirmation`), next to the rest of the
//! `furumusic` chain; this module holds their SQL so the store and its tests
//! share one definition.
//!
//! The tables mirror the SQLite reference store of `furumi-library`
//! (`PODCASTS.md`, "SQLite reference store") with a `user_id` column in
//! every key: each web account is one virtual device with its own podcast
//! catalog, registers and fetch state.
/// Tables, the derived season view and indexes (`m_0056`).
pub(crate) const CREATE_PODCAST_STORE: &[&str] = &[
// Local channel metadata. `feed_url` is the URL this account fetches;
// `meta_from_feed` is set once the account refreshed the feed itself
// (until then the winning subscription register provides the metadata).
"CREATE TABLE IF NOT EXISTS furumusic__podcast (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
feed_url TEXT NOT NULL,
title TEXT NOT NULL,
author TEXT,
description TEXT,
link TEXT,
image_url TEXT,
meta_from_feed BOOLEAN NOT NULL DEFAULT FALSE,
created_at_ms BIGINT NOT NULL,
updated_at_ms BIGINT NOT NULL,
PRIMARY KEY (user_id, podcast_id)
)",
// Bounded grow-only alias set (`merge_feed_aliases`), synced inside
// subscription records and `podcast_feed_aliases_added`.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_feed_alias (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
feed_url TEXT NOT NULL,
PRIMARY KEY (user_id, podcast_id, feed_url)
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_feed_alias_url
ON furumusic__podcast_feed_alias (user_id, feed_url)",
// Episode catalog (`EpisodeMeta`). `source` is `feed` once this account
// parsed the episode from RSS, `sync` while it is only known from a
// `PodcastEpisodeRef`. `search_norm` caches `normalize_name(title)` and
// `normalize_name(description)` separated by a newline (never produced
// by the normalization), NULL until computed.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_episode (
user_id BIGINT NOT NULL,
episode_id TEXT NOT NULL,
podcast_id TEXT NOT NULL,
guid TEXT,
title TEXT NOT NULL,
description TEXT,
season BIGINT,
episode_number BIGINT,
episode_type TEXT NOT NULL DEFAULT 'full',
published_at_ms BIGINT,
duration_ms BIGINT,
audio_url TEXT NOT NULL,
mime TEXT,
byte_len BIGINT,
image_url TEXT,
link TEXT,
source TEXT NOT NULL CHECK (source IN ('feed', 'sync')),
in_feed BOOLEAN NOT NULL DEFAULT FALSE,
search_norm TEXT,
first_seen_at_ms BIGINT NOT NULL,
updated_at_ms BIGINT NOT NULL,
PRIMARY KEY (user_id, episode_id),
FOREIGN KEY (user_id, podcast_id)
REFERENCES furumusic__podcast (user_id, podcast_id) ON DELETE CASCADE
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_episode_date
ON furumusic__podcast_episode (user_id, podcast_id, published_at_ms DESC)",
"CREATE INDEX IF NOT EXISTS idx_podcast_episode_season
ON furumusic__podcast_episode (user_id, podcast_id, season, episode_number)",
// Seasons are derived from the episodes; a view cannot go stale.
// `season` NULL groups the episodes without `itunes:season`.
"CREATE OR REPLACE VIEW furumusic__podcast_season AS
SELECT user_id, podcast_id, season,
COUNT(*) AS episode_count,
MIN(published_at_ms) AS first_published_at_ms,
MAX(published_at_ms) AS last_published_at_ms
FROM furumusic__podcast_episode
GROUP BY user_id, podcast_id, season",
// Synced registers. No foreign keys to the catalog: operations for
// podcasts or episodes this account has not fetched are kept as they
// arrive, in any order.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_subscription (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
subscribed BOOLEAN NOT NULL,
podcast_json JSONB NOT NULL,
generation BIGINT NOT NULL CHECK (generation >= 0),
hlc_ms BIGINT NOT NULL,
op_id TEXT NOT NULL,
PRIMARY KEY (user_id, podcast_id)
)",
"CREATE TABLE IF NOT EXISTS furumusic__podcast_share (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
shared BOOLEAN NOT NULL,
generation BIGINT NOT NULL CHECK (generation >= 0),
hlc_ms BIGINT NOT NULL,
op_id TEXT NOT NULL,
PRIMARY KEY (user_id, podcast_id)
)",
"CREATE TABLE IF NOT EXISTS furumusic__podcast_progress (
user_id BIGINT NOT NULL,
episode_id TEXT NOT NULL,
podcast_id TEXT NOT NULL,
position_ms BIGINT NOT NULL,
status TEXT NOT NULL
CHECK (status IN ('not_started', 'in_progress', 'completed')),
generation BIGINT NOT NULL CHECK (generation >= 0),
session_id TEXT NOT NULL,
hlc_ms BIGINT NOT NULL,
device_id TEXT NOT NULL,
op_id TEXT NOT NULL,
PRIMARY KEY (user_id, episode_id)
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_progress_podcast
ON furumusic__podcast_progress (user_id, podcast_id)",
// Grow-only set of listening sessions.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_listen_session (
user_id BIGINT NOT NULL,
session_id TEXT NOT NULL,
episode_id TEXT NOT NULL,
podcast_id TEXT NOT NULL,
device_id TEXT NOT NULL,
started_at_ms BIGINT NOT NULL,
ended_at_ms BIGINT NOT NULL,
start_position_ms BIGINT NOT NULL,
end_position_ms BIGINT NOT NULL,
listened_ms BIGINT NOT NULL,
completed BOOLEAN NOT NULL,
PRIMARY KEY (user_id, session_id)
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_listen_session_episode
ON furumusic__podcast_listen_session (user_id, episode_id)",
"CREATE INDEX IF NOT EXISTS idx_podcast_listen_session_end
ON furumusic__podcast_listen_session (user_id, ended_at_ms DESC)",
// Local-only fetch state: never part of sync output.
"CREATE TABLE IF NOT EXISTS furumusic__podcast_fetch_state (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
etag TEXT,
last_modified TEXT,
checked_at_ms BIGINT,
last_success_at_ms BIGINT,
next_check_at_ms BIGINT,
last_error TEXT,
consecutive_failures BIGINT NOT NULL DEFAULT 0,
cover_url TEXT,
cover_path TEXT,
PRIMARY KEY (user_id, podcast_id),
FOREIGN KEY (user_id, podcast_id)
REFERENCES furumusic__podcast (user_id, podcast_id) ON DELETE CASCADE
)",
"CREATE INDEX IF NOT EXISTS idx_podcast_fetch_state_due
ON furumusic__podcast_fetch_state (next_check_at_ms)",
];
/// Backwards operation of `m_0056`.
pub(crate) const DROP_PODCAST_STORE: &[&str] = &[
"DROP VIEW IF EXISTS furumusic__podcast_season",
"DROP TABLE IF EXISTS furumusic__podcast_fetch_state",
"DROP TABLE IF EXISTS furumusic__podcast_listen_session",
"DROP TABLE IF EXISTS furumusic__podcast_progress",
"DROP TABLE IF EXISTS furumusic__podcast_share",
"DROP TABLE IF EXISTS furumusic__podcast_subscription",
"DROP TABLE IF EXISTS furumusic__podcast_episode",
"DROP TABLE IF EXISTS furumusic__podcast_feed_alias",
"DROP TABLE IF EXISTS furumusic__podcast",
];
/// Local privacy confirmations of shared podcasts (`m_0058`).
///
/// `PodcastShareState::privacy_confirmed` is client-held state (frid
/// `PODCASTS.md`, "Publication"): it is never synced and never part of a
/// snapshot. A row records that the account confirmed publishing exactly
/// `feed_url` / `id_feed_url` for exactly the share decision `share_op_id`
/// despite `PodcastCardSpec::privacy_warnings`. A new share decision (on any
/// device) or a feed move to other URLs makes the row stale, so the user is
/// asked again (see `crate::podcasts::discovery`).
pub(crate) const CREATE_SHARE_CONFIRMATION: &[&str] = &[
"CREATE TABLE IF NOT EXISTS furumusic__podcast_share_confirmation (
user_id BIGINT NOT NULL,
podcast_id TEXT NOT NULL,
share_op_id TEXT NOT NULL,
feed_url TEXT NOT NULL,
id_feed_url TEXT,
confirmed_at_ms BIGINT NOT NULL,
PRIMARY KEY (user_id, podcast_id)
)",
];
/// Backwards operation of [`CREATE_SHARE_CONFIRMATION`].
pub(crate) const DROP_SHARE_CONFIRMATION: &[&str] =
&["DROP TABLE IF EXISTS furumusic__podcast_share_confirmation"];
/// Placeholder in [`IMPORT_PODCAST_RELAY`] for the account filter: the
/// migration imports every account (`TRUE`), tests narrow it to one.
pub(crate) const USER_FILTER: &str = "{user_filter}";
/// Largest generation the `BIGINT` columns hold; larger records are skipped
/// like the store rejects them.
const GENERATION_FITS: &str = "(record_json->>'generation')::numeric <= 9223372036854775807";
/// Import of the opaque relay state kept by 0.13.0 (`m_0057`).
///
/// `furumusic__fed_podcast_relay` holds the contract registers as JSON,
/// already validated and merged with the `music_dht::podcast` rules when they
/// were received, so they are copied as they are. The catalog rows the store
/// would have created from the episode references of received operations
/// (`PodcastEpisodeRef`, `source = 'sync'`) are rebuilt from the operation
/// log. Every statement is idempotent (`ON CONFLICT DO NOTHING`); the relay
/// table is kept (see `docs/administration.md`).
pub(crate) const IMPORT_PODCAST_RELAY: &[&str] = &[
// Catalog rows from the subscription registers.
"INSERT INTO furumusic__podcast
(user_id, podcast_id, feed_url, title, author, description, link, image_url,
meta_from_feed, created_at_ms, updated_at_ms)
SELECT user_id, record_key,
record_json->'podcast'->>'feed_url', record_json->'podcast'->>'title',
record_json->'podcast'->>'author', record_json->'podcast'->>'description',
record_json->'podcast'->>'link', record_json->'podcast'->>'image_url',
FALSE, updated_at_ms, updated_at_ms
FROM furumusic__fed_podcast_relay
WHERE section = 'subscription' AND {user_filter}
ON CONFLICT DO NOTHING",
"INSERT INTO furumusic__podcast_subscription
(user_id, podcast_id, subscribed, podcast_json, generation, hlc_ms, op_id)
SELECT user_id, record_key, (record_json->>'subscribed')::boolean,
record_json->'podcast', (record_json->>'generation')::bigint,
(record_json->>'hlc_ms')::bigint, record_json->>'op_id'
FROM furumusic__fed_podcast_relay
WHERE section = 'subscription' AND {generation_fits} AND {user_filter}
ON CONFLICT DO NOTHING",
// The relayed subscription already carries the bounded, merged alias set.
"INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url)
SELECT user_id, record_key, jsonb_array_elements_text(record_json->'feed_aliases')
FROM furumusic__fed_podcast_relay
WHERE section = 'subscription' AND {user_filter}
AND jsonb_typeof(record_json->'feed_aliases') = 'array'
ON CONFLICT DO NOTHING",
// Aliases received before any subscription register of their podcast.
"INSERT INTO furumusic__podcast_feed_alias (user_id, podcast_id, feed_url)
SELECT user_id, record_key, jsonb_array_elements_text(record_json)
FROM furumusic__fed_podcast_relay
WHERE section = 'aliases' AND {user_filter} AND jsonb_typeof(record_json) = 'array'
ON CONFLICT DO NOTHING",
"INSERT INTO furumusic__podcast_share
(user_id, podcast_id, shared, generation, hlc_ms, op_id)
SELECT user_id, record_key, (record_json->>'shared')::boolean,
(record_json->>'generation')::bigint, (record_json->>'hlc_ms')::bigint,
record_json->>'op_id'
FROM furumusic__fed_podcast_relay
WHERE section = 'share' AND {generation_fits} AND {user_filter}
ON CONFLICT DO NOTHING",
"INSERT INTO furumusic__podcast_progress
(user_id, episode_id, podcast_id, position_ms, status, generation, session_id,
hlc_ms, device_id, op_id)
SELECT user_id, record_key, record_json->>'podcast_id',
(record_json->>'position_ms')::bigint, record_json->>'status',
(record_json->>'generation')::bigint, record_json->>'session_id',
(record_json->>'hlc_ms')::bigint, record_json->>'device_id',
record_json->>'op_id'
FROM furumusic__fed_podcast_relay
WHERE section = 'progress' AND {generation_fits} AND {user_filter}
ON CONFLICT DO NOTHING",
"INSERT INTO furumusic__podcast_listen_session
(user_id, session_id, episode_id, podcast_id, device_id, started_at_ms,
ended_at_ms, start_position_ms, end_position_ms, listened_ms, completed)
SELECT user_id, record_key, record_json->>'episode_id', record_json->>'podcast_id',
record_json->>'device_id', (record_json->>'started_at_ms')::bigint,
(record_json->>'ended_at_ms')::bigint,
(record_json->>'start_position_ms')::bigint,
(record_json->>'end_position_ms')::bigint,
(record_json->>'listened_ms')::bigint, (record_json->>'completed')::boolean
FROM furumusic__fed_podcast_relay
WHERE section = 'session' AND {user_filter}
ON CONFLICT DO NOTHING",
// Catalog rows for podcasts only known from episode references.
"INSERT INTO furumusic__podcast
(user_id, podcast_id, feed_url, title, meta_from_feed, created_at_ms, updated_at_ms)
SELECT DISTINCT ON (user_id, payload_json->'episode'->>'podcast_id')
user_id, payload_json->'episode'->>'podcast_id',
payload_json->'episode'->>'feed_url', payload_json->'episode'->>'podcast_title',
FALSE, received_at_ms, received_at_ms
FROM furumusic__fed_sync_ops
WHERE kind IN ('podcast_progress_set', 'podcast_listen_recorded')
AND jsonb_typeof(payload_json->'episode') = 'object' AND {user_filter}
ORDER BY user_id, payload_json->'episode'->>'podcast_id', hlc_ms DESC, op_id DESC
ON CONFLICT DO NOTHING",
// Episodes only known from references (the newest reference wins, like
// the first one would have for the store: feed metadata replaces it on
// the first refresh anyway).
"INSERT INTO furumusic__podcast_episode
(user_id, episode_id, podcast_id, title, season, episode_number, episode_type,
published_at_ms, duration_ms, audio_url, mime, image_url, source, in_feed,
first_seen_at_ms, updated_at_ms)
SELECT DISTINCT ON (user_id, payload_json->'episode'->>'episode_id')
user_id, payload_json->'episode'->>'episode_id',
payload_json->'episode'->>'podcast_id', payload_json->'episode'->>'episode_title',
(payload_json->'episode'->>'season')::bigint,
(payload_json->'episode'->>'episode_number')::bigint, 'full',
(payload_json->'episode'->>'published_at_ms')::bigint,
(payload_json->'episode'->>'duration_ms')::bigint,
payload_json->'episode'->>'audio_url', payload_json->'episode'->>'mime',
payload_json->'episode'->>'image_url', 'sync', FALSE,
received_at_ms, received_at_ms
FROM furumusic__fed_sync_ops
WHERE kind IN ('podcast_progress_set', 'podcast_listen_recorded')
AND jsonb_typeof(payload_json->'episode') = 'object' AND {user_filter}
ORDER BY user_id, payload_json->'episode'->>'episode_id', hlc_ms DESC, op_id DESC
ON CONFLICT DO NOTHING",
];
/// The import statements with `filter` as the account condition.
pub(crate) fn import_statements(filter: &str) -> Vec<String> {
IMPORT_PODCAST_RELAY
.iter()
.map(|sql| {
sql.replace("{generation_fits}", GENERATION_FITS)
.replace(USER_FILTER, filter)
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn import_statements_fill_every_placeholder() {
for sql in import_statements("user_id = 7") {
assert!(!sql.contains('{'), "{sql}");
assert!(sql.contains("user_id = 7"), "{sql}");
assert!(sql.contains("ON CONFLICT DO NOTHING"), "{sql}");
}
}
#[test]
fn every_store_table_is_dropped_backwards() {
let drops: Vec<&str> = DROP_PODCAST_STORE
.iter()
.chain(DROP_SHARE_CONFIRMATION)
.copied()
.collect();
for sql in CREATE_PODCAST_STORE.iter().chain(CREATE_SHARE_CONFIRMATION) {
let Some(rest) = sql
.strip_prefix("CREATE TABLE IF NOT EXISTS ")
.or_else(|| sql.strip_prefix("CREATE OR REPLACE VIEW "))
else {
continue;
};
let name = rest.split_whitespace().next().unwrap();
assert!(
drops.iter().any(|drop| drop.ends_with(&format!(" {name}"))),
"{name}"
);
}
}
}
+474
View File
@@ -0,0 +1,474 @@
//! Public podcasts found by the federated search, seen by one account.
//!
//! The federated search (`/api/player/federation/search/events`) asks the
//! discovery overlay (`PodcastDht::search`) next to the music DHT and merges
//! both with `merge_search_results`: exact title matches first, music before
//! podcasts inside a tier. This module turns the podcast hits into
//! [`PodcastHitView`]s for the account that searched:
//!
//! * only the hit's `feed_url` (the origin URL the id hashes from) is
//! offered; `claimed_feed_urls` are unverified publisher claims and are
//! not passed on;
//! * every text and URL of a card is a claim of some publisher. Artwork and
//! web-page links are kept only as absolute `http(s)` URLs; the browser
//! renders all text as text;
//! * `subscribed` / `local_podcast_id` mark a podcast the account already
//! has, by id or through a known feed URL or alias (`resolve_podcast_id`).
//!
//! [`preview`] reads the found podcast's feed (through the SSRF-guarded
//! fetcher, nothing stored) so the user can look at its seasons and
//! episodes before subscribing; subscribing uses the origin URL, so the new
//! private subscription gets the card's id (see `api::subscribe_found`).
use std::collections::BTreeMap;
use anyhow::Result;
use music_dht::podcast::feed::ParsedFeed;
use music_dht::podcast::{FeedUrl, PodcastId};
use music_dht::podcast_dht::{FederatedSearchHit, PodcastSearchHit, merge_search_results};
use music_dht::{LibraryItem, normalize_name};
use serde::Serialize;
use super::{PodcastStore, PodcastSummary, SeasonFilter};
/// Podcast hits asked from the overlay per search.
pub(crate) const PODCAST_SEARCH_LIMIT: usize = 20;
/// Episodes listed in a preview (newest first), per season filter.
pub(crate) const PREVIEW_EPISODES: usize = 50;
/// Longest URL kept from a card or a feed.
const MAX_URL_BYTES: usize = 2048;
/// One public podcast in the federated search results.
///
/// Serialized with `"type": "podcast"` (music hits are `"type": "music"`).
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct PodcastHitView {
#[serde(rename = "type")]
pub(crate) hit_type: &'static str,
/// Always `podcast` (music hits: `artist`, `release`, `track`).
pub(crate) kind: &'static str,
pub(crate) podcast_id: String,
/// The origin feed URL: the only URL to subscribe with.
pub(crate) feed_url: String,
pub(crate) title: String,
pub(crate) author: Option<String>,
pub(crate) description: Option<String>,
/// Absolute `http(s)` artwork URL, if the card has a valid one.
pub(crate) image_url: Option<String>,
/// Absolute `http(s)` web page, if the card has a valid one.
pub(crate) link: Option<String>,
pub(crate) language: Option<String>,
pub(crate) categories: Vec<String>,
pub(crate) episode_count: Option<u32>,
pub(crate) latest_episode_at_ms: Option<i64>,
/// Distinct publisher devices (not subscribers, not a trust signal).
pub(crate) publisher_count: usize,
/// Position in the merged music and podcast results.
pub(crate) rank: usize,
/// The normalized title equals the normalized query.
pub(crate) exact_match: bool,
/// The account has an active subscription to this podcast.
pub(crate) subscribed: bool,
/// The account's podcast for this hit (its id, or the podcast a known
/// feed URL or alias resolves to), when the account knows it.
pub(crate) local_podcast_id: Option<String>,
}
/// An absolute `http(s)` URL without credentials, at most 2 048 bytes.
pub(crate) fn http_url(value: Option<&str>) -> Option<String> {
let value = value?.trim();
if value.is_empty() || value.len() > MAX_URL_BYTES {
return None;
}
let url = reqwest::Url::parse(value).ok()?;
(matches!(url.scheme(), "http" | "https")
&& url.host_str().is_some()
&& url.username().is_empty()
&& url.password().is_none())
.then(|| value.to_string())
}
/// Podcast hits in their merged order with the music results:
/// `(rank, exact_match, hit)`.
pub(crate) fn ranked_hits(
query: &str,
music: &[LibraryItem],
podcasts: Vec<PodcastSearchHit>,
) -> Vec<(usize, bool, PodcastSearchHit)> {
let normalized = normalize_name(query);
merge_search_results(query, music.to_vec(), podcasts)
.into_iter()
.enumerate()
.filter_map(|(rank, hit)| match hit {
FederatedSearchHit::Podcast(hit) => {
let exact = !normalized.is_empty() && normalize_name(&hit.card.title) == normalized;
Some((rank, exact, hit))
}
FederatedSearchHit::Music(_) => None,
})
.collect()
}
/// The account's podcast for a found podcast: the podcast with that id, or
/// the podcast one of `feed_urls` resolves to through the account's known
/// feed URLs and aliases (the card's origin URL; after a fetch also where
/// the feed moved, see `runner::document_urls`).
///
/// A podcast the account is subscribed to comes first, so an old,
/// unsubscribed podcast of the same feed (followed once under its former
/// address) never hides the current subscription; otherwise the first
/// known one, in the order id, then `feed_urls`.
pub(crate) async fn known_podcast(
store: &PodcastStore,
user_id: i64,
podcast_id: &PodcastId,
feed_urls: &[FeedUrl],
) -> Result<Option<PodcastSummary>> {
let mut ids = vec![podcast_id.clone()];
for feed_url in feed_urls {
let resolved = store.resolve_podcast_id(user_id, feed_url).await?;
if !ids.contains(&resolved) {
ids.push(resolved);
}
}
let mut first = None;
for id in ids {
let Some(summary) = store.podcast(user_id, &id).await? else {
continue;
};
if summary.subscribed {
return Ok(Some(summary));
}
first.get_or_insert(summary);
}
Ok(first)
}
/// Views of the podcast hits of one search for `user_id`, in rank order.
pub(crate) async fn hit_views(
store: &PodcastStore,
user_id: i64,
query: &str,
music: &[LibraryItem],
podcasts: Vec<PodcastSearchHit>,
) -> Result<Vec<PodcastHitView>> {
let mut views = Vec::new();
for (rank, exact_match, hit) in ranked_hits(query, music, podcasts) {
let known = known_podcast(
store,
user_id,
&hit.podcast_id,
std::slice::from_ref(&hit.feed_url),
)
.await?;
let card = hit.card;
views.push(PodcastHitView {
hit_type: "podcast",
kind: "podcast",
podcast_id: hit.podcast_id.to_string(),
feed_url: hit.feed_url.to_string(),
title: card.title,
author: card.author,
description: card.description,
image_url: http_url(card.image_url.as_deref()),
link: http_url(card.link.as_deref()),
language: card.language,
categories: card.categories,
episode_count: card.episode_count,
latest_episode_at_ms: card.latest_episode_at_ms,
publisher_count: hit.publishers.len(),
rank,
exact_match,
subscribed: known.as_ref().is_some_and(|summary| summary.subscribed),
local_podcast_id: known.map(|summary| summary.podcast_id.to_string()),
});
}
Ok(views)
}
/// One season group of a preview (`season: null`: episodes without one).
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct PreviewSeason {
pub(crate) season: Option<u32>,
pub(crate) episode_count: usize,
}
/// One episode of a preview.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct PreviewEpisode {
pub(crate) title: String,
pub(crate) published_at_ms: Option<i64>,
pub(crate) duration_ms: Option<i64>,
pub(crate) season: Option<u32>,
pub(crate) episode_number: Option<u32>,
}
/// A found podcast read from its feed: channel, seasons and the newest
/// episodes. Nothing of it is stored.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct PodcastPreview {
pub(crate) podcast_id: String,
/// The URL that was read (the card's origin URL).
pub(crate) feed_url: String,
/// The feed answered with a permanent redirect chain to another URL.
pub(crate) moved: bool,
pub(crate) title: String,
pub(crate) author: Option<String>,
pub(crate) description: Option<String>,
pub(crate) image_url: Option<String>,
pub(crate) link: Option<String>,
pub(crate) episode_count: usize,
pub(crate) seasons: Vec<PreviewSeason>,
/// Episodes of the requested season (all episodes without a filter).
pub(crate) listed_count: usize,
/// The newest of them, at most [`PREVIEW_EPISODES`].
pub(crate) episodes: Vec<PreviewEpisode>,
pub(crate) subscribed: bool,
pub(crate) local_podcast_id: Option<String>,
}
/// The preview of a parsed feed (`known`: the account's podcast for it),
/// listing the episodes of `season`.
pub(crate) fn preview(
podcast_id: &PodcastId,
feed_url: &FeedUrl,
moved: bool,
feed: &ParsedFeed,
known: Option<&PodcastSummary>,
season: SeasonFilter,
) -> PodcastPreview {
let mut seasons: BTreeMap<Option<u32>, usize> = BTreeMap::new();
for episode in &feed.episodes {
*seasons.entry(episode.season).or_default() += 1;
}
let mut episodes: Vec<&_> = feed
.episodes
.iter()
.filter(|episode| match season {
SeasonFilter::Any => true,
SeasonFilter::Season(number) => episode.season == Some(number),
SeasonFilter::NoSeason => episode.season.is_none(),
})
.collect();
let listed_count = episodes.len();
// Newest first; undated episodes keep their document order at the end.
episodes.sort_by_key(|episode| std::cmp::Reverse(episode.published_at_ms));
PodcastPreview {
podcast_id: podcast_id.to_string(),
feed_url: feed_url.to_string(),
moved,
title: feed.title.clone(),
author: feed.author.clone(),
description: feed.description.clone(),
image_url: http_url(feed.image_url.as_deref()),
link: http_url(feed.link.as_deref()),
episode_count: feed.episodes.len(),
// Numbered seasons first, then the episodes without a season.
seasons: seasons
.iter()
.filter(|(season, _)| season.is_some())
.chain(seasons.iter().filter(|(season, _)| season.is_none()))
.map(|(season, count)| PreviewSeason {
season: *season,
episode_count: *count,
})
.collect(),
listed_count,
episodes: episodes
.into_iter()
.take(PREVIEW_EPISODES)
.map(|episode| PreviewEpisode {
title: episode.title.clone(),
published_at_ms: episode.published_at_ms,
duration_ms: episode.duration_ms,
season: episode.season,
episode_number: episode.episode_number,
})
.collect(),
subscribed: known.is_some_and(|summary| summary.subscribed),
local_podcast_id: known.map(|summary| summary.podcast_id.to_string()),
}
}
#[cfg(test)]
mod tests {
use super::*;
use music_dht::podcast_dht::{PodcastCard, PodcastCardSpec};
use music_dht::{ItemId, ItemKind, NetworkId, SecretKey};
fn hit(path: &str, title: &str) -> PodcastSearchHit {
let feed_url = FeedUrl::parse(&format!("https://feeds.example/{path}")).unwrap();
let podcast_id = PodcastId::from_feed_url(&feed_url);
let publisher = SecretKey::from_bytes(&[7; 32]).public();
let card = PodcastCard::from_spec(
PodcastCardSpec {
podcast_id: podcast_id.clone(),
feed_url: feed_url.clone(),
id_feed_url: None,
title: title.into(),
author: None,
description: None,
image_url: None,
link: None,
language: None,
categories: Vec::new(),
episode_count: None,
latest_episode_at_ms: None,
},
NetworkId::from_name("search-test"),
publisher,
1,
1_700_000_000_000,
);
PodcastSearchHit {
podcast_id,
feed_url,
card,
publishers: vec![publisher],
claimed_feed_urls: Vec::new(),
}
}
fn music(name: &str) -> LibraryItem {
LibraryItem {
id: ItemId::from_bytes([1; 32]),
owner: SecretKey::from_bytes(&[5; 32]).public(),
kind: ItemKind::Track,
name: name.into(),
normalized_name: normalize_name(name),
artist_names: Vec::new(),
featured_artist_names: Vec::new(),
year: None,
release_type: None,
release_title: None,
track_number: None,
disc_number: None,
duration_seconds: None,
content_id: None,
revision: 1,
deleted: false,
updated_at_ms: 1,
}
}
#[test]
fn podcast_hits_take_their_place_among_music_results() {
let ranked = ranked_hits(
"Night Radio",
&[music("Night Radio Theme"), music("Night Radio")],
vec![
hit("a.xml", "Night Radio"),
hit("b.xml", "Night Radio Talks"),
],
);
let ranks: Vec<_> = ranked
.iter()
.map(|(rank, exact, hit)| (*rank, *exact, hit.card.title.as_str()))
.collect();
// Exact matches lead (music first inside a tier), then the rest.
assert_eq!(
ranks,
vec![(1, true, "Night Radio"), (3, false, "Night Radio Talks")]
);
}
#[test]
fn only_absolute_http_urls_without_credentials_are_kept() {
for (value, kept) in [
("https://cdn.example/art.jpg", true),
("http://example.org/show", true),
("javascript:alert(1)", false),
("data:image/png;base64,AAAA", false),
("//cdn.example/art.jpg", false),
("https://user:secret@cdn.example/art.jpg", false),
("file:///etc/passwd", false),
("", false),
] {
assert_eq!(http_url(Some(value)).is_some(), kept, "{value}");
}
let long = format!("https://cdn.example/{}", "a".repeat(MAX_URL_BYTES));
assert_eq!(http_url(Some(&long)), None);
assert_eq!(http_url(None), None);
}
#[tokio::test]
async fn podcast_hits_that_cannot_be_prepared_are_reported_as_failed() {
use crate::federation::client::{MusicSearch, PodcastSearch};
// The federation handle of a unit test has no database: ranking the
// hits for the account fails. The search still completes.
let federation = crate::federation::handle();
let music = MusicSearch {
events: Vec::new(),
items: vec![music("Night Radio Theme")],
};
let (state, events) = federation
.podcast_search_events(
"search",
"night radio",
1,
&music,
PodcastSearch::Found(vec![hit("a.xml", "Night Radio")]),
)
.await;
assert_eq!(state, "failed");
assert!(events.is_empty());
for (search, expected) in [
(PodcastSearch::Found(Vec::new()), "running"),
(PodcastSearch::Off, "off"),
(PodcastSearch::Failed, "failed"),
] {
let (state, events) = federation
.podcast_search_events("search", "night radio", 1, &music, search)
.await;
assert_eq!(state, expected);
assert!(events.is_empty());
}
}
#[test]
fn a_preview_lists_seasons_and_the_newest_episodes() {
let xml = r#"<?xml version="1.0"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"><channel>
<title>Preview Show</title><link>javascript:alert(1)</link>
<item><title>Old</title><guid>1</guid><itunes:season>1</itunes:season>
<pubDate>Mon, 01 Jan 2024 10:00:00 GMT</pubDate>
<enclosure url="https://cdn.example/1.mp3" type="audio/mpeg"/></item>
<item><title>Bonus</title><guid>2</guid>
<enclosure url="https://cdn.example/2.mp3" type="audio/mpeg"/></item>
<item><title>New</title><guid>3</guid><itunes:season>2</itunes:season>
<pubDate>Mon, 01 Jul 2024 10:00:00 GMT</pubDate>
<enclosure url="https://cdn.example/3.mp3" type="audio/mpeg"/></item>
</channel></rss>"#;
let url = FeedUrl::parse("https://feeds.example/preview.xml").unwrap();
let feed =
music_dht::podcast::feed::parse_feed_with_base(xml.as_bytes(), Some(url.as_str()))
.unwrap();
let id = PodcastId::from_feed_url(&url);
let preview_of = |season| preview(&id, &url, false, &feed, None, season);
let preview = preview_of(SeasonFilter::Any);
assert_eq!(preview.title, "Preview Show");
assert_eq!(preview.link, None, "only http(s) links");
assert_eq!(preview.episode_count, 3);
let seasons: Vec<_> = preview
.seasons
.iter()
.map(|season| (season.season, season.episode_count))
.collect();
assert_eq!(seasons, vec![(Some(1), 1), (Some(2), 1), (None, 1)]);
let titles: Vec<_> = preview.episodes.iter().map(|e| e.title.as_str()).collect();
assert_eq!(titles, vec!["New", "Old", "Bonus"]);
assert_eq!(preview.listed_count, 3);
assert!(!preview.subscribed && preview.local_podcast_id.is_none());
// One season (older seasons stay reachable), or the episodes
// without a season; the season groups stay complete.
let first = preview_of(SeasonFilter::Season(1));
let titles: Vec<_> = first.episodes.iter().map(|e| e.title.as_str()).collect();
assert_eq!(titles, vec!["Old"]);
assert_eq!((first.listed_count, first.seasons.len()), (1, 3));
let unnumbered = preview_of(SeasonFilter::NoSeason);
assert_eq!(unnumbered.episodes[0].title, "Bonus");
assert_eq!(unnumbered.listed_count, 1);
assert!(preview_of(SeasonFilter::Season(9)).episodes.is_empty());
}
}
+798
View File
@@ -0,0 +1,798 @@
//! Public podcasts in the federated search against PostgreSQL: the search
//! events (music unchanged, podcast hits added), the "already subscribed"
//! marker and subscribing from a card with the card's id.
//!
//! Ignored by default, like the other store tests: set
//! `FURU_DEVICE_SYNC_TEST_DATABASE_URL` to a disposable database. Feeds come
//! from a local test HTTP server.
use std::collections::BTreeSet;
use music_dht::device_sync::{SyncOpPayload, SyncOpWire, random_hex};
use music_dht::podcast::{FeedUrl, PodcastId, PodcastMeta};
use music_dht::podcast_dht::{PodcastCard, PodcastCardSpec, PodcastSearchHit};
use music_dht::{ItemId, ItemKind, LibraryItem, NetworkId, SecretKey, normalize_name};
use serde_json::{Value, json};
use super::api;
use super::postgres_tests::{cleanup, local_fetcher, random_user, test_pool};
use super::test_server::{Route, TestServer};
use super::*;
use crate::federation::client::{music_search_events, podcast_events};
fn url(value: &str) -> FeedUrl {
FeedUrl::parse(value).unwrap()
}
/// A search hit as `PodcastDht::search` returns it, for the origin URL
/// `feed` (published by one device).
fn hit(feed: &str, title: &str) -> PodcastSearchHit {
let feed_url = url(feed);
let podcast_id = PodcastId::from_feed_url(&feed_url);
let publisher = SecretKey::from_bytes(&[9; 32]).public();
let card = PodcastCard::from_spec(
PodcastCardSpec {
podcast_id: podcast_id.clone(),
feed_url: feed_url.clone(),
id_feed_url: None,
title: title.into(),
author: Some("Host".into()),
description: Some("<b>Not</b> markup".into()),
image_url: Some("https://cdn.example/art.jpg".into()),
link: Some("https://show.example/".into()),
language: Some("en".into()),
categories: vec!["Technology".into()],
episode_count: Some(12),
latest_episode_at_ms: Some(1_700_000_000_000),
},
NetworkId::from_name("search-tests"),
publisher,
1,
1_700_000_000_000,
);
PodcastSearchHit {
podcast_id,
feed_url,
card,
publishers: vec![publisher],
claimed_feed_urls: Vec::new(),
}
}
fn item(kind: ItemKind, name: &str, content_id: Option<&str>) -> LibraryItem {
LibraryItem {
id: ItemId::from_bytes(*blake3::hash(name.as_bytes()).as_bytes()),
owner: SecretKey::from_bytes(&[5; 32]).public(),
kind,
name: name.into(),
normalized_name: normalize_name(name),
artist_names: vec!["Night Owls".into()],
featured_artist_names: Vec::new(),
year: Some(2024),
release_type: None,
release_title: (kind == ItemKind::Track).then(|| "Night Radio".into()),
track_number: Some(1),
disc_number: None,
duration_seconds: Some(180.0),
content_id: content_id.map(str::to_string),
revision: 1,
deleted: false,
updated_at_ms: 1,
}
}
fn keys(value: &Value) -> BTreeSet<String> {
value.as_object().unwrap().keys().cloned().collect()
}
fn set(names: &[&str]) -> BTreeSet<String> {
names.iter().map(|name| name.to_string()).collect()
}
fn meta(feed: &str, title: &str) -> PodcastMeta {
PodcastMeta {
feed_url: url(feed),
title: title.into(),
author: None,
description: None,
link: None,
image_url: None,
}
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_federated_search_keeps_music_events_and_adds_podcast_hits() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let (user, other) = (random_user(), random_user());
cleanup(&pool, [user, other]).await;
let tag = random_hex(6);
let content_id = format!("b3:{}", blake3::hash(tag.as_bytes()).to_hex());
// Music: the contract of the existing events is unchanged; only the
// `type` marker is added.
let music = vec![
item(ItemKind::Track, "Night Radio Theme", Some(&content_id)),
item(ItemKind::Artist, "Night Owls", None),
item(ItemKind::Release, "Night Radio", None),
];
let own = SecretKey::from_bytes(&[1; 32]).public();
let music_events = music_search_events(&pool, "search-1", &music, own)
.await
.unwrap();
let music_json: Vec<Value> = music_events
.iter()
.map(|event| serde_json::to_value(event).unwrap())
.collect();
let kinds: Vec<&str> = music_json
.iter()
.map(|event| event["kind"].as_str().unwrap())
.collect();
assert_eq!(
kinds,
vec![
"federation.track",
"federation.artist",
"federation.release"
]
);
for (index, event) in music_json.iter().enumerate() {
assert_eq!(
keys(event),
set(&[
"search_id",
"sequence",
"kind",
"type",
"peer",
"entity_key",
"entity"
])
);
assert_eq!(event["type"], "music");
assert_eq!(event["search_id"], "search-1");
assert_eq!(event["sequence"], index as u64 + 1);
}
let track = &music_json[0];
assert_eq!(track["entity_key"], json!({ "content_id": content_id }));
assert_eq!(
keys(&track["entity"]),
set(&["key", "metadata", "availability"])
);
assert_eq!(track["entity"]["metadata"]["title"], "Night Radio Theme");
assert_eq!(track["entity"]["availability"]["state"], "federated");
assert_eq!(track["peer"], music[0].owner.to_string());
assert_eq!(
keys(&music_json[1]["entity"]),
set(&["key", "name", "image_url", "peers"])
);
assert_eq!(
keys(&music_json[2]["entity"]),
set(&["key", "title", "artists", "year", "cover_url", "sources"])
);
// The account has podcast A, and podcast C through an alias.
let feed_a = format!("https://podcasts.example/a-{tag}.xml");
let feed_b = format!("https://podcasts.example/b-{tag}.xml");
let feed_c = format!("https://podcasts.example/c-{tag}.xml");
let alias_c = format!("https://moved.example/c-{tag}.xml");
let id_a = store
.subscribe(user, &meta(&feed_a, "Night Radio"))
.await
.unwrap()
.podcast_id;
let id_c = store
.subscribe(user, &meta(&feed_c, "Cellar Talks"))
.await
.unwrap()
.podcast_id;
let aliases = SyncOpWire {
op_id: format!("tui-{tag}:1"),
origin_device_id: format!("tui-{tag}"),
seq: 1,
hlc_ms: 9_000_000_000_000,
payload: SyncOpPayload::PodcastFeedAliasesAdded {
podcast_id: id_c.clone(),
feed_aliases: vec![url(&feed_c), url(&alias_c)],
},
};
store.apply_remote_op(user, &aliases).await.unwrap();
// Someone published a card for the alias URL: another podcast id, but
// the account already has it.
let hits = vec![
hit(&feed_b, "Night Radio Stories"),
hit(&alias_c, "Night Radio Cellar"),
hit(&feed_a, "Night Radio"),
];
let first = music_events.len() as u64 + 1;
let events = podcast_events(
&pool,
user,
"search-1",
"night radio",
&music,
first,
hits.clone(),
)
.await
.unwrap();
let json: Vec<Value> = events
.iter()
.map(|event| serde_json::to_value(event).unwrap())
.collect();
assert_eq!(json.len(), 3);
for (index, event) in json.iter().enumerate() {
assert_eq!(event["kind"], "federation.podcast");
assert_eq!(event["type"], "podcast");
assert_eq!(event["peer"], Value::Null);
assert_eq!(event["sequence"], first + index as u64);
assert_eq!(event["entity"]["type"], "podcast");
assert_eq!(event["entity"]["kind"], "podcast");
assert_eq!(
event["entity_key"],
json!({ "podcast_id": event["entity"]["podcast_id"] })
);
// The publisher's personal state never appears: no registers,
// devices or publisher keys, only the card's public projection.
assert_eq!(
keys(&event["entity"]),
set(&[
"type",
"kind",
"podcast_id",
"feed_url",
"title",
"author",
"description",
"image_url",
"link",
"language",
"categories",
"episode_count",
"latest_episode_at_ms",
"publisher_count",
"rank",
"exact_match",
"subscribed",
"local_podcast_id",
])
);
}
// The exact title leads (after the exact music release), then the rest
// in the overlay's order.
let entity = |index: usize| &json[index]["entity"];
assert_eq!(entity(0)["podcast_id"], id_a.to_string());
assert_eq!(entity(0)["exact_match"], true);
assert_eq!(entity(0)["rank"], 1);
assert_eq!(entity(0)["subscribed"], true);
assert_eq!(entity(0)["local_podcast_id"], id_a.to_string());
assert_eq!(entity(0)["feed_url"], feed_a);
assert_eq!(entity(0)["description"], "<b>Not</b> markup");
assert_eq!(entity(0)["publisher_count"], 1);
assert_eq!(entity(1)["podcast_id"], hits[0].podcast_id.to_string());
assert_eq!(entity(1)["subscribed"], false);
assert_eq!(entity(1)["local_podcast_id"], Value::Null);
assert_eq!(entity(2)["podcast_id"], hits[1].podcast_id.to_string());
assert_ne!(hits[1].podcast_id, id_c);
assert_eq!(entity(2)["subscribed"], true, "known through the alias");
assert_eq!(entity(2)["local_podcast_id"], id_c.to_string());
// The marker belongs to the searching account.
let theirs = podcast_events(&pool, other, "search-2", "night radio", &music, 1, hits)
.await
.unwrap();
assert!(
theirs
.iter()
.all(|event| event.entity["subscribed"] == false
&& event.entity["local_podcast_id"].is_null())
);
// Unsubscribed: known, but no longer marked as subscribed.
api::unsubscribe(&store, user, &id_a).await.unwrap();
let events = podcast_events(
&pool,
user,
"search-3",
"night radio",
&[],
1,
vec![hit(&feed_a, "Night Radio")],
)
.await
.unwrap();
assert_eq!(events[0].entity["subscribed"], false);
assert_eq!(events[0].entity["local_podcast_id"], id_a.to_string());
assert_eq!(events[0].entity["rank"], 0);
sqlx::query("DELETE FROM furumusic__track_ref WHERE content_id = $1")
.bind(&content_id)
.execute(&pool)
.await
.unwrap();
cleanup(&pool, [user, other]).await;
}
fn rss(title: &str, items: &str) -> String {
format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel>
<title>{title}</title>
<link>https://show.example/</link>
<description>Found in the federation</description>
{items}
</channel>
</rss>"#
)
}
const ITEMS: &str = r#"
<item><title>Pilot</title><guid>s1e1</guid><itunes:season>1</itunes:season>
<itunes:episode>1</itunes:episode><pubDate>Mon, 01 Jan 2024 10:00:00 GMT</pubDate>
<enclosure url="/audio/s1e1.mp3" type="audio/mpeg" length="1000"/></item>
<item><title>Second</title><guid>s1e2</guid><itunes:season>1</itunes:season>
<itunes:episode>2</itunes:episode><pubDate>Mon, 08 Jan 2024 10:00:00 GMT</pubDate>
<enclosure url="/audio/s1e2.mp3" type="audio/mpeg" length="1000"/></item>
<item><title>Bonus</title><guid>bonus</guid>
<enclosure url="/audio/bonus.mp3" type="audio/mpeg" length="1000"/></item>"#;
async fn log_len(pool: &sqlx::PgPool, user: i64) -> i64 {
sqlx::query_scalar("SELECT COUNT(*) FROM furumusic__fed_sync_ops WHERE user_id = $1")
.bind(user)
.fetch_one(pool)
.await
.unwrap()
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_subscribing_from_a_card_keeps_the_card_id() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let (user, other) = (random_user(), random_user());
cleanup(&pool, [user, other]).await;
let server = TestServer::start();
server.set(
"/show.xml",
Route::new(rss("Found Show", ITEMS), "application/rss+xml"),
);
server.set("/old.xml", Route::redirect(301, "/new.xml"));
server.set(
"/new.xml",
Route::new(rss("Moved Show", ITEMS), "application/rss+xml"),
);
let fetcher = local_fetcher();
let now = super::now_ms();
let show = hit(&server.url("/show.xml"), "Found Show");
let body = |hit: &PodcastSearchHit| json!({ "podcast_id": hit.podcast_id.to_string(), "feed_url": hit.feed_url.to_string() });
// Invalid requests fetch nothing.
let other_id = PodcastId::from_feed_url(&url("https://podcasts.example/other.xml"));
for (request, status) in [
(json!({}), 400),
(
json!({ "podcast_id": "x", "feed_url": show.feed_url.to_string() }),
400,
),
(
json!({ "podcast_id": show.podcast_id.to_string(), "feed_url": "ftp://x/y" }),
400,
),
// A URL the id does not come from (a claimed move, another podcast).
(
json!({ "podcast_id": other_id.to_string(), "feed_url": show.feed_url.to_string() }),
422,
),
] {
let error = api::preview_found(&store, &fetcher, user, request.clone())
.await
.unwrap_err();
assert_eq!(error.status, status, "{request}: {}", error.message);
let error = api::subscribe_found(&store, &fetcher, user, request.clone(), now)
.await
.unwrap_err();
assert_eq!(error.status, status, "{request}: {}", error.message);
}
assert_eq!(server.requests("/show.xml").len(), 0);
// The production policy never reaches the local server.
let refused = api::preview_found(&store, &FeedFetcher::new().unwrap(), user, body(&show))
.await
.unwrap_err();
assert_eq!(refused.status, 422, "{}", refused.message);
// The preview reads the feed and stores nothing.
let preview = api::preview_found(&store, &fetcher, user, body(&show))
.await
.unwrap();
assert_eq!(preview.title, "Found Show");
assert_eq!(preview.podcast_id, show.podcast_id.to_string());
assert_eq!(preview.episode_count, 3);
let seasons: Vec<_> = preview
.seasons
.iter()
.map(|season| (season.season, season.episode_count))
.collect();
assert_eq!(seasons, vec![(Some(1), 2), (None, 1)]);
assert_eq!(preview.episodes[0].title, "Second");
assert!(!preview.subscribed && !preview.moved);
assert_eq!(server.requests("/show.xml").len(), 1);
assert!(
store
.podcast(user, &show.podcast_id)
.await
.unwrap()
.is_none()
);
assert_eq!(log_len(&pool, user).await, 0);
// Subscribing: the account's own private subscription with the card's id.
let subscribed = api::subscribe_found(&store, &fetcher, user, body(&show), now)
.await
.unwrap();
assert_eq!(subscribed.podcast.podcast_id, show.podcast_id.to_string());
assert!(!subscribed.already_subscribed);
assert_eq!(subscribed.new_episodes, 3);
assert!(subscribed.podcast.subscribed);
assert!(
!subscribed.podcast.shared,
"a subscription from a card is private"
);
assert_eq!(server.requests("/show.xml").len(), 2);
let marked = super::search::hit_views(&store, user, "found show", &[], vec![show.clone()])
.await
.unwrap();
assert!(marked[0].subscribed);
// Again: already subscribed, nothing fetched or written.
let log = log_len(&pool, user).await;
let again = api::subscribe_found(&store, &fetcher, user, body(&show), now)
.await
.unwrap();
assert!(again.already_subscribed);
assert_eq!(again.podcast.podcast_id, show.podcast_id.to_string());
assert_eq!(server.requests("/show.xml").len(), 2);
assert_eq!(log_len(&pool, user).await, log);
let preview = api::preview_found(&store, &fetcher, user, body(&show))
.await
.unwrap();
assert!(preview.subscribed);
assert_eq!(
preview.local_podcast_id.as_deref(),
Some(show.podcast_id.as_str())
);
// One season at a time; a bad season is refused before any fetch.
let mut seasonal = body(&show);
seasonal["season"] = json!("1");
let first = api::preview_found(&store, &fetcher, user, seasonal)
.await
.unwrap();
assert_eq!(first.listed_count, 2);
assert!(
first
.episodes
.iter()
.all(|episode| episode.season == Some(1))
);
let mut unnumbered = body(&show);
unnumbered["season"] = json!("none");
let none = api::preview_found(&store, &fetcher, user, unnumbered)
.await
.unwrap();
assert_eq!(none.listed_count, 1);
assert!(none.episodes.iter().all(|episode| episode.season.is_none()));
let requests = server.requests("/show.xml").len();
let mut bad = body(&show);
bad["season"] = json!("first");
let error = api::preview_found(&store, &fetcher, user, bad)
.await
.unwrap_err();
assert_eq!(error.status, 400);
assert_eq!(server.requests("/show.xml").len(), requests);
// Another account subscribing from the same card gets its own podcast
// with the same id; the first account's state is untouched.
let theirs = api::subscribe_found(&store, &fetcher, other, body(&show), now)
.await
.unwrap();
assert_eq!(theirs.podcast.podcast_id, show.podcast_id.to_string());
assert!(!theirs.already_subscribed);
// A card whose origin URL moved permanently: the id stays the card's,
// the new URL becomes an alias and is fetched from now on.
let old = hit(&server.url("/old.xml"), "Moved Show");
let moved = api::subscribe_found(&store, &fetcher, user, body(&old), now)
.await
.unwrap();
assert_eq!(moved.podcast.podcast_id, old.podcast_id.to_string());
assert_eq!(moved.podcast.feed_url, server.url("/new.xml"));
let aliases = store.feed_aliases(user, &old.podcast_id).await.unwrap();
assert!(aliases.contains(&url(&server.url("/new.xml"))));
// Someone's card for the new URL is the same podcast for this account.
let new = hit(&server.url("/new.xml"), "Moved Show");
assert_ne!(new.podcast_id, old.podcast_id);
let requests = server.requests("/new.xml").len();
let again = api::subscribe_found(&store, &fetcher, user, body(&new), now)
.await
.unwrap();
assert!(again.already_subscribed);
assert_eq!(again.podcast.podcast_id, old.podcast_id.to_string());
assert_eq!(server.requests("/new.xml").len(), requests);
assert!(
store
.podcast(user, &new.podcast_id)
.await
.unwrap()
.is_none()
);
// The reverse: an account that follows the feed only under its new
// address finds the card of the old one. The preview fetches the old
// URL, sees the move and marks it as already subscribed; subscribing
// writes nothing and creates no second podcast (nor does "add RSS").
let third = random_user();
let direct = runner::subscribe_url(&store, &fetcher, third, &url(&server.url("/new.xml")), now)
.await
.unwrap();
assert_eq!(direct.podcast_id, new.podcast_id);
let marked = super::search::hit_views(&store, third, "moved show", &[], vec![old.clone()])
.await
.unwrap();
// The search list cannot know without fetching the feed.
assert!(!marked[0].subscribed);
let preview = api::preview_found(&store, &fetcher, third, body(&old))
.await
.unwrap();
assert!(preview.moved && preview.subscribed);
assert_eq!(
preview.local_podcast_id.as_deref(),
Some(new.podcast_id.as_str())
);
let log = log_len(&pool, third).await;
let again = api::subscribe_found(&store, &fetcher, third, body(&old), now)
.await
.unwrap();
assert!(again.already_subscribed);
assert_eq!(again.podcast.podcast_id, new.podcast_id.to_string());
let added = runner::subscribe_url(&store, &fetcher, third, &url(&server.url("/old.xml")), now)
.await
.unwrap();
assert!(added.already_subscribed);
assert_eq!(added.podcast_id, new.podcast_id);
assert_eq!(log_len(&pool, third).await, log);
assert!(
store
.podcast(third, &old.podcast_id)
.await
.unwrap()
.is_none()
);
let subscriptions: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM furumusic__podcast_subscription WHERE user_id = $1",
)
.bind(third)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(subscriptions, 1);
cleanup(&pool, [user, other, third]).await;
}
/// The catalog row and the episodes of one podcast of the account:
/// `(title, description, meta_from_feed, [(episode title, in_feed)])`.
async fn podcast_snapshot(
pool: &sqlx::PgPool,
user: i64,
podcast_id: &PodcastId,
) -> (String, Option<String>, bool, Vec<(String, bool)>) {
let (title, description, meta_from_feed): (String, Option<String>, bool) = sqlx::query_as(
"SELECT title, description, meta_from_feed FROM furumusic__podcast
WHERE user_id = $1 AND podcast_id = $2",
)
.bind(user)
.bind(podcast_id.as_str())
.fetch_one(pool)
.await
.unwrap();
let episodes: Vec<(String, bool)> = sqlx::query_as(
"SELECT title, in_feed FROM furumusic__podcast_episode
WHERE user_id = $1 ORDER BY title",
)
.bind(user)
.fetch_all(pool)
.await
.unwrap();
(title, description, meta_from_feed, episodes)
}
async fn active_subscriptions(pool: &sqlx::PgPool, user: i64) -> i64 {
sqlx::query_scalar(
"SELECT COUNT(*) FROM furumusic__podcast_subscription
WHERE user_id = $1 AND subscribed",
)
.bind(user)
.fetch_one(pool)
.await
.unwrap()
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_a_feed_claiming_a_followed_address_never_rewrites_that_podcast() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let user = random_user();
cleanup(&pool, [user]).await;
let server = TestServer::start();
let real = server.url("/real.xml");
server.set(
"/real.xml",
Route::new(rss("Real Show", ITEMS), "application/rss+xml"),
);
// The old address serves a frozen copy that names the new address; a
// feed of somebody else claims to have moved there too.
let claim = |title: &str, guid: &str| {
rss(
title,
&format!(
r#"<itunes:new-feed-url>{real}</itunes:new-feed-url>
<item><title>{title} item</title><guid>{guid}</guid>
<enclosure url="/audio/{guid}.mp3" type="audio/mpeg" length="1000"/></item>"#
),
)
};
server.set(
"/stub.xml",
Route::new(claim("Stale Copy", "stale"), "application/rss+xml"),
);
server.set(
"/evil.xml",
Route::new(claim("Injected", "evil"), "application/rss+xml"),
);
let fetcher = local_fetcher();
let now = super::now_ms();
let real_id = PodcastId::from_feed_url(&url(&real));
let followed = runner::subscribe_url(&store, &fetcher, user, &url(&real), now)
.await
.unwrap();
assert_eq!(followed.podcast_id, real_id);
let before = podcast_snapshot(&pool, user, &real_id).await;
assert_eq!(before.0, "Real Show");
assert_eq!(before.3.len(), 3);
assert!(before.3.iter().all(|(_, in_feed)| *in_feed));
let register_before = store.subscription(user, &real_id).await.unwrap().unwrap();
let log = log_len(&pool, user).await;
// "Add RSS" of the old address and "Subscribe" on a card of the other
// feed: the account already follows the feed they name, nothing of
// theirs is written into it.
for added in [
runner::subscribe_url(&store, &fetcher, user, &url(&server.url("/stub.xml")), now)
.await
.unwrap(),
runner::subscribe_url(&store, &fetcher, user, &url(&server.url("/evil.xml")), now)
.await
.unwrap(),
] {
assert!(added.already_subscribed);
assert_eq!(added.podcast_id, real_id);
assert_eq!(added.refresh.new_episodes, 0);
}
let evil = hit(&server.url("/evil.xml"), "Injected");
let body =
json!({ "podcast_id": evil.podcast_id.to_string(), "feed_url": evil.feed_url.to_string() });
let found = api::subscribe_found(&store, &fetcher, user, body.clone(), now)
.await
.unwrap();
assert!(found.already_subscribed);
assert_eq!(found.podcast.podcast_id, real_id.to_string());
assert_eq!(podcast_snapshot(&pool, user, &real_id).await, before);
assert_eq!(log_len(&pool, user).await, log);
assert_eq!(active_subscriptions(&pool, user).await, 1);
// Unsubscribed, the same claim subscribes the account's podcast again
// with its own stored metadata; the claiming document is not ingested
// (the refresh job fetches the podcast's own address).
store.unsubscribe(user, &real_id).await.unwrap();
let again = runner::subscribe_url(&store, &fetcher, user, &url(&server.url("/evil.xml")), now)
.await
.unwrap();
assert!(!again.already_subscribed);
assert_eq!(again.podcast_id, real_id);
assert_eq!(again.refresh.new_episodes, 0);
assert_eq!(podcast_snapshot(&pool, user, &real_id).await, before);
let register = store.subscription(user, &real_id).await.unwrap().unwrap();
assert!(register.subscribed);
assert_eq!(register.podcast, register_before.podcast);
assert!(
store
.podcast(user, &evil.podcast_id)
.await
.unwrap()
.is_none()
);
// Its own address is still ingested as before (a refresh).
let own = runner::subscribe_url(&store, &fetcher, user, &url(&real), now)
.await
.unwrap();
assert!(own.already_subscribed);
assert_eq!(own.refresh.unchanged_episodes, 3);
cleanup(&pool, [user]).await;
}
#[tokio::test]
#[ignore = "requires a disposable database in FURU_DEVICE_SYNC_TEST_DATABASE_URL"]
async fn postgres_a_current_subscription_wins_over_an_old_unsubscribed_podcast() {
let pool = test_pool().await;
let store = PodcastStore::new(pool.clone());
let user = random_user();
cleanup(&pool, [user]).await;
let server = TestServer::start();
server.set(
"/old.xml",
Route::new(rss("Moving Show", ITEMS), "application/rss+xml"),
);
server.set(
"/new.xml",
Route::new(rss("Moving Show", ITEMS), "application/rss+xml"),
);
let fetcher = local_fetcher();
let now = super::now_ms();
let old = hit(&server.url("/old.xml"), "Moving Show");
let new_id = PodcastId::from_feed_url(&url(&server.url("/new.xml")));
// Followed once under the old address, then unsubscribed; the feed
// moved, and the account follows it under the new address only.
runner::subscribe_url(&store, &fetcher, user, &old.feed_url, now)
.await
.unwrap();
store.unsubscribe(user, &old.podcast_id).await.unwrap();
server.set("/old.xml", Route::redirect(301, "/new.xml"));
let current = runner::subscribe_url(&store, &fetcher, user, &url(&server.url("/new.xml")), now)
.await
.unwrap();
assert_eq!(current.podcast_id, new_id);
assert!(!current.already_subscribed);
let known = super::search::known_podcast(
&store,
user,
&old.podcast_id,
&[old.feed_url.clone(), url(&server.url("/new.xml"))],
)
.await
.unwrap()
.unwrap();
assert_eq!(known.podcast_id, new_id);
assert!(known.subscribed);
let body =
json!({ "podcast_id": old.podcast_id.to_string(), "feed_url": old.feed_url.to_string() });
let preview = api::preview_found(&store, &fetcher, user, body.clone())
.await
.unwrap();
assert!(preview.subscribed);
assert_eq!(preview.local_podcast_id.as_deref(), Some(new_id.as_str()));
let log = log_len(&pool, user).await;
let found = api::subscribe_found(&store, &fetcher, user, body, now)
.await
.unwrap();
assert!(found.already_subscribed);
assert_eq!(found.podcast.podcast_id, new_id.to_string());
let added = runner::subscribe_url(&store, &fetcher, user, &old.feed_url, now)
.await
.unwrap();
assert!(added.already_subscribed);
assert_eq!(added.podcast_id, new_id);
assert_eq!(log_len(&pool, user).await, log);
assert_eq!(active_subscriptions(&pool, user).await, 1);
let old_podcast = store.podcast(user, &old.podcast_id).await.unwrap().unwrap();
assert!(!old_podcast.subscribed);
cleanup(&pool, [user]).await;
}
+375
View File
@@ -0,0 +1,375 @@
//! Test support of the podcast runtime: a scripted resolver, a client that
//! may reach loopback, and a minimal HTTP/1.1 server on 127.0.0.1 (no
//! internet). One request per connection (`Connection: close`), so every
//! request opens a new connection and resolves its host again.
//!
//! Adapted from the furumi-tui prototype's test server: routes can redirect,
//! send extra headers and answer conditional requests, and every request's
//! headers are recorded.
use std::collections::{HashMap, VecDeque};
use std::io::{BufRead, BufReader, Write};
use std::net::{IpAddr, TcpListener, TcpStream};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
use super::net::{AddressPolicy, HostLookup, LookupFuture, SafeHttp, SafeHttpConfig};
#[derive(Clone)]
pub(crate) struct Route {
pub(crate) status: u16,
pub(crate) body: Arc<Vec<u8>>,
pub(crate) content_type: Option<String>,
pub(crate) etag: Option<String>,
pub(crate) last_modified: Option<String>,
/// Honour `Range` requests with 206 responses.
pub(crate) ranges: bool,
/// Send `Content-Length` (otherwise the body is close-delimited).
pub(crate) send_length: bool,
/// Close the connection once the body reaches this absolute offset.
pub(crate) cut_after: Option<usize>,
/// Extra response headers.
pub(crate) headers: Vec<(String, String)>,
/// Pause between the headers and the body.
pub(crate) body_delay: Option<Duration>,
}
impl Route {
pub(crate) fn new(body: impl Into<Vec<u8>>, content_type: &str) -> Route {
Route {
status: 200,
body: Arc::new(body.into()),
content_type: Some(content_type.to_string()),
etag: None,
last_modified: None,
ranges: true,
send_length: true,
cut_after: None,
headers: Vec::new(),
body_delay: None,
}
}
/// A redirect to `location` with `status`.
pub(crate) fn redirect(status: u16, location: &str) -> Route {
Route {
status,
body: Arc::new(Vec::new()),
content_type: None,
ranges: false,
..Route::new(Vec::new(), "text/plain")
}
.header("Location", location)
}
pub(crate) fn status(mut self, status: u16) -> Route {
self.status = status;
self
}
pub(crate) fn etag(mut self, etag: &str) -> Route {
self.etag = Some(etag.to_string());
self
}
pub(crate) fn last_modified(mut self, date: &str) -> Route {
self.last_modified = Some(date.to_string());
self
}
pub(crate) fn no_ranges(mut self) -> Route {
self.ranges = false;
self
}
pub(crate) fn no_length(mut self) -> Route {
self.send_length = false;
self
}
pub(crate) fn no_content_type(mut self) -> Route {
self.content_type = None;
self
}
pub(crate) fn cut(mut self, after: usize) -> Route {
self.cut_after = Some(after);
self
}
pub(crate) fn delay_body(mut self, delay: Duration) -> Route {
self.body_delay = Some(delay);
self
}
pub(crate) fn header(mut self, name: &str, value: &str) -> Route {
self.headers.push((name.to_string(), value.to_string()));
self
}
}
/// Lowercased request headers of one request.
pub(crate) type RequestHeaders = HashMap<String, String>;
#[derive(Default)]
struct ServerState {
routes: HashMap<String, Route>,
requests: HashMap<String, Vec<RequestHeaders>>,
}
pub(crate) struct TestServer {
port: u16,
state: Arc<Mutex<ServerState>>,
}
impl TestServer {
pub(crate) fn start() -> TestServer {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server");
let port = listener.local_addr().expect("local addr").port();
let state = Arc::new(Mutex::new(ServerState::default()));
let accept_state = Arc::clone(&state);
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(stream) = stream else { continue };
let state = Arc::clone(&accept_state);
std::thread::spawn(move || handle(stream, &state));
}
});
TestServer { port, state }
}
/// `http://127.0.0.1:<port><path>`.
pub(crate) fn url(&self, path: &str) -> String {
format!("http://127.0.0.1:{}{path}", self.port)
}
/// `http://<host>:<port><path>` (for hosts mapped by a test resolver).
pub(crate) fn url_on(&self, host: &str, path: &str) -> String {
format!("http://{host}:{}{path}", self.port)
}
pub(crate) fn set(&self, path: &str, route: Route) {
self.lock().routes.insert(path.to_string(), route);
}
/// Headers of every request to `path`, in order.
pub(crate) fn requests(&self, path: &str) -> Vec<RequestHeaders> {
self.lock().requests.get(path).cloned().unwrap_or_default()
}
/// Requests to any path.
pub(crate) fn request_count(&self) -> usize {
self.lock().requests.values().map(Vec::len).sum()
}
fn lock(&self) -> std::sync::MutexGuard<'_, ServerState> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
}
fn handle(stream: TcpStream, state: &Mutex<ServerState>) {
let _ = stream.set_read_timeout(Some(Duration::from_secs(10)));
let mut reader = BufReader::new(match stream.try_clone() {
Ok(stream) => stream,
Err(_) => return,
});
let mut request_line = String::new();
if reader.read_line(&mut request_line).is_err() {
return;
}
let path = request_line
.split_whitespace()
.nth(1)
.unwrap_or("/")
.to_string();
let mut headers = HashMap::new();
loop {
let mut line = String::new();
match reader.read_line(&mut line) {
Ok(0) | Err(_) => return,
Ok(_) => {}
}
let line = line.trim_end();
if line.is_empty() {
break;
}
if let Some((name, value)) = line.split_once(':') {
headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string());
}
}
let route = {
let mut state = state.lock().unwrap_or_else(PoisonError::into_inner);
state
.requests
.entry(path.clone())
.or_default()
.push(headers.clone());
state.routes.get(&path).cloned()
};
let mut out = stream;
let Some(route) = route else {
let _ = out
.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
return;
};
let not_modified = match (&route.etag, headers.get("if-none-match")) {
(Some(etag), Some(sent)) => etag == sent,
_ => matches!(
(&route.last_modified, headers.get("if-modified-since")),
(Some(date), Some(sent)) if date == sent
),
};
if route.status == 200 && not_modified {
let _ = write!(
out,
"HTTP/1.1 304 Not Modified\r\nConnection: close\r\n\r\n"
);
return;
}
let total = route.body.len();
let mut status = route.status;
let mut start = 0usize;
let mut end = total;
let mut extra = String::new();
if route.status == 200
&& route.ranges
&& let Some(range) = headers.get("range").and_then(|r| r.strip_prefix("bytes="))
&& let Some((from, to)) = range.split_once('-')
{
let (from, to) = match (from.trim().parse::<usize>(), to.trim().parse::<usize>()) {
(Ok(from), Ok(to)) => (from, (to + 1).min(total)),
(Ok(from), Err(_)) => (from, total),
(Err(_), Ok(suffix)) => (total.saturating_sub(suffix), total),
_ => (0, total),
};
if from >= total {
let _ = write!(
out,
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Range: bytes */{total}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
);
return;
}
status = 206;
start = from;
end = to;
extra.push_str(&format!(
"Content-Range: bytes {start}-{}/{total}\r\n",
end - 1
));
}
if route.ranges {
extra.push_str("Accept-Ranges: bytes\r\n");
}
if let Some(content_type) = &route.content_type {
extra.push_str(&format!("Content-Type: {content_type}\r\n"));
}
if let Some(etag) = &route.etag {
extra.push_str(&format!("ETag: {etag}\r\n"));
}
if let Some(date) = &route.last_modified {
extra.push_str(&format!("Last-Modified: {date}\r\n"));
}
for (name, value) in &route.headers {
extra.push_str(&format!("{name}: {value}\r\n"));
}
if route.send_length {
extra.push_str(&format!("Content-Length: {}\r\n", end - start));
}
if write!(
out,
"HTTP/1.1 {status} Status\r\n{extra}Connection: close\r\n\r\n"
)
.is_err()
{
return;
}
let _ = out.flush();
if let Some(delay) = route.body_delay {
std::thread::sleep(delay);
}
let limit = route.cut_after.map_or(end, |cut| cut.clamp(start, end));
for chunk in route.body[start..limit].chunks(64 * 1024) {
if out.write_all(chunk).is_err() {
return;
}
}
let _ = out.flush();
let _ = out.shutdown(std::net::Shutdown::Both);
}
/// Deterministic pseudo-random bytes.
pub(crate) fn noise(len: usize, seed: u64) -> Vec<u8> {
let mut state = seed | 1;
let mut out = Vec::with_capacity(len);
while out.len() < len {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
out.extend_from_slice(&state.to_le_bytes());
}
out.truncate(len);
out
}
// ---------------------------------------------------------------------------
// Resolver and client
// ---------------------------------------------------------------------------
/// Test resolver: each host answers from a script; the last answer repeats.
#[derive(Default)]
pub(crate) struct ScriptedLookup {
answers: std::sync::Mutex<HashMap<String, VecDeque<Vec<IpAddr>>>>,
pub(crate) calls: AtomicUsize,
}
impl ScriptedLookup {
pub(crate) fn with(entries: &[(&str, &[&[&str]])]) -> Arc<Self> {
let lookup = Self::default();
{
let mut answers = lookup.answers.lock().unwrap();
for (host, script) in entries {
answers.insert(
host.to_string(),
script
.iter()
.map(|answer| {
answer
.iter()
.map(|a| a.parse::<IpAddr>().unwrap())
.collect()
})
.collect(),
);
}
}
Arc::new(lookup)
}
}
impl HostLookup for ScriptedLookup {
fn lookup(&self, host: &str) -> LookupFuture {
self.calls.fetch_add(1, Ordering::SeqCst);
let mut answers = self.answers.lock().unwrap();
let answer = match answers.get_mut(host) {
Some(script) if script.len() > 1 => Ok(script.pop_front().unwrap()),
Some(script) => Ok(script.front().cloned().unwrap_or_default()),
None => Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("unknown test host {host}"),
)),
};
Box::pin(async move { answer })
}
}
pub(crate) const TEST_HTTP: SafeHttpConfig = SafeHttpConfig {
connect_timeout: Duration::from_secs(5),
read_timeout: Duration::from_secs(5),
total_timeout: Some(Duration::from_secs(20)),
max_redirects: 3,
};
pub(crate) fn test_http(lookup: Arc<ScriptedLookup>) -> SafeHttp {
SafeHttp::with_lookup(TEST_HTTP, AddressPolicy::test_allow_loopback(), lookup).unwrap()
}
+81
View File
@@ -0,0 +1,81 @@
//! Pure unit tests of the podcast store helpers (no database).
use music_dht::podcast::ProgressStatus;
use super::*;
#[test]
fn search_text_never_matches_across_title_and_description() {
let norm = episode_search_norm("Hello, World!", Some("WORLD peace"));
assert_eq!(norm, "hello world\nworld peace");
// A needle is itself normalized, so it never contains the separator.
let needle = music_dht::normalize_name("world world");
assert!(!norm.contains(&needle));
assert!(norm.contains(&music_dht::normalize_name("WORLD PEACE")));
assert_eq!(episode_search_norm("Only", None), "only\n");
}
#[test]
fn progress_status_round_trips_through_sql() {
for status in [
ProgressStatus::NotStarted,
ProgressStatus::InProgress,
ProgressStatus::Completed,
] {
assert_eq!(status_from_sql(status_to_sql(status)), status);
}
// The CHECK constraint admits exactly these values.
for sql in schema::CREATE_PODCAST_STORE {
if sql.contains("CREATE TABLE IF NOT EXISTS furumusic__podcast_progress (") {
for value in ["not_started", "in_progress", "completed"] {
assert!(sql.contains(&format!("'{value}'")));
}
}
}
}
#[test]
fn generations_beyond_bigint_are_rejected() {
assert_eq!(generation_to_sql(0).unwrap(), 0);
assert_eq!(generation_to_sql(i64::MAX as u64).unwrap(), i64::MAX);
assert!(matches!(
generation_to_sql(i64::MAX as u64 + 1),
Err(PodcastStoreError::Invalid(_))
));
assert_eq!(generation_from_sql(-1), 0);
}
#[test]
fn contract_numbers_outside_u32_are_dropped() {
assert_eq!(u32_from_sql(Some(7)), Some(7));
assert_eq!(u32_from_sql(Some(-1)), None);
assert_eq!(u32_from_sql(Some(i64::from(u32::MAX) + 1)), None);
assert_eq!(u32_from_sql(None), None);
}
#[test]
fn progress_columns_are_prefixed() {
assert_eq!(
progress_columns("g", "g_"),
"g.episode_id AS g_episode_id, g.podcast_id AS g_podcast_id, \
g.position_ms AS g_position_ms, g.status AS g_status, \
g.generation AS g_generation, g.session_id AS g_session_id, \
g.hlc_ms AS g_hlc_ms, g.device_id AS g_device_id, g.op_id AS g_op_id"
);
}
#[test]
fn queries_sort_undated_and_unnumbered_episodes_last() {
for sort in [
EpisodeSort::NewestFirst,
EpisodeSort::OldestFirst,
EpisodeSort::EpisodeNumber,
EpisodeSort::EpisodeNumberDesc,
] {
let order = queries::order_by_for_tests(sort);
assert!(
order.starts_with("e.published_at_ms IS NULL") || order.starts_with("e.season IS NULL")
);
assert!(order.ends_with("e.episode_id"), "{order}");
}
}
+1
View File
@@ -1311,6 +1311,7 @@ async fn run_scheduled_job(
&& job_name != "lastfm_scrobble"
&& job_name != "archive_cleanup"
&& job_name != "artwork_backfill"
&& job_name != "podcast_refresh"
{
tracing::warn!(job = job_name, "Skipping: agent_enabled=false");
return;
+1 -1
View File
@@ -70,7 +70,7 @@ html { overscroll-behavior:none; }
.task-form-body { overflow:visible; }
.run-log-body { min-height:300px; overflow:visible; }
.run-log-output { min-height:50dvh; max-height:70dvh; }
.settings-layout { grid-template-columns:minmax(0,1fr); grid-template-areas:"access" "oidc" "agent" "agentstatus" "similarity" "similaritystatus" "downloads" "federation" "lastfm" "developer" "actions"; }
.settings-layout { grid-template-columns:minmax(0,1fr); grid-template-areas:"access" "oidc" "agent" "agentstatus" "similarity" "similaritystatus" "downloads" "federation" "podcasts" "lastfm" "developer" "actions"; }
.settings-federation-body { grid-template-columns:minmax(0,1fr); }
.repair-shell { grid-template-columns:250px minmax(0,1fr); }
}
+2
View File
@@ -859,6 +859,7 @@ tbody tr:hover {
"similarity similarity similarity similarity similarity similarity similarity similarity similaritystatus similaritystatus similaritystatus similaritystatus"
"downloads downloads downloads downloads downloads downloads downloads downloads downloads downloads downloads downloads"
"federation federation federation federation federation federation federation federation federation federation federation federation"
"podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts"
"lastfm lastfm lastfm lastfm lastfm lastfm lastfm lastfm developer developer developer developer"
"actions actions actions actions actions actions actions actions actions actions actions actions";
gap: 16px;
@@ -881,6 +882,7 @@ tbody tr:hover {
.settings-similarity-status { grid-area: similaritystatus; }
.settings-downloads { grid-area: downloads; }
.settings-federation { grid-area: federation; }
.settings-podcasts { grid-area: podcasts; }
.settings-lastfm { grid-area: lastfm; }
.settings-developer { grid-area: developer; }
.settings-section-narrow { border-left: 2px solid rgba(29, 185, 84, 0.55); }
+23 -1
View File
@@ -1167,6 +1167,27 @@
</div>
</div>
</section>
<section class="panel settings-section settings-section-full settings-podcasts" id="settings-podcasts">
<div class="panel-head">
<div class="panel-title">
<strong>Podcasts</strong>
<span>Episode audio comes from the publisher</span>
</div>
</div>
<div class="settings-grid">
<div class="setting-toggle settings-wide">
<label>
<span>Podcast audio proxy</span>
<span class="source-pill" :class="sourceClass('podcast_audio_proxy_enabled')" x-text="settingSource('podcast_audio_proxy_enabled')"></span>
</label>
<div class="setting-toggle-row">
<span x-text="settingsDraft.podcast_audio_proxy_enabled ? 'Stream through this server when needed' : 'Browsers stream from the publisher'"></span>
<input type="checkbox" x-model="settingsDraft.podcast_audio_proxy_enabled" />
</div>
<div class="setting-help">By default the browser loads episode audio directly from the publisher. Enable the proxy for publishers the browser cannot play directly (missing CORS, plain HTTP on an HTTPS site). The server then fetches the audio from the publisher on demand and passes it through without storing it; it only connects to public addresses.</div>
</div>
</div>
</section>
<section class="panel settings-section settings-section-narrow settings-similarity-status">
<div class="panel-head">
@@ -2179,7 +2200,8 @@ function adminV2() {
download_proxies: [],
torrent_proxy_id: '',
youtube_proxy_id: '',
youtube_cookie_id: ''
youtube_cookie_id: '',
podcast_audio_proxy_enabled: false
},
settingsProbe: { status: 'idle', ok: false },
settingsProbeLoading: false,
+108
View File
@@ -48,6 +48,114 @@
</div>
</template>
<!-- Podcast: confirm publishing a feed URL that looks private -->
<template x-if="$store.podcasts.shareDialog">
<div class="modal-overlay" @click.self="$store.podcasts.cancelShare()" @keydown.escape.window="$store.podcasts.cancelShare()">
<div class="modal-box podcast-share-dialog" role="dialog" aria-modal="true" aria-labelledby="podcast-share-dialog-title">
<h3 id="podcast-share-dialog-title">{{ t.player_podcast_share_dialog_title }}</h3>
<p class="podcast-share-dialog-title" x-text="$store.podcasts.shareDialog.title"></p>
<p>{{ t.player_podcast_share_dialog_text }}</p>
<ul class="podcast-share-warnings">
<template x-for="(warning, index) in $store.podcasts.shareDialog.warnings" :key="index">
<li x-text="$store.podcasts.warningText(warning)"></li>
</template>
</ul>
<div class="podcast-share-url" x-text="$store.podcasts.shareDialog.feed_url"></div>
<div class="podcast-share-url" x-show="$store.podcasts.shareDialog.id_feed_url" x-text="$store.podcasts.shareDialog.id_feed_url"></div>
<p class="podcast-share-hint">{{ t.player_podcast_share_dialog_hint }}</p>
<div class="modal-footer">
<button class="modal-btn modal-btn-ghost" x-init="$nextTick(() => $el.focus())" @click="$store.podcasts.cancelShare()">{{ t.player_cancel }}</button>
<button class="modal-btn modal-btn-danger" @click="$store.podcasts.confirmShare()">{{ t.player_podcast_share_dialog_confirm }}</button>
</div>
</div>
</div>
</template>
<!-- Podcast found in the federation: card, preview from its feed, subscribe -->
<template x-if="$store.podcasts.found">
<div class="modal-overlay" @click.self="$store.podcasts.closeFound()" @keydown.escape.window="$store.podcasts.closeFound()">
<div class="modal-box podcast-found-dialog" role="dialog" aria-modal="true" aria-labelledby="podcast-found-title">
<div class="podcast-found-head">
<div class="podcast-found-cover">
<template x-if="$store.podcasts.found.hit.image_url || $store.podcasts.found.preview?.image_url">
<img :src="$store.podcasts.found.hit.image_url || $store.podcasts.found.preview?.image_url" alt="" aria-hidden="true" referrerpolicy="no-referrer">
</template>
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5">
<rect x="9" y="2" width="6" height="12" rx="3"/><path d="M5 10a7 7 0 0014 0M12 17v5M8 22h8"/>
</svg>
</div>
<div class="podcast-found-heading">
<div class="podcast-found-badges">
<span class="podcast-type-badge">{{ t.player_podcast_type }}</span>
<span class="podcast-subscribed-badge" x-show="$store.podcasts.found.subscribed">{{ t.player_podcast_found_subscribed }}</span>
</div>
<h3 id="podcast-found-title" x-text="$store.podcasts.found.hit.title"></h3>
<div class="podcast-found-author" x-show="$store.podcasts.found.hit.author" x-text="$store.podcasts.found.hit.author"></div>
<div class="podcast-found-meta" x-text="$store.podcasts.foundMeta($store.podcasts.found.hit)"></div>
<a class="podcast-found-link" x-show="$store.podcasts.found.hit.link"
:href="$store.podcasts.found.hit.link" target="_blank" rel="noopener noreferrer nofollow"
x-text="$store.podcasts.found.hit.link"></a>
</div>
</div>
<p class="podcast-found-description" x-show="$store.podcasts.found.hit.description" x-text="$store.podcasts.found.hit.description"></p>
<div class="podcast-found-label">{{ t.player_podcast_found_feed }}</div>
<div class="podcast-share-url" x-text="$store.podcasts.found.hit.feed_url"></div>
<p class="podcast-share-hint">{{ t.player_podcast_found_note }}</p>
<div class="podcast-found-preview">
<template x-if="$store.podcasts.found.previewLoading">
<div class="podcast-found-status">{{ t.player_podcast_found_loading }}</div>
</template>
<template x-if="$store.podcasts.found.previewError">
<div class="podcast-found-status error">
<span x-text="$store.podcasts.found.previewError"></span>
<button class="modal-btn modal-btn-ghost" @click="$store.podcasts.loadFoundPreview()">{{ t.player_podcast_found_retry }}</button>
</div>
</template>
<template x-if="$store.podcasts.found.preview">
<div>
<div class="podcast-found-status" x-show="$store.podcasts.found.preview.moved">{{ t.player_podcast_found_moved }}</div>
<div class="podcast-chips podcast-found-seasons" role="group" aria-label="{{ t.player_podcast_season }}">
<button class="podcast-chip" type="button"
:class="{ active: $store.podcasts.found.season === 'all' }"
@click="$store.podcasts.selectFoundSeason('all')">{{ t.player_podcast_all_seasons }}</button>
<template x-for="season in $store.podcasts.found.preview.seasons" :key="$store.podcasts.seasonValue(season)">
<button class="podcast-chip" type="button"
:class="{ active: $store.podcasts.found.season === $store.podcasts.seasonValue(season) }"
@click="$store.podcasts.selectFoundSeason($store.podcasts.seasonValue(season))"
x-text="`${$store.podcasts.seasonLabel(season)} · ${season.episode_count}`"></button>
</template>
</div>
<ul class="podcast-found-episodes">
<template x-for="(episode, index) in $store.podcasts.found.preview.episodes" :key="index">
<li>
<div class="podcast-found-episode-title" x-text="episode.title"></div>
<div class="podcast-found-episode-meta" x-text="$store.podcasts.episodeMeta(episode)"></div>
</li>
</template>
</ul>
<div class="podcast-found-status" x-show="$store.podcasts.foundMore($store.podcasts.found.preview)" x-text="$store.podcasts.foundMore($store.podcasts.found.preview)"></div>
</div>
</template>
</div>
<div class="podcast-found-status error" x-show="$store.podcasts.found.error" x-text="$store.podcasts.found.error"></div>
<div class="podcast-found-status" x-show="$store.podcasts.found.message" x-text="$store.podcasts.found.message"></div>
<div class="modal-footer">
<button class="modal-btn modal-btn-ghost" @click="$store.podcasts.closeFound()">{{ t.player_close }}</button>
<template x-if="$store.podcasts.found.subscribed">
<button class="modal-btn modal-btn-primary" @click="$store.podcasts.openFoundPodcast()">{{ t.player_podcast_found_open }}</button>
</template>
<template x-if="!$store.podcasts.found.subscribed">
<button class="modal-btn modal-btn-primary" :disabled="$store.podcasts.found.subscribing"
@click="$store.podcasts.subscribeFound()"
x-text="$store.podcasts.foundButtonLabel()"></button>
</template>
</div>
</div>
</div>
</template>
<!-- Create / Rename Playlist Modal -->
<template x-if="$store.playlists.modal">
<div class="modal-overlay" @click.self="$store.playlists.modal = null">
File diff suppressed because it is too large. Load diff
+341 -6
View File
@@ -75,6 +75,12 @@
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M21 15v4a2 2 0 01-2 2H5a2 2 0 01-2-2v-4"/><path d="M17 8l-5-5-5 5"/><path d="M12 3v12"/></svg>
{{ t.player_my_uploads }}
</div>
<div class="sidebar-nav-item"
:class="{ active: $store.library.view === 'podcasts' || $store.library.view === 'podcast_detail' }"
@click="$store.podcasts.go()">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><rect x="9" y="2" width="6" height="12" rx="3"/><path d="M5 10a7 7 0 0014 0M12 17v5M8 22h8"/></svg>
{{ t.player_podcasts }}
</div>
</div>
<div class="sidebar-section">
<div class="sidebar-section-title">
@@ -187,6 +193,12 @@
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M21 15v4a2 2 0 01-2 2H5a2 2 0 01-2-2v-4"/><path d="M17 8l-5-5-5 5"/><path d="M12 3v12"/></svg>
{{ t.player_my_uploads }}
</div>
<div class="sidebar-nav-item"
:class="{ active: $store.library.view === 'podcasts' || $store.library.view === 'podcast_detail' }"
@click="$store.podcasts.go(); $store.mobile.closeLibrary()">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><rect x="9" y="2" width="6" height="12" rx="3"/><path d="M5 10a7 7 0 0014 0M12 17v5M8 22h8"/></svg>
{{ t.player_podcasts }}
</div>
</div>
<div class="mobile-drawer-section">
@@ -613,6 +625,41 @@
</div>
</template>
</div>
<div class="search-podcasts-section"
x-show="!$store.library.similaritySearchLabel && ($store.library.federationSearch.podcasts || []).length > 0"
x-cloak>
<h3 class="search-subsection-title">{{ t.player_podcast_found_title }}</h3>
<div class="search-releases-row">
<template x-for="hit in ($store.library.federationSearch.podcasts || [])"
:key="hit.podcast_id">
<div class="search-release-card federation-entity-card podcast-found-card"
role="button" tabindex="0"
@click="$store.podcasts.openFound(hit)"
@keydown.enter="$store.podcasts.openFound(hit)">
<div class="search-release-cover">
<template x-if="hit.image_url">
<img class="artwork-image" :src="hit.image_url" alt="" aria-hidden="true"
loading="lazy" referrerpolicy="no-referrer"
@load="$event.currentTarget.classList.add('artwork-loaded')"
@error="$event.currentTarget.classList.remove('artwork-loaded')">
</template>
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5">
<rect x="9" y="2" width="6" height="12" rx="3"/><path d="M5 10a7 7 0 0014 0M12 17v5M8 22h8"/>
</svg>
</div>
<div class="podcast-found-badges">
<span class="podcast-type-badge">{{ t.player_podcast_type }}</span>
<span class="podcast-subscribed-badge" x-show="hit.subscribed">{{ t.player_podcast_found_subscribed }}</span>
</div>
<div class="card-title" x-text="hit.title"></div>
<div class="card-subtitle" x-text="$store.podcasts.foundSubtitle(hit)"></div>
</div>
</template>
</div>
</div>
<div class="federation-search-status podcast-search-failed"
x-show="!$store.library.similaritySearchLabel && $store.library.federationSearch.podcastDiscovery === 'failed'"
x-cloak>{{ t.player_podcast_found_failed }}</div>
<template x-for="(track, idx) in $store.library.federationSearch.tracks"
:key="track.key.content_id">
<div class="track-row federation-track-row"
@@ -703,6 +750,286 @@
</div>
</template>
<!-- Podcasts -->
<template x-if="$store.library.view === 'podcasts'">
<div class="podcasts-view">
<h1 class="section-title">{{ t.player_podcasts }}</h1>
<form class="podcast-add" @submit.prevent="$store.podcasts.add()">
<input class="podcast-input podcast-add-input"
type="text"
inputmode="url"
autocomplete="off"
spellcheck="false"
placeholder="{{ t.player_podcast_add_placeholder }}"
aria-label="{{ t.player_podcast_add_placeholder }}"
x-model="$store.podcasts.addUrl"
:disabled="$store.podcasts.adding">
<button class="release-action-btn primary" type="submit"
:disabled="$store.podcasts.adding || !$store.podcasts.addUrl.trim()">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><line x1="12" y1="5" x2="12" y2="19"/><line x1="5" y1="12" x2="19" y2="12"/></svg>
<span x-text="$store.podcasts.adding ? '{{ t.player_podcast_adding }}' : '{{ t.player_podcast_add }}'"></span>
</button>
</form>
<div class="podcast-note">{{ t.player_podcast_private_note }}</div>
<div class="podcast-message error" x-show="$store.podcasts.addError" x-cloak x-text="$store.podcasts.addError"></div>
<div class="podcast-message" x-show="$store.podcasts.addMessage" x-cloak x-text="$store.podcasts.addMessage"></div>
<label class="podcast-toggle">
<input type="checkbox" :checked="$store.podcasts.showAll" @change="$store.podcasts.toggleShowAll()">
<span>{{ t.player_podcast_show_unsubscribed }}</span>
</label>
<div class="podcast-message error" x-show="$store.podcasts.listError" x-cloak x-text="$store.podcasts.listError"></div>
<div class="card-grid podcast-grid">
<template x-for="podcast in $store.podcasts.list" :key="podcast.podcast_id">
<div class="card podcast-card" @click="$store.podcasts.open(podcast.podcast_id)">
<div class="card-img">
<template x-if="podcast.image_url">
<img :src="podcast.image_url" alt="" aria-hidden="true" loading="lazy" referrerpolicy="no-referrer" @error="podcast.image_url = null">
</template>
<template x-if="!podcast.image_url">
<span class="placeholder-icon"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><rect x="9" y="2" width="6" height="12" rx="3"/><path d="M5 10a7 7 0 0014 0M12 17v5M8 22h8"/></svg></span>
</template>
<span class="podcast-unplayed-badge"
x-show="podcast.unplayed_count > 0"
x-text="podcast.unplayed_count"
:title="podcast.unplayed_count + ' ' + T.podcastUnplayed"></span>
</div>
<div class="card-title" x-text="podcast.title"></div>
<div class="card-subtitle" x-text="podcast.author || ''"></div>
<div class="card-subtitle" x-text="$store.podcasts.podcastCounts(podcast)"></div>
<div class="card-subtitle podcast-unsubscribed-label" x-show="!podcast.subscribed" x-cloak>{{ t.player_podcast_unsubscribed }}</div>
<div class="card-subtitle podcast-shared-label" x-show="podcast.subscribed && podcast.shared" x-cloak>{{ t.player_podcast_shared_badge }}</div>
<div class="card-subtitle podcast-fetch-error"
x-show="podcast.fetch && podcast.fetch.last_error"
x-cloak
:title="podcast.fetch && podcast.fetch.last_error"
x-text="'{{ t.player_podcast_fetch_error }}'"></div>
</div>
</template>
</div>
<template x-if="!$store.podcasts.listLoading && $store.podcasts.list.length === 0 && !$store.podcasts.listError">
<div class="empty-state"><p>{{ t.player_podcasts_empty }}</p></div>
</template>
<template x-if="$store.podcasts.listLoading && $store.podcasts.list.length === 0">
<div class="loading-spinner"><div class="spinner"></div></div>
</template>
</div>
</template>
<!-- Podcast: seasons and episodes -->
<template x-if="$store.library.view === 'podcast_detail'">
<div class="podcast-detail">
<div class="breadcrumb">
<a @click="$store.podcasts.go()">{{ t.player_podcasts }}</a>
<span>/</span>
<span x-text="$store.podcasts.current ? $store.podcasts.current.title : ''"></span>
</div>
<div class="podcast-message error" x-show="$store.podcasts.detailError" x-cloak x-text="$store.podcasts.detailError"></div>
<template x-if="!$store.podcasts.current && !$store.podcasts.detailError">
<div class="loading-spinner"><div class="spinner"></div></div>
</template>
<template x-if="$store.podcasts.current">
<div class="artist-header podcast-header">
<div class="artist-img podcast-cover">
<template x-if="$store.podcasts.current.image_url">
<img :src="$store.podcasts.current.image_url" alt="" aria-hidden="true" referrerpolicy="no-referrer" @error="$store.podcasts.current.image_url = null">
</template>
<template x-if="!$store.podcasts.current.image_url">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><rect x="9" y="2" width="6" height="12" rx="3"/><path d="M5 10a7 7 0 0014 0M12 17v5M8 22h8"/></svg>
</template>
</div>
<div class="podcast-header-info">
<div class="artist-name" x-text="$store.podcasts.current.title"></div>
<div class="artist-stats">
<span x-show="$store.podcasts.current.author" x-text="$store.podcasts.current.author"></span>
<span x-show="$store.podcasts.current.author">•</span>
<span x-text="$store.podcasts.podcastCounts($store.podcasts.current)"></span>
<span class="podcast-unsubscribed-label" x-show="!$store.podcasts.current.subscribed" x-cloak>• {{ t.player_podcast_unsubscribed }}</span>
</div>
<div class="podcast-fetch-line"
:class="{ error: $store.podcasts.current.fetch && $store.podcasts.current.fetch.last_error }"
x-text="$store.podcasts.fetchStatus($store.podcasts.current)"></div>
<div class="podcast-feed-url" :title="$store.podcasts.current.feed_url">
<span>{{ t.player_podcast_feed }}:</span>
<span x-text="$store.podcasts.current.feed_url"></span>
</div>
<div class="release-actions">
<button class="release-action-btn secondary"
x-show="$store.podcasts.current.subscribed"
:disabled="$store.podcasts.refreshing"
@click="$store.podcasts.refresh()">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M21 12a9 9 0 11-3-6.7L21 8"/><path d="M21 3v5h-5"/></svg>
<span x-text="$store.podcasts.refreshing ? '{{ t.player_podcast_refreshing }}' : '{{ t.player_podcast_refresh }}'"></span>
</button>
<button class="release-action-btn secondary podcast-share-toggle"
x-show="$store.podcasts.current.subscribed && $store.podcasts.share"
x-cloak
:class="{ active: $store.podcasts.share && $store.podcasts.share.shared }"
:aria-pressed="$store.podcasts.share && $store.podcasts.share.shared ? 'true' : 'false'"
:disabled="$store.podcasts.sharing || !$store.podcasts.share || (!$store.podcasts.share.shared && !$store.podcasts.share.can_share)"
:title="$store.podcasts.share && $store.podcasts.share.card_error ? '{{ t.player_podcast_share_unavailable }}' : ''"
@click="$store.podcasts.toggleShare()">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><circle cx="18" cy="5" r="3"/><circle cx="6" cy="12" r="3"/><circle cx="18" cy="19" r="3"/><line x1="8.6" y1="13.5" x2="15.4" y2="17.5"/><line x1="15.4" y1="6.5" x2="8.6" y2="10.5"/></svg>
<span x-text="$store.podcasts.sharing ? '{{ t.player_podcast_sharing }}' : ($store.podcasts.share && $store.podcasts.share.shared ? '{{ t.player_podcast_unshare }}' : '{{ t.player_podcast_share }}')"></span>
</button>
<button class="release-action-btn secondary podcast-unsubscribe"
x-show="$store.podcasts.current.subscribed"
@click="$store.podcasts.unsubscribe()">
<span>{{ t.player_podcast_unsubscribe }}</span>
</button>
<button class="release-action-btn primary"
x-show="!$store.podcasts.current.subscribed"
x-cloak
@click="$store.podcasts.subscribe()">
<span>{{ t.player_podcast_subscribe }}</span>
</button>
</div>
<div class="podcast-share-status"
x-show="$store.podcasts.current.subscribed && $store.podcasts.share"
x-cloak
:class="{ pending: $store.podcasts.share && $store.podcasts.share.pending_confirmation }">
<span x-text="$store.podcasts.shareStatus()"></span>
<button class="podcast-share-review"
type="button"
x-show="$store.podcasts.share && $store.podcasts.share.pending_confirmation"
:disabled="$store.podcasts.sharing"
@click="$store.podcasts.reviewShare()">{{ t.player_podcast_share_review }}</button>
</div>
<div class="podcast-message error" x-show="$store.podcasts.shareError" x-cloak x-text="$store.podcasts.shareError"></div>
<div class="podcast-message" x-show="$store.podcasts.refreshMessage" x-cloak x-text="$store.podcasts.refreshMessage"></div>
<div class="podcast-message error" x-show="$store.podcasts.actionError" x-cloak x-text="$store.podcasts.actionError"></div>
</div>
</div>
</template>
<template x-if="$store.podcasts.current && $store.podcasts.current.description">
<p class="podcast-description" x-text="$store.podcasts.current.description"></p>
</template>
<div class="podcast-filters">
<div class="podcast-chips" role="group" aria-label="{{ t.player_podcast_season }}">
<button class="podcast-chip"
:class="{ active: $store.podcasts.season === 'all' }"
@click="$store.podcasts.setSeason('all')">{{ t.player_podcast_all_seasons }}</button>
<template x-for="season in $store.podcasts.seasons" :key="$store.podcasts.seasonValue(season)">
<button class="podcast-chip"
:class="{ active: $store.podcasts.season === $store.podcasts.seasonValue(season) }"
@click="$store.podcasts.setSeason($store.podcasts.seasonValue(season))">
<span x-text="$store.podcasts.seasonLabel(season)"></span>
<span class="podcast-chip-count" x-show="season.unplayed_count > 0" x-text="season.unplayed_count"></span>
</button>
</template>
</div>
<div class="podcast-filter-row">
<input class="podcast-input podcast-search"
type="search"
placeholder="{{ t.player_podcast_search }}"
aria-label="{{ t.player_podcast_search }}"
x-model="$store.podcasts.query"
@input="$store.podcasts.search()">
<select class="podcast-input podcast-select"
aria-label="{{ t.player_podcast_filter_all }}"
:value="$store.podcasts.played"
@change="$store.podcasts.setPlayed($event.target.value)">
<option value="all">{{ t.player_podcast_filter_all }}</option>
<option value="unplayed">{{ t.player_podcast_filter_unplayed }}</option>
<option value="in_progress">{{ t.player_podcast_filter_in_progress }}</option>
<option value="played">{{ t.player_podcast_filter_played }}</option>
</select>
<select class="podcast-input podcast-select"
aria-label="{{ t.player_podcast_sort_newest }}"
:value="$store.podcasts.sort"
@change="$store.podcasts.setSort($event.target.value)">
<option value="newest">{{ t.player_podcast_sort_newest }}</option>
<option value="oldest">{{ t.player_podcast_sort_oldest }}</option>
<option value="number">{{ t.player_podcast_sort_number }}</option>
</select>
</div>
</div>
<div class="podcast-message error" x-show="$store.podcasts.episodesError" x-cloak x-text="$store.podcasts.episodesError"></div>
<div class="podcast-episodes">
<template x-for="episode in $store.podcasts.episodes" :key="episode.episode_id">
<div class="podcast-episode"
:class="{ current: $store.podcasts.isCurrentEpisode(episode), played: episode.status === 'completed' }">
<div class="podcast-episode-main">
<button class="podcast-episode-play"
:disabled="$store.podcasts.jamBlocked()"
:title="$store.podcasts.jamBlocked() ? T.podcastJamBlocked : (episode.status === 'in_progress' ? '{{ t.player_podcast_resume }}' : '{{ t.player_podcast_play }}')"
:aria-label="episode.status === 'in_progress' ? '{{ t.player_podcast_resume }}' : '{{ t.player_podcast_play }}'"
@click="$store.podcasts.isPlayingEpisode(episode) ? $store.player.pause() : $store.podcasts.play(episode)">
<template x-if="!$store.podcasts.isPlayingEpisode(episode)">
<svg viewBox="0 0 24 24" fill="currentColor"><path d="M8 5v14l11-7z"/></svg>
</template>
<template x-if="$store.podcasts.isPlayingEpisode(episode)">
<svg viewBox="0 0 24 24" fill="currentColor"><path d="M6 4h4v16H6zM14 4h4v16h-4z"/></svg>
</template>
</button>
<div class="podcast-episode-info">
<div class="podcast-episode-title">
<span x-text="episode.title"></span>
<span class="podcast-played-badge" x-show="episode.status === 'completed'" x-cloak>✓ {{ t.player_podcast_played_badge }}</span>
</div>
<div class="podcast-episode-meta">
<span x-text="$store.podcasts.episodeMeta(episode)"></span>
<span x-show="episode.completed_count > 0" x-cloak x-text="' · {{ t.player_podcast_played_times }}: ' + episode.completed_count"></span>
<span class="podcast-not-in-feed" x-show="!episode.in_feed" x-cloak>· {{ t.player_podcast_not_in_feed }}</span>
</div>
<div class="podcast-progress" x-show="episode.status === 'in_progress'" x-cloak>
<div class="podcast-progress-bar"><div :style="`width:${$store.podcasts.progressPercent(episode)}%`"></div></div>
<span x-text="$store.podcasts.progressLabel(episode)"></span>
</div>
</div>
</div>
<div class="podcast-episode-actions">
<button class="podcast-episode-action"
x-show="episode.status !== 'not_started'"
:disabled="$store.podcasts.jamBlocked() || !!$store.podcasts.busy[episode.episode_id]"
@click="$store.podcasts.restart(episode)">{{ t.player_podcast_restart }}</button>
<button class="podcast-episode-action"
x-show="episode.status !== 'completed'"
:disabled="!!$store.podcasts.busy[episode.episode_id]"
@click="$store.podcasts.markPlayed(episode)">{{ t.player_podcast_mark_played }}</button>
<button class="podcast-episode-action"
x-show="episode.status !== 'not_started'"
:disabled="!!$store.podcasts.busy[episode.episode_id]"
@click="$store.podcasts.markUnplayed(episode)">{{ t.player_podcast_mark_unplayed }}</button>
<button class="podcast-episode-action"
:disabled="$store.podcasts.jamBlocked()"
@click="$store.podcasts.addToQueue(episode)">{{ t.player_podcast_add_queue }}</button>
</div>
<template x-if="episode.description">
<p class="podcast-episode-description"
:class="{ expanded: $store.podcasts.expanded[episode.episode_id] }"
@click="$store.podcasts.toggleDescription(episode)"
x-text="episode.description"></p>
</template>
</div>
</template>
</div>
<template x-if="!$store.podcasts.episodesLoading && $store.podcasts.episodes.length === 0 && $store.podcasts.current">
<div class="empty-state"><p>{{ t.player_podcast_no_episodes }}</p></div>
</template>
<template x-if="$store.podcasts.episodesLoading">
<div class="loading-spinner"><div class="spinner"></div></div>
</template>
<button class="release-action-btn secondary podcast-load-more"
x-show="$store.podcasts.hasMore && !$store.podcasts.episodesLoading"
x-cloak
@click="$store.podcasts.loadEpisodes(false)">{{ t.player_podcast_load_more }}</button>
<template x-if="$store.podcasts.history.length > 0">
<section class="podcast-history">
<h2 class="artist-release-group-title">{{ t.player_podcast_history }}</h2>
<template x-for="entry in $store.podcasts.history" :key="entry.episode_id + '-' + entry.ended_at_ms">
<div class="podcast-history-row">
<span class="podcast-history-title" x-text="entry.episode_title || entry.episode_id"></span>
<span class="podcast-history-meta" x-text="$store.podcasts.formatDateTime(entry.ended_at_ms) + ' · ' + ($store.podcasts.formatDuration(entry.listened_ms) || '0:00') + (entry.completed ? ' · ✓' : '')"></span>
</div>
</template>
</section>
</template>
</div>
</template>
<!-- Artists Grid -->
<template x-if="$store.library.view === 'artists' || $store.library.view === 'my_uploads'">
<div>
@@ -1522,7 +1849,7 @@
<div class="queue-track"
:data-queue-index="item.index"
:class="{ active: item.state === 'current', current: item.state === 'current', played: item.state === 'played', dragging: $store.queue._dragIdx === item.index, synthetic: item.synthetic, 'foreign-jam-track': $store.queue.isForeignJamTrack(item.track), unplayable: $store.player._isUnplayableQueueItem(item.track) }"
:title="$store.player._unplayableQueueLabel(item.track) || null"
:title="$store.player._queueItemNote(item.track) || null"
:style="$store.queue.isForeignJamTrack(item.track) ? $store.queue.contributorStyle(item.track) : ''"
@click="item.index >= 0 ? $store.queue.playFromIndex(item.index) : $store.player.play(item.track)"
:draggable="!item.synthetic"
@@ -1576,15 +1903,15 @@
<div class="queue-track-info">
<div class="queue-track-title" x-text="item.track.title"></div>
<div class="queue-track-unplayable"
x-show="$store.player._isUnplayableQueueItem(item.track)"
x-show="$store.player._queueItemNote(item.track)"
x-cloak
x-text="$store.player._unplayableQueueLabel(item.track)"></div>
x-text="$store.player._queueItemNote(item.track)"></div>
<div class="queue-track-artist">
<template x-for="(artist, artistIdx) in $store.library.trackArtistLinks(item.track)" :key="artist.label + '-' + artist.id + '-' + artistIdx">
<span>
<template x-if="artistIdx > 0"><span>, </span></template>
<a class="artist-link"
@click.stop="artist.id ? $store.library.openArtist(artist.id) : $store.library.openFederatedArtist(artist.label.replace(/^ft\\.\\s*/, ''))"
@click.stop="$store.library.openArtistLink(artist)"
x-text="artist.label"></a>
</span>
</template>
@@ -1635,6 +1962,7 @@
<div class="player-track-title-row">
<div class="player-track-title" x-text="$store.player.currentTrack.title"></div>
<button data-member-only class="like-btn player-current-like"
x-show="!$store.player._isPodcastEpisode($store.player.currentTrack)"
:class="{ liked: $store.likes.has($store.player.currentTrack.id) }"
@click.stop="$store.likes.toggle($store.player.currentTrack.id)"
title="{{ t.player_like }}"
@@ -1642,6 +1970,7 @@
<svg viewBox="0 0 24 24" :fill="$store.likes.has($store.player.currentTrack.id) ? 'currentColor' : 'none'" stroke="currentColor" stroke-width="2"><path d="M20.84 4.61a5.5 5.5 0 00-7.78 0L12 5.67l-1.06-1.06a5.5 5.5 0 00-7.78 7.78L12 21.23l8.84-8.84a5.5 5.5 0 000-7.78z"/></svg>
</button>
<button data-member-only class="track-action-btn player-current-share"
x-show="!$store.player._isPodcastEpisode($store.player.currentTrack)"
@click.stop="$store.sharing.copyTrack($store.player.currentTrack, $event.currentTarget)"
title="{{ t.player_share_track }}"
aria-label="{{ t.player_share_track }}">
@@ -1652,16 +1981,22 @@
<template x-for="(artist, artistIdx) in $store.library.trackArtistLinks($store.player.currentTrack)" :key="artist.label + '-' + artist.id + '-' + artistIdx">
<span>
<template x-if="artistIdx > 0"><span>, </span></template>
<a class="artist-link" @click.stop="$store.library.openArtist(artist.id)" x-text="artist.label"></a>
<a class="artist-link" @click.stop="$store.library.openArtistLink(artist)" x-text="artist.label"></a>
</span>
</template>
<template x-if="$store.player.currentTrack.release_year">
<span class="player-release-year" x-text="' · ' + $store.player.currentTrack.release_year"></span>
</template>
</div>
<div class="player-track-release" x-show="$store.player.currentTrack.release_title">
<div class="player-track-release" x-show="$store.player.currentTrack.release_title && !$store.player._isPodcastEpisode($store.player.currentTrack)">
<a class="artist-link" @click.stop="$store.player.currentTrack.release_id && $store.library.openRelease($store.player.currentTrack.release_id)" x-text="$store.player.currentTrack.release_title"></a>
</div>
<div class="player-episode-status"
x-show="$store.player._isPodcastEpisode($store.player.currentTrack) && ($store.player.episodeNotice || $store.player.episodeLoading)"
x-cloak
:class="{ error: !!$store.player.episodeNotice }"
:title="$store.player.episodeNotice || null"
x-text="$store.player.episodeNotice || '{{ t.player_podcast_loading }}'"></div>
</div>
</div>
</template>
+396
View File
@@ -7033,4 +7033,400 @@ button.user-stat:hover {
.guest-player .playlist-track-row .playlist-track-info { grid-column:2; grid-row:1; }
.guest-player .playlist-track-row .track-actions { grid-column:3; grid-row:1; }
}
/* Podcasts */
.podcast-add {
display: flex;
gap: 8px;
align-items: center;
flex-wrap: wrap;
margin-bottom: 8px;
}
.podcast-input {
background: var(--bg-elevated);
border: 1px solid var(--border-color);
border-radius: 8px;
color: var(--text-primary);
font-size: 13px;
padding: 8px 12px;
min-width: 0;
}
.podcast-input:focus { outline: none; border-color: var(--accent); }
.podcast-add-input { flex: 1 1 320px; }
.podcast-note,
.podcast-message,
.podcast-fetch-line,
.podcast-feed-url {
font-size: 12px;
color: var(--text-secondary);
margin: 4px 0;
}
.podcast-message.error,
.podcast-fetch-line.error,
.podcast-fetch-error,
.player-episode-status.error { color: #f15e6c; }
.podcast-feed-url {
color: var(--text-subdued);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
max-width: 100%;
}
.podcast-feed-url span + span { margin-left: 4px; }
.podcast-toggle {
display: inline-flex;
gap: 6px;
align-items: center;
font-size: 12px;
color: var(--text-secondary);
margin: 8px 0 16px;
cursor: pointer;
}
.podcast-card .card-img { position: relative; }
.podcast-unplayed-badge {
position: absolute;
top: 8px;
right: 8px;
min-width: 24px;
padding: 2px 8px;
border-radius: 12px;
background: var(--accent);
color: #000;
font-size: 12px;
font-weight: 700;
text-align: center;
}
.podcast-unsubscribed-label { color: var(--text-subdued); }
.podcast-shared-label { color: var(--accent); }
.podcast-share-toggle.active {
border-color: var(--accent);
color: var(--accent);
}
.podcast-share-status {
font-size: 12px;
color: var(--text-secondary);
margin: 6px 0 4px;
max-width: 720px;
line-height: 1.45;
}
.podcast-share-status.pending { color: #f0b84d; }
.podcast-share-review {
margin-left: 8px;
background: transparent;
border: 1px solid currentColor;
border-radius: 12px;
color: inherit;
cursor: pointer;
font: inherit;
padding: 1px 8px;
}
.podcast-share-dialog { max-width: 520px; }
.search-podcasts-section { margin: 4px 0 18px; }
.search-subsection-title {
color: var(--text-secondary);
font-size: 13px;
font-weight: 600;
letter-spacing: 0.02em;
margin: 0 0 8px;
text-transform: uppercase;
}
.podcast-found-badges {
display: flex;
flex-wrap: wrap;
gap: 4px;
margin-bottom: 4px;
}
.podcast-type-badge,
.podcast-subscribed-badge {
border-radius: 8px;
font-size: 10px;
font-weight: 600;
line-height: 1.6;
padding: 0 6px;
}
.podcast-type-badge { background: var(--bg-active); color: var(--text-secondary); }
.podcast-subscribed-badge { background: var(--accent); color: #000; }
.podcast-found-dialog {
max-width: 600px;
max-height: calc(100vh - 48px);
overflow-y: auto;
}
.podcast-found-head {
display: flex;
gap: 16px;
align-items: flex-start;
}
.podcast-found-cover {
position: relative;
flex: 0 0 96px;
width: 96px;
height: 96px;
border-radius: 8px;
background: var(--bg-elevated);
overflow: hidden;
display: flex;
align-items: center;
justify-content: center;
}
.podcast-found-cover img { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: cover; }
.podcast-found-cover svg { width: 36px; height: 36px; color: var(--text-subdued); }
.podcast-found-heading { min-width: 0; }
.podcast-found-heading h3 { margin: 2px 0 4px; overflow-wrap: anywhere; }
.podcast-found-author,
.podcast-found-meta {
color: var(--text-secondary);
font-size: 13px;
overflow-wrap: anywhere;
}
.podcast-found-link {
color: var(--accent);
display: block;
font-size: 12px;
margin-top: 4px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.podcast-found-description {
color: var(--text-secondary);
font-size: 13px;
line-height: 1.5;
margin: 12px 0;
white-space: pre-line;
overflow-wrap: anywhere;
}
.podcast-found-label {
color: var(--text-subdued);
font-size: 12px;
margin-top: 8px;
}
.podcast-found-dialog .podcast-share-hint {
color: var(--text-subdued);
font-size: 12px;
line-height: 1.5;
}
.podcast-found-preview { margin: 8px 0; }
.podcast-found-seasons { margin: 8px 0; }
.podcast-found-status {
color: var(--text-secondary);
font-size: 13px;
margin: 6px 0;
display: flex;
gap: 8px;
align-items: center;
}
.podcast-found-status.error { color: #ff6b6b; }
.podcast-found-episodes {
list-style: none;
margin: 0;
padding: 0;
max-height: 240px;
overflow-y: auto;
}
.podcast-found-episodes li {
border-bottom: 1px solid var(--border-color);
padding: 6px 2px;
}
.podcast-found-episode-title {
font-size: 13px;
overflow-wrap: anywhere;
}
.podcast-found-episode-meta {
color: var(--text-subdued);
font-size: 12px;
}
@media (max-width: 600px) {
.podcast-found-head { flex-direction: column; }
}
.podcast-share-dialog p {
color: var(--text-secondary);
font-size: 13px;
line-height: 1.5;
margin: 8px 0;
}
.podcast-share-dialog .podcast-share-dialog-title {
color: var(--text-primary);
font-weight: 600;
}
.podcast-share-warnings {
color: #f0b84d;
font-size: 13px;
margin: 4px 0 8px 18px;
}
.podcast-share-url {
background: var(--bg-elevated);
border-radius: 6px;
color: var(--text-primary);
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12px;
margin: 4px 0;
padding: 6px 8px;
word-break: break-all;
}
.podcast-share-dialog .podcast-share-hint { color: var(--text-subdued); }
.podcast-header-info { min-width: 0; }
.podcast-description {
color: var(--text-secondary);
font-size: 13px;
line-height: 1.5;
margin: 8px 0 16px;
max-width: 900px;
white-space: pre-line;
}
.podcast-filters {
display: flex;
flex-direction: column;
gap: 10px;
margin: 12px 0 16px;
}
.podcast-chips,
.podcast-filter-row {
display: flex;
flex-wrap: wrap;
gap: 8px;
align-items: center;
}
.podcast-search { flex: 1 1 220px; }
.podcast-chip {
background: var(--bg-active);
color: var(--text-primary);
border: none;
border-radius: 16px;
padding: 6px 12px;
font-size: 12px;
cursor: pointer;
display: inline-flex;
gap: 6px;
align-items: center;
}
.podcast-chip.active { background: var(--accent); color: #000; }
.podcast-chip-count {
font-size: 11px;
opacity: 0.75;
}
.podcast-episodes { display: flex; flex-direction: column; }
.podcast-episode {
border-bottom: 1px solid var(--border-color);
padding: 12px 4px;
}
.podcast-episode.current .podcast-episode-title { color: var(--accent); }
.podcast-episode.played .podcast-episode-title > span:first-child { color: var(--text-secondary); }
.podcast-episode-main {
display: flex;
gap: 12px;
align-items: flex-start;
}
.podcast-episode-play {
flex: 0 0 auto;
width: 36px;
height: 36px;
border-radius: 50%;
border: none;
background: var(--accent);
color: #000;
display: flex;
align-items: center;
justify-content: center;
cursor: pointer;
}
.podcast-episode-play svg { width: 16px; height: 16px; }
.podcast-episode-play:disabled { opacity: 0.4; cursor: default; }
.podcast-episode-info { min-width: 0; flex: 1; }
.podcast-episode-title {
font-size: 14px;
font-weight: 600;
display: flex;
gap: 8px;
flex-wrap: wrap;
align-items: baseline;
}
.podcast-played-badge {
font-size: 11px;
font-weight: 600;
color: var(--accent);
}
.podcast-episode-meta {
font-size: 12px;
color: var(--text-secondary);
margin-top: 2px;
}
.podcast-not-in-feed { color: var(--text-subdued); }
.podcast-progress {
display: flex;
gap: 8px;
align-items: center;
font-size: 11px;
color: var(--text-secondary);
margin-top: 6px;
}
.podcast-progress-bar {
width: 120px;
max-width: 40%;
height: 4px;
border-radius: 2px;
background: var(--bg-active);
overflow: hidden;
}
.podcast-progress-bar > div { height: 100%; background: var(--accent); }
.podcast-episode-actions {
display: flex;
flex-wrap: wrap;
gap: 6px;
margin: 8px 0 0 48px;
}
.podcast-episode-action {
background: transparent;
color: var(--text-secondary);
border: 1px solid var(--border-color);
border-radius: 14px;
padding: 4px 10px;
font-size: 12px;
cursor: pointer;
}
.podcast-episode-action:hover:not(:disabled) { color: var(--text-primary); border-color: var(--text-subdued); }
.podcast-episode-action:disabled { opacity: 0.45; cursor: default; }
.podcast-episode-description {
margin: 8px 0 0 48px;
font-size: 12px;
line-height: 1.5;
color: var(--text-subdued);
white-space: pre-line;
display: -webkit-box;
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
overflow: hidden;
cursor: pointer;
}
.podcast-episode-description.expanded { display: block; }
.podcast-load-more { margin: 16px auto; }
.podcast-history { margin-top: 24px; }
.podcast-history-row {
display: flex;
justify-content: space-between;
gap: 12px;
font-size: 12px;
padding: 6px 0;
border-bottom: 1px solid var(--border-color);
}
.podcast-history-title {
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.podcast-history-meta { color: var(--text-secondary); flex: 0 0 auto; }
.player-episode-status {
margin-top: 2px;
font-size: 11px;
color: var(--text-secondary);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
@media (max-width: 560px) {
.podcast-episode-actions,
.podcast-episode-description { margin-left: 0; }
.podcast-history-row { flex-direction: column; gap: 2px; }
.podcast-history-meta { flex: none; }
}
</style>
File diff suppressed because it is too large. Load diff
+63 -49
View File
@@ -4,9 +4,11 @@ const {test} = require('node:test');
const vm = require('node:vm');
const template = readFileSync(`${__dirname}/../templates/player/scripts.html`, 'utf8');
// Queue items received from trusted devices that this player cannot play.
// Queue items received from trusted devices. Podcast episodes play through
// the podcast API (tests/player_podcasts.test.cjs); Jam placeholders and
// unresolved federated music never play here.
const podcast = {
id: -11, title: 'Episode 42', unavailable: true, stream_url: '',
id: -11, title: 'Episode 42', unavailable: false, stream_url: '',
source: {kind: 'podcast_episode', episode_id: 'ep1:x'},
wire_track: {id: 0, title: 'Episode 42', source: {kind: 'podcast_episode', episode_id: 'ep1:x'}},
};
@@ -26,42 +28,51 @@ function player(queue) {
const audio = {paused: false, src: 'old', pause() { this.paused = true; }, removeAttribute(name) { if (name === 'src') this.src = ''; }, load() {}};
const stores = {queue: Object.assign({syncCurrentIndexToTrack() {}}, queue)};
const context = vm.createContext({
audio, T: {queuePodcastUnplayable: 'Podcast · not playable', queuePrivateItem: 'Private item · hidden'},
audio, T: {queuePodcastItem: 'Podcast episode', queuePrivateItem: 'Private item · hidden'},
Alpine: {store: name => stores[name]},
});
vm.runInContext('this.methods = {' + methods + '};', context);
return Object.assign(context.methods, {audio, currentTrack: null, currentTime: 0, volume: 0.5, shuffle: false, repeatMode: 'off'});
}
test('podcast and private items are labelled and never played locally', () => {
test('episodes play here, placeholders never; both stay private in a Jam', () => {
const p = player({tracks: [], currentIndex: 0});
assert.equal(p._isUnplayableQueueItem(podcast), true);
assert.equal(p._isUnplayableQueueItem(podcast), false);
assert.equal(p._isPodcastEpisode(podcast), true);
assert.equal(p._isUnplayableQueueItem(privateItem), true);
assert.equal(p._isUnplayableQueueItem(remoteMusic), false);
assert.equal(p._isUnplayableQueueItem(song(1)), false);
assert.equal(p._unplayableQueueLabel(podcast), 'Podcast · not playable');
// Jam rules treat episodes and placeholders as private.
assert.equal(p._isPrivateQueueItem(podcast), true);
assert.equal(p._isPrivateQueueItem(privateItem), true);
assert.equal(p._isPrivateQueueItem(song(1)), false);
assert.equal(p._unplayableQueueLabel(podcast), '');
assert.equal(p._unplayableQueueLabel(privateItem), 'Private item · hidden');
assert.equal(p._unplayableQueueLabel(song(1)), '');
assert.equal(p._queueItemNote(podcast), 'Podcast episode');
assert.equal(p._queueItemNote(privateItem), 'Private item · hidden');
assert.equal(p._queueItemNote(song(1)), '');
// A browser item that only carries the verbatim wire item is recognised too.
assert.equal(p._isUnplayableQueueItem({id: -1, wire_track: podcast.wire_track}), true);
assert.equal(p._isPodcastEpisode({id: -1, wire_track: podcast.wire_track}), true);
assert.equal(p._isUnplayableQueueItem({id: -1, wire_track: privateItem.wire_track}), true);
});
test('skipping finds the next playable item in the direction of travel', () => {
const p = player({tracks: [], currentIndex: 0});
const tracks = [song(1), podcast, privateItem, song(2)];
assert.equal(p._playableQueueIndex(tracks, 1, 1), 3);
assert.equal(p._playableQueueIndex(tracks, 2, -1), 0);
assert.equal(p._playableQueueIndex([podcast], 0, 1), -1);
const tracks = [song(1), privateItem, podcast, privateItem, song(2)];
assert.equal(p._playableQueueIndex(tracks, 1, 1), 2);
assert.equal(p._playableQueueIndex(tracks, 3, 1), 4);
assert.equal(p._playableQueueIndex(tracks, 1, -1), 0);
assert.equal(p._playableQueueIndex([privateItem], 0, 1), -1);
});
test('showing an unplayable item stops and unloads the previous audio', () => {
const p = player({tracks: [podcast], currentIndex: 0});
const p = player({tracks: [privateItem], currentIndex: 0});
p.isPlaying = true;
p._showUnplayableQueueItem(podcast);
p._showUnplayableQueueItem(privateItem);
assert.equal(p.audio.paused, true);
assert.equal(p.audio.src, '');
assert.equal(p.isPlaying, false);
assert.equal(p.currentTrack, podcast);
assert.equal(p.currentTrack, privateItem);
assert.equal(p._historyTrack, null);
});
@@ -115,6 +126,7 @@ function activePlayer(queueTracks, currentIndex, audioState = {}, options = {})
'_showUnplayableQueueItem', '_syncLocalPlaybackState',
'_queueItemSourceKind', '_isUnplayableQueueItem', '_playableQueueIndex',
'_jamParticipantCommandAllowed', '_dropJamPlaceholders',
'_isPodcastEpisode', '_isPrivateQueueItem', '_episodeRef', '_endEpisode', '_reportEpisode',
];
const audio = Object.assign({
paused: true, currentTime: 0, duration: NaN, src: '',
@@ -155,16 +167,18 @@ function activePlayer(queueTracks, currentIndex, audioState = {}, options = {})
}
const episode = Object.assign({}, podcast, {duration_seconds: 1800});
// An item this player cannot play (a Jam placeholder) with a duration.
const placeholder = Object.assign({}, privateItem, {duration_seconds: 1800});
test('handoff of a podcast episode keeps its position for the next handoff', () => {
test('handoff of an unplayable item keeps its position for the next handoff', () => {
for (const audioTime of [95, 0]) {
// <audio> still holds a music track (95 s) or the page is fresh (0 s).
const p = activePlayer([song(1)], 0, {currentTime: audioTime, src: '/api/player/stream/1'});
p._executeRemoteCommand({command: 'transfer_state', payload: {
tracks: [song(1), episode], index: 1, track: episode,
tracks: [song(1), placeholder], index: 1, track: placeholder,
position_seconds: 1200, duration_seconds: 1800, paused: false,
}});
assert.equal(p.currentTrack, episode);
assert.equal(p.currentTrack, placeholder);
assert.equal(p.currentTime, 1200);
assert.equal(p.isPlaying, false);
assert.equal(p.audio.src, '', 'the previous track is unloaded');
@@ -184,35 +198,35 @@ test('library tracks still report the audio position', () => {
test('resume on an unplayable item skips it, also with repeat one', () => {
for (const repeatMode of ['one', 'off', 'all']) {
const p = activePlayer([podcast], 0);
const p = activePlayer([placeholder], 0);
p.repeatMode = repeatMode;
p.currentTime = 30;
p.resume();
p.next();
assert.equal(p.currentTrack, podcast);
assert.equal(p.currentTrack, placeholder);
assert.equal(p.isPlaying, false);
assert.equal(p.currentTime, 30, 'the item keeps its position');
assert.deepEqual(p.audio.played, []);
}
const p = activePlayer([song(1), podcast, song(2)], 1);
const p = activePlayer([song(1), placeholder, song(2)], 1);
p.repeatMode = 'one';
p.resume();
assert.equal(p.currentTrack.id, 2);
assert.deepEqual(p.audio.played, ['/api/player/stream/2']);
// Repeat all wraps around to the first playable item.
const wrap = activePlayer([song(1), podcast], 1);
const wrap = activePlayer([song(1), placeholder], 1);
wrap.repeatMode = 'all';
wrap.resume();
assert.equal(wrap.currentTrack.id, 1);
});
test('an episode reached without its position reports it as unknown, never as 0', () => {
// Next from the last music track lands on a trailing episode.
const p = activePlayer([song(1), episode], 0, {currentTime: 200, src: '/api/player/stream/1'});
test('an unplayable item reached without its position reports it as unknown, never as 0', () => {
// Next from the last music track lands on a trailing placeholder.
const p = activePlayer([song(1), placeholder], 0, {currentTime: 200, src: '/api/player/stream/1'});
p.isPlaying = true;
p.currentTime = 200;
p.next();
assert.equal(p.currentTrack, episode);
assert.equal(p.currentTrack, placeholder);
assert.equal(p.isPlaying, false);
let reported = p._devicePlaybackStatePayload();
assert.equal(reported.index, 1);
@@ -230,29 +244,29 @@ test('an episode reached without its position reports it as unknown, never as 0'
assert.equal(p.currentTrack.id, 1);
assert.equal(p._devicePlaybackStatePayload().position_unknown, undefined);
// Another browser or a Jam guest plays the episode row (position 0 sent).
const remote = activePlayer([song(1), episode], 0, {currentTime: 30, src: '/api/player/stream/1'});
// Another browser or a Jam guest plays the placeholder row (position 0 sent).
const remote = activePlayer([song(1), placeholder], 0, {currentTime: 30, src: '/api/player/stream/1'});
remote._executeRemoteCommand({command: 'play_from_index', payload: {
tracks: [song(1), episode], index: 1, track: episode, position_seconds: 0, paused: false,
tracks: [song(1), placeholder], index: 1, track: placeholder, position_seconds: 0, paused: false,
}});
assert.equal(remote.currentTrack, episode);
assert.equal(remote.currentTrack, placeholder);
assert.equal(remote._devicePlaybackStatePayload().position_unknown, true);
// A transfer passes on what the previous device did not know.
const relay = activePlayer([song(1)], 0);
relay._executeRemoteCommand({command: 'transfer_state', payload: {
tracks: [song(1), episode], index: 1, track: episode, position_seconds: 0,
tracks: [song(1), placeholder], index: 1, track: placeholder, position_seconds: 0,
position_unknown: true, paused: true,
}});
assert.equal(relay._devicePlaybackStatePayload().position_unknown, true);
// A mirrored state with a known position keeps it.
const mirror = activePlayer([song(1), episode], 0);
mirror._syncLocalPlaybackState({tracks: [song(1), episode], index: 1, track: episode, position_seconds: 900, paused: true});
const mirror = activePlayer([song(1), placeholder], 0);
mirror._syncLocalPlaybackState({tracks: [song(1), placeholder], index: 1, track: placeholder, position_seconds: 900, paused: true});
assert.equal(mirror._devicePlaybackStatePayload().position_seconds, 900);
assert.equal(mirror._devicePlaybackStatePayload().position_unknown, undefined);
});
test('repeat all and shuffle continue past a trailing unplayable item', () => {
const wrap = activePlayer([song(1), episode], 0, {currentTime: 200, src: '/api/player/stream/1'});
const wrap = activePlayer([song(1), placeholder], 0, {currentTime: 200, src: '/api/player/stream/1'});
wrap.repeatMode = 'all';
wrap.isPlaying = true;
wrap.next();
@@ -260,49 +274,49 @@ test('repeat all and shuffle continue past a trailing unplayable item', () => {
assert.equal(wrap.queue.currentIndex, 0);
assert.deepEqual(wrap.audio.played, ['/api/player/stream/1']);
// Shuffle picks the episode (index 1 of 3) and continues with music
// Shuffle picks the placeholder (index 1 of 3) and continues with music
// (searching towards the pick from the current item).
const shuffled = activePlayer([song(1), episode, song(2)], 2, {src: '/api/player/stream/2'}, {random: 0.5});
const shuffled = activePlayer([song(1), placeholder, song(2)], 2, {src: '/api/player/stream/2'}, {random: 0.5});
shuffled.shuffle = true;
shuffled.next();
assert.equal(shuffled.currentTrack.id, 1);
assert.equal(shuffled.isPlaying, false, 'the stub audio does not fire play events');
assert.deepEqual(shuffled.audio.played, ['/api/player/stream/1']);
// A trailing episode picked by shuffle wraps to the first playable item.
const trailing = activePlayer([song(1), song(2), episode], 1, {src: '/api/player/stream/2'}, {random: 0.9});
// A trailing placeholder picked by shuffle wraps to the first playable item.
const trailing = activePlayer([song(1), song(2), placeholder], 1, {src: '/api/player/stream/2'}, {random: 0.9});
trailing.shuffle = true;
trailing.next();
assert.equal(trailing.currentTrack.id, 1);
// Without repeat the queue ends on the episode, as before.
const stop = activePlayer([song(1), episode], 0, {src: '/api/player/stream/1'});
// Without repeat the queue ends on the placeholder, as before.
const stop = activePlayer([song(1), placeholder], 0, {src: '/api/player/stream/1'});
stop.next();
assert.equal(stop.currentTrack, episode);
assert.equal(stop.currentTrack, placeholder);
assert.equal(stop.isPlaying, false);
});
test('prev and seek work on a handed-off episode', () => {
test('prev and seek work on a handed-off unplayable item', () => {
const p = activePlayer([song(1), song(3)], 0, {currentTime: 95, src: '/api/player/stream/1'});
p._executeRemoteCommand({command: 'transfer_state', payload: {
tracks: [song(1), episode], index: 1, track: episode,
tracks: [song(1), placeholder], index: 1, track: placeholder,
position_seconds: 1200, duration_seconds: 1800, paused: false,
}});
// Seeking moves the episode's own position (nothing is loaded in <audio>).
// Seeking moves the placeholder's own position (nothing is loaded in <audio>).
p.seek(1500);
assert.equal(p.currentTime, 1500);
assert.equal(p._devicePlaybackStatePayload().position_seconds, 1500);
// Prev goes to the previous item instead of restarting the episode.
// Prev goes to the previous item instead of restarting the placeholder.
p.prev();
assert.equal(p.currentTrack.id, 1);
assert.deepEqual(p.audio.played, ['/api/player/stream/1']);
// At the start of the queue Prev leaves the episode and its position alone.
const first = activePlayer([episode, song(1)], 0);
// At the start of the queue Prev leaves the placeholder and its position alone.
const first = activePlayer([placeholder, song(1)], 0);
first._executeRemoteCommand({command: 'transfer_state', payload: {
tracks: [episode, song(1)], index: 0, track: episode, position_seconds: 1200, paused: true,
tracks: [placeholder, song(1)], index: 0, track: placeholder, position_seconds: 1200, paused: true,
}});
first.prev();
assert.equal(first.currentTrack, episode);
assert.equal(first.currentTrack, placeholder);
assert.equal(first.currentTime, 1200);
});