forked from Furumi/furumi-web
Proxy podcast audio from publishers on request
Browsers stream episode audio from the publisher by default. For
publishers they cannot play directly (CORS, mixed content), the
administrator can enable podcast_audio_proxy_enabled (admin Settings,
FURU_PODCAST_AUDIO_PROXY_ENABLED). The endpoints live in a new app at
/api/podcasts:
- GET /episodes/{episode_id}/audio streams an episode of the signed-in
account. The URL comes from the account's catalog, never from the
request. Requests go through SafeHttp with one sanitized byte Range and
no client headers; 200/206/416 pass through, only audio content types
are served, with nosniff and a sandbox CSP, and responses stop at 1 GiB
or 4 hours.
- GET /episodes/{episode_id}/source returns the publisher URL and, when
enabled, the proxy URL.
- POST /{podcast_id}/refresh refreshes one subscribed feed now.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bf9a6abc3f
commit
2c0c3d21aa
12 files changed
+1164
-5
No files matched your search
@@ -545,6 +545,8 @@ struct AdminSettingsValues {
|
||||
youtube_proxy_id: String,
|
||||
#[serde(default)]
|
||||
youtube_cookie_id: String,
|
||||
#[serde(default)]
|
||||
podcast_audio_proxy_enabled: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -615,6 +617,7 @@ struct AdminSettingsSources {
|
||||
torrent_proxy_id: &'static str,
|
||||
youtube_proxy_id: &'static str,
|
||||
youtube_cookie_id: &'static str,
|
||||
podcast_audio_proxy_enabled: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -667,6 +670,8 @@ pub(super) struct UpdateSettingsRequest {
|
||||
youtube_proxy_id: String,
|
||||
#[serde(default)]
|
||||
youtube_cookie_id: String,
|
||||
#[serde(default)]
|
||||
podcast_audio_proxy_enabled: bool,
|
||||
}
|
||||
|
||||
fn default_similarity_model() -> String {
|
||||
@@ -1370,6 +1375,10 @@ pub async fn update_settings(
|
||||
("torrent_proxy_id", torrent_proxy_id),
|
||||
("youtube_proxy_id", youtube_proxy_id),
|
||||
("youtube_cookie_id", youtube_cookie_id),
|
||||
(
|
||||
"podcast_audio_proxy_enabled",
|
||||
body.podcast_audio_proxy_enabled.to_string(),
|
||||
),
|
||||
];
|
||||
for (key, value) in fields {
|
||||
let mut entry = ConfigEntry::new(key.to_string(), value);
|
||||
@@ -1714,6 +1723,7 @@ fn settings_dto(
|
||||
torrent_proxy_id: config.torrent_proxy_id,
|
||||
youtube_proxy_id: config.youtube_proxy_id,
|
||||
youtube_cookie_id: config.youtube_cookie_id,
|
||||
podcast_audio_proxy_enabled: config.podcast_audio_proxy_enabled,
|
||||
},
|
||||
sources: AdminSettingsSources {
|
||||
auth_password_enabled: sources.auth_password_enabled.code(),
|
||||
@@ -1750,6 +1760,7 @@ fn settings_dto(
|
||||
torrent_proxy_id: sources.torrent_proxy_id.code(),
|
||||
youtube_proxy_id: sources.youtube_proxy_id.code(),
|
||||
youtube_cookie_id: sources.youtube_cookie_id.code(),
|
||||
podcast_audio_proxy_enabled: sources.podcast_audio_proxy_enabled.code(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,6 +149,7 @@ pub struct ConfigSources {
|
||||
pub torrent_proxy_id: ConfigSource,
|
||||
pub youtube_proxy_id: ConfigSource,
|
||||
pub youtube_cookie_id: ConfigSource,
|
||||
pub podcast_audio_proxy_enabled: ConfigSource,
|
||||
}
|
||||
|
||||
impl Default for ConfigSources {
|
||||
@@ -190,6 +191,7 @@ impl Default for ConfigSources {
|
||||
torrent_proxy_id: ConfigSource::Default,
|
||||
youtube_proxy_id: ConfigSource::Default,
|
||||
youtube_cookie_id: ConfigSource::Default,
|
||||
podcast_audio_proxy_enabled: ConfigSource::Default,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -407,6 +409,10 @@ pub struct AppConfig {
|
||||
pub youtube_proxy_id: String,
|
||||
/// Saved cookie-file id used by yt-dlp; empty means no cookies.
|
||||
pub youtube_cookie_id: String,
|
||||
/// Whether the server proxies podcast episode audio from publishers for
|
||||
/// browsers that cannot load it directly (CORS, mixed content). Off by
|
||||
/// default: browsers stream from the publisher URL.
|
||||
pub podcast_audio_proxy_enabled: bool,
|
||||
}
|
||||
|
||||
impl Default for AppConfig {
|
||||
@@ -451,6 +457,7 @@ impl Default for AppConfig {
|
||||
torrent_proxy_id: String::new(),
|
||||
youtube_proxy_id: String::new(),
|
||||
youtube_cookie_id: String::new(),
|
||||
podcast_audio_proxy_enabled: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -493,6 +500,7 @@ impl_env_overrides!(
|
||||
torrent_proxy_id,
|
||||
youtube_proxy_id,
|
||||
youtube_cookie_id,
|
||||
podcast_audio_proxy_enabled,
|
||||
);
|
||||
|
||||
impl AppConfig {
|
||||
@@ -636,6 +644,7 @@ impl AppConfig {
|
||||
apply_db_field!(torrent_proxy_id);
|
||||
apply_db_field!(youtube_proxy_id);
|
||||
apply_db_field!(youtube_cookie_id);
|
||||
apply_db_field!(podcast_audio_proxy_enabled);
|
||||
}
|
||||
|
||||
pub fn parsed_download_proxies(&self) -> anyhow::Result<Vec<DownloadProxy>> {
|
||||
@@ -693,6 +702,7 @@ mod tests {
|
||||
assert!(cfg.torrent_downloads_enabled);
|
||||
assert!(cfg.youtube_downloads_enabled);
|
||||
assert!(cfg.parsed_download_proxies().unwrap().is_empty());
|
||||
assert!(!cfg.podcast_audio_proxy_enabled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -477,6 +477,10 @@ impl Project for FuruProject {
|
||||
" API:\n",
|
||||
" FURU_SWAGGER_ENABLED Enable Swagger UI at /swagger/ (default: false)\n",
|
||||
"\n",
|
||||
" Podcasts:\n",
|
||||
" FURU_PODCAST_AUDIO_PROXY_ENABLED Stream episode audio through this\n",
|
||||
" server (default: false)\n",
|
||||
"\n",
|
||||
"QUICK START\n",
|
||||
" export FURU_DATABASE_URL=postgres://user:pass@localhost/furumusic\n",
|
||||
" furumusic run",
|
||||
@@ -581,6 +585,10 @@ impl Project for FuruProject {
|
||||
"/admin",
|
||||
);
|
||||
apps.register_with_views(api::ApiApp, "/api");
|
||||
apps.register_with_views(
|
||||
podcasts::http::PodcastsApp::new(Arc::clone(&self.app_config)),
|
||||
"/api/podcasts",
|
||||
);
|
||||
apps.register_with_views(
|
||||
player::PlayerApp::new(
|
||||
Arc::clone(&self.app_config),
|
||||
|
||||
@@ -0,0 +1,379 @@
|
||||
//! HTTP endpoints of the podcast runtime, mounted at `/api/podcasts`:
|
||||
//!
|
||||
//! * `POST /{podcast_id}/refresh` — manual feed refresh of a subscribed
|
||||
//! podcast (ignores the failure backoff, at most every 30 s).
|
||||
//! * `GET /episodes/{episode_id}/source` — how the browser gets the audio:
|
||||
//! the publisher URL (default, the browser streams it directly) and the
|
||||
//! proxy URL when the administrator enabled the proxy.
|
||||
//! * `GET /episodes/{episode_id}/audio` — the optional audio proxy
|
||||
//! (`podcast_audio_proxy_enabled`). It takes an episode id, never a URL,
|
||||
//! and only serves episodes of the signed-in account.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use cot::db::Database;
|
||||
use cot::http::StatusCode;
|
||||
use cot::http::header::{
|
||||
ACCEPT_RANGES, CACHE_CONTROL, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, RANGE, RETRY_AFTER,
|
||||
};
|
||||
use cot::json::Json;
|
||||
use cot::request::extractors::Path;
|
||||
use cot::response::IntoResponse;
|
||||
use cot::router::method::{get, post};
|
||||
use cot::router::{Route, Router};
|
||||
use cot::session::Session;
|
||||
use cot::{App, Body};
|
||||
use futures_util::TryStreamExt;
|
||||
use music_dht::podcast::{EpisodeId, PodcastId};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::fetch::FeedFetcher;
|
||||
use super::proxy::AudioProxy;
|
||||
use super::runner::{self, ManualRefreshError, TargetResult};
|
||||
use super::{PodcastStore, now_ms};
|
||||
use crate::auth;
|
||||
use crate::config::AppConfig;
|
||||
|
||||
fn json_error(status: StatusCode, message: &str) -> cot::response::Response {
|
||||
let body = serde_json::json!({ "error": message });
|
||||
cot::http::Response::builder()
|
||||
.status(status)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.header(CACHE_CONTROL, "private, no-store")
|
||||
.body(Body::fixed(body.to_string()))
|
||||
.expect("valid response")
|
||||
}
|
||||
|
||||
fn internal(error: impl std::fmt::Display) -> cot::Error {
|
||||
cot::Error::internal(error.to_string())
|
||||
}
|
||||
|
||||
/// Shared state of the handlers.
|
||||
#[derive(Clone)]
|
||||
struct State {
|
||||
config: Arc<AppConfig>,
|
||||
pool: Arc<tokio::sync::OnceCell<sqlx::PgPool>>,
|
||||
fetcher: Arc<tokio::sync::OnceCell<FeedFetcher>>,
|
||||
proxy: Arc<tokio::sync::OnceCell<AudioProxy>>,
|
||||
}
|
||||
|
||||
impl State {
|
||||
async fn store(&self) -> cot::Result<PodcastStore> {
|
||||
let pool = self
|
||||
.pool
|
||||
.get_or_try_init(|| async {
|
||||
sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(5)
|
||||
.connect(&self.config.database_url)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
Ok(PodcastStore::new(pool.clone()))
|
||||
}
|
||||
|
||||
async fn fetcher(&self) -> cot::Result<&FeedFetcher> {
|
||||
self.fetcher
|
||||
.get_or_try_init(|| async { FeedFetcher::new() })
|
||||
.await
|
||||
.map_err(internal)
|
||||
}
|
||||
|
||||
async fn proxy(&self) -> cot::Result<&AudioProxy> {
|
||||
self.proxy
|
||||
.get_or_try_init(|| async { AudioProxy::new() })
|
||||
.await
|
||||
.map_err(internal)
|
||||
}
|
||||
}
|
||||
|
||||
/// The podcast runtime endpoints (see the module documentation).
|
||||
pub(crate) struct PodcastsApp {
|
||||
state: State,
|
||||
}
|
||||
|
||||
impl PodcastsApp {
|
||||
pub(crate) fn new(config: Arc<AppConfig>) -> Self {
|
||||
Self {
|
||||
state: State {
|
||||
config,
|
||||
pool: Arc::new(tokio::sync::OnceCell::new()),
|
||||
fetcher: Arc::new(tokio::sync::OnceCell::new()),
|
||||
proxy: Arc::new(tokio::sync::OnceCell::new()),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct PodcastPath {
|
||||
podcast_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct EpisodePath {
|
||||
episode_id: String,
|
||||
}
|
||||
|
||||
/// Path of the proxy endpoint for an episode.
|
||||
pub(crate) fn proxy_path(episode_id: &EpisodeId) -> String {
|
||||
format!("/api/podcasts/episodes/{episode_id}/audio")
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RefreshResponse {
|
||||
status: &'static str,
|
||||
new_episodes: usize,
|
||||
updated_episodes: usize,
|
||||
unchanged_episodes: usize,
|
||||
skipped_items: usize,
|
||||
feed_url: Option<String>,
|
||||
error: Option<String>,
|
||||
next_check_at_ms: Option<i64>,
|
||||
}
|
||||
|
||||
impl RefreshResponse {
|
||||
fn from_result(result: TargetResult) -> Self {
|
||||
let mut response = Self {
|
||||
status: "",
|
||||
new_episodes: 0,
|
||||
updated_episodes: 0,
|
||||
unchanged_episodes: 0,
|
||||
skipped_items: 0,
|
||||
feed_url: None,
|
||||
error: None,
|
||||
next_check_at_ms: None,
|
||||
};
|
||||
match result {
|
||||
TargetResult::Updated(outcome) => {
|
||||
response.status = "updated";
|
||||
response.new_episodes = outcome.new_episodes;
|
||||
response.updated_episodes = outcome.updated_episodes;
|
||||
response.unchanged_episodes = outcome.unchanged_episodes;
|
||||
response.skipped_items = outcome.skipped_items;
|
||||
response.feed_url = outcome.feed_url.map(String::from);
|
||||
}
|
||||
TargetResult::NotModified => response.status = "not_modified",
|
||||
TargetResult::Failed {
|
||||
error,
|
||||
next_check_at_ms,
|
||||
..
|
||||
} => {
|
||||
response.status = "failed";
|
||||
response.error = Some(error);
|
||||
response.next_check_at_ms = Some(next_check_at_ms);
|
||||
}
|
||||
TargetResult::Skipped(reason) => {
|
||||
response.status = "skipped";
|
||||
response.error = Some(reason);
|
||||
}
|
||||
}
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
async fn refresh_handler(
|
||||
state: State,
|
||||
auth_ctx: auth::AuthContext,
|
||||
session: Session,
|
||||
db: Database,
|
||||
path: Path<PodcastPath>,
|
||||
) -> cot::Result<cot::response::Response> {
|
||||
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
|
||||
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
|
||||
};
|
||||
let Ok(podcast_id) = PodcastId::parse(&path.0.podcast_id) else {
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, "invalid podcast id"));
|
||||
};
|
||||
let store = state.store().await?;
|
||||
let fetcher = state.fetcher().await?;
|
||||
match runner::refresh_now(&store, fetcher, user.id, &podcast_id, now_ms()).await {
|
||||
Ok(result) => Json(RefreshResponse::from_result(result)).into_response(),
|
||||
Err(error) => match error.downcast_ref::<ManualRefreshError>() {
|
||||
Some(ManualRefreshError::UnknownPodcast) => {
|
||||
Ok(json_error(StatusCode::NOT_FOUND, "unknown podcast"))
|
||||
}
|
||||
Some(ManualRefreshError::NotSubscribed) => {
|
||||
Ok(json_error(StatusCode::CONFLICT, &error.to_string()))
|
||||
}
|
||||
Some(ManualRefreshError::TooSoon { retry_after_ms }) => {
|
||||
let mut response = json_error(StatusCode::TOO_MANY_REQUESTS, &error.to_string());
|
||||
response.headers_mut().insert(
|
||||
RETRY_AFTER,
|
||||
((retry_after_ms + 999) / 1000)
|
||||
.to_string()
|
||||
.parse()
|
||||
.expect("digits"),
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
None => Err(internal(format!("{error:#}"))),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct SourceResponse {
|
||||
episode_id: String,
|
||||
/// Publisher URL; the browser streams it directly by default.
|
||||
direct_url: String,
|
||||
/// Same-origin proxy path, only when the proxy is enabled.
|
||||
proxy_url: Option<String>,
|
||||
mime: Option<String>,
|
||||
}
|
||||
|
||||
async fn source_handler(
|
||||
state: State,
|
||||
auth_ctx: auth::AuthContext,
|
||||
session: Session,
|
||||
db: Database,
|
||||
path: Path<EpisodePath>,
|
||||
) -> cot::Result<cot::response::Response> {
|
||||
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
|
||||
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
|
||||
};
|
||||
let Ok(episode_id) = EpisodeId::parse(&path.0.episode_id) else {
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, "invalid episode id"));
|
||||
};
|
||||
let store = state.store().await?;
|
||||
let Some((audio_url, mime)) = store
|
||||
.audio_source(user.id, &episode_id)
|
||||
.await
|
||||
.map_err(internal)?
|
||||
else {
|
||||
return Ok(json_error(StatusCode::NOT_FOUND, "unknown episode"));
|
||||
};
|
||||
let (config, _) = AppConfig::load_with_db(&db).await;
|
||||
let mut response = Json(SourceResponse {
|
||||
proxy_url: config
|
||||
.podcast_audio_proxy_enabled
|
||||
.then(|| proxy_path(&episode_id)),
|
||||
episode_id: episode_id.to_string(),
|
||||
direct_url: audio_url,
|
||||
mime,
|
||||
})
|
||||
.into_response()?;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CACHE_CONTROL, "private, no-store".parse().expect("static"));
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn audio_handler(
|
||||
state: State,
|
||||
auth_ctx: auth::AuthContext,
|
||||
session: Session,
|
||||
db: Database,
|
||||
path: Path<EpisodePath>,
|
||||
request: cot::request::Request,
|
||||
) -> cot::Result<cot::response::Response> {
|
||||
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
|
||||
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
|
||||
};
|
||||
let (config, _) = AppConfig::load_with_db(&db).await;
|
||||
if !config.podcast_audio_proxy_enabled {
|
||||
return Ok(json_error(
|
||||
StatusCode::NOT_FOUND,
|
||||
"the podcast audio proxy is disabled; stream from the publisher URL",
|
||||
));
|
||||
}
|
||||
let Ok(episode_id) = EpisodeId::parse(&path.0.episode_id) else {
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, "invalid episode id"));
|
||||
};
|
||||
let store = state.store().await?;
|
||||
// The URL comes from this account's catalog, never from the request.
|
||||
let Some((audio_url, mime)) = store
|
||||
.audio_source(user.id, &episode_id)
|
||||
.await
|
||||
.map_err(internal)?
|
||||
else {
|
||||
return Ok(json_error(StatusCode::NOT_FOUND, "unknown episode"));
|
||||
};
|
||||
let range = request
|
||||
.headers()
|
||||
.get(RANGE)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let proxy = state.proxy().await?;
|
||||
let audio = match proxy.fetch(&audio_url, mime.as_deref(), range).await {
|
||||
Ok(audio) => audio,
|
||||
Err(error) => {
|
||||
tracing::info!(user_id = user.id, episode = %episode_id, "podcast proxy: {error}");
|
||||
let status =
|
||||
StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::BAD_GATEWAY);
|
||||
return Ok(json_error(status, &error.to_string()));
|
||||
}
|
||||
};
|
||||
let mut builder = cot::http::Response::builder()
|
||||
.status(StatusCode::from_u16(audio.status).unwrap_or(StatusCode::BAD_GATEWAY))
|
||||
.header(CACHE_CONTROL, "private, no-store")
|
||||
.header("X-Content-Type-Options", "nosniff")
|
||||
.header("Content-Security-Policy", "default-src 'none'; sandbox")
|
||||
.header("Content-Disposition", "inline");
|
||||
if let Some(content_type) = &audio.content_type {
|
||||
builder = builder.header(CONTENT_TYPE, content_type.as_str());
|
||||
}
|
||||
if let Some(length) = audio.content_length {
|
||||
builder = builder.header(CONTENT_LENGTH, length.to_string());
|
||||
}
|
||||
if let Some(content_range) = &audio.content_range {
|
||||
builder = builder.header(CONTENT_RANGE, content_range.as_str());
|
||||
}
|
||||
if audio.accept_ranges {
|
||||
builder = builder.header(ACCEPT_RANGES, "bytes");
|
||||
}
|
||||
let body = audio
|
||||
.body
|
||||
.map_err(|error| cot::Error::internal(error.to_string()));
|
||||
builder.body(Body::streaming(body)).map_err(internal)
|
||||
}
|
||||
|
||||
impl App for PodcastsApp {
|
||||
fn name(&self) -> &'static str {
|
||||
"podcasts"
|
||||
}
|
||||
|
||||
fn router(&self) -> Router {
|
||||
let refresh_state = self.state.clone();
|
||||
let source_state = self.state.clone();
|
||||
let audio_state = self.state.clone();
|
||||
Router::with_urls([
|
||||
Route::with_handler_and_name(
|
||||
"/episodes/{episode_id}/source",
|
||||
get(
|
||||
move |auth_ctx: auth::AuthContext,
|
||||
session: Session,
|
||||
db: Database,
|
||||
path: Path<EpisodePath>| {
|
||||
source_handler(source_state.clone(), auth_ctx, session, db, path)
|
||||
},
|
||||
),
|
||||
"podcasts_episode_source",
|
||||
),
|
||||
Route::with_handler_and_name(
|
||||
"/episodes/{episode_id}/audio",
|
||||
get(
|
||||
move |auth_ctx: auth::AuthContext,
|
||||
session: Session,
|
||||
db: Database,
|
||||
path: Path<EpisodePath>,
|
||||
request: cot::request::Request| {
|
||||
audio_handler(audio_state.clone(), auth_ctx, session, db, path, request)
|
||||
},
|
||||
),
|
||||
"podcasts_episode_audio",
|
||||
),
|
||||
Route::with_handler_and_name(
|
||||
"/{podcast_id}/refresh",
|
||||
post(
|
||||
move |auth_ctx: auth::AuthContext,
|
||||
session: Session,
|
||||
db: Database,
|
||||
path: Path<PodcastPath>| {
|
||||
refresh_handler(refresh_state.clone(), auth_ctx, session, db, path)
|
||||
},
|
||||
),
|
||||
"podcasts_refresh",
|
||||
),
|
||||
])
|
||||
}
|
||||
}
|
||||
+19
-3
@@ -41,17 +41,31 @@
|
||||
//!
|
||||
//! RSS refreshes ([`PodcastStore::apply_feed`]) touch the catalog and the
|
||||
//! alias set only; they never write a register, progress or a session.
|
||||
//!
|
||||
//! # Runtime
|
||||
//!
|
||||
//! * `net` — the SSRF-guarded HTTP client used for every feed and audio
|
||||
//! request (public addresses only, checked per connection and per
|
||||
//! redirect hop).
|
||||
//! * `fetch` and [`runner`] — conditional feed downloads and their schedule
|
||||
//! (hourly checks, exponential backoff, one request per feed URL), run by
|
||||
//! the `podcast_refresh` job and the manual refresh endpoint.
|
||||
//! * `proxy` and [`http`] — the optional audio proxy and the
|
||||
//! `/api/podcasts` endpoints. Audio is streamed from the publisher, never
|
||||
//! stored.
|
||||
|
||||
// The podcast UI and its HTTP API, which call the local actions and most
|
||||
// queries, land in the next commits of this series; until then only sync,
|
||||
// the refresh job and the tests use the store. Remove this allowance with
|
||||
// them.
|
||||
// the refresh job, the proxy endpoints and the tests use the store. Remove
|
||||
// this allowance with them.
|
||||
#![allow(dead_code, unused_imports)]
|
||||
|
||||
mod actions;
|
||||
mod apply;
|
||||
mod fetch;
|
||||
pub(crate) mod http;
|
||||
mod net;
|
||||
mod proxy;
|
||||
mod queries;
|
||||
mod refresh;
|
||||
pub(crate) mod runner;
|
||||
@@ -64,6 +78,8 @@ mod net_tests;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod postgres_tests;
|
||||
#[cfg(test)]
|
||||
mod proxy_tests;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_server;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -80,11 +96,11 @@ use sqlx::postgres::PgRow;
|
||||
|
||||
pub(crate) use actions::{CheckpointOutcome, ProgressSession, ProgressWrite, SubscribeOutcome};
|
||||
pub(crate) use apply::{ApplyOutcome, SnapshotApplyStats};
|
||||
pub(crate) use fetch::FeedFetcher;
|
||||
pub(crate) use queries::{
|
||||
EpisodeListItem, EpisodeQuery, EpisodeSort, PlayedFilter, PodcastListFilter,
|
||||
PodcastListenEntry, PodcastSummary, SeasonFilter, SeasonSummary,
|
||||
};
|
||||
pub(crate) use fetch::FeedFetcher;
|
||||
pub(crate) use refresh::{FeedRefresh, FetchState, RefreshOutcome};
|
||||
|
||||
/// Typed errors of [`PodcastStore`]; other failures are database errors.
|
||||
|
||||
@@ -1566,6 +1566,13 @@ async fn postgres_podcast_refresh_job_schedules_backs_off_and_moves_feeds() {
|
||||
let episode_id = episodes[0].episode.episode_id.clone();
|
||||
assert_eq!(episodes[0].episode.audio_url, server.url("/audio/one.mp3"));
|
||||
|
||||
// The proxy and the source endpoint only find episodes of the account.
|
||||
assert_eq!(
|
||||
store.audio_source(a, &episode_id).await.unwrap(),
|
||||
Some((server.url("/audio/one.mp3"), Some("audio/mpeg".to_string())))
|
||||
);
|
||||
assert_eq!(store.audio_source(c, &episode_id).await.unwrap(), None);
|
||||
|
||||
// Nothing is due before the interval; afterwards the request is
|
||||
// conditional and the catalog stays as it is.
|
||||
assert!(due_for(&store, &users, t0 + 1000).await.is_empty());
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
//! Optional audio proxy: streams a stored episode enclosure from the
|
||||
//! publisher for browsers that cannot load it directly (CORS, mixed
|
||||
//! content). Audio is passed through, never stored or imported.
|
||||
//!
|
||||
//! The caller resolves the URL from the database for an episode of the
|
||||
//! authenticated account; a client never supplies a URL. Requests go
|
||||
//! through [`SafeHttp`] (SSRF guard, every redirect hop checked), carry
|
||||
//! only `Range`, `Accept`, `Accept-Encoding: identity` and the user agent,
|
||||
//! and responses are limited in size, time and content type.
|
||||
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::StreamExt;
|
||||
use futures_util::stream::BoxStream;
|
||||
use reqwest::header::{
|
||||
ACCEPT, ACCEPT_ENCODING, ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, HeaderMap,
|
||||
HeaderValue, RANGE,
|
||||
};
|
||||
use reqwest::{StatusCode, Url};
|
||||
|
||||
use super::net::{GuardError, SafeHttp, SafeHttpConfig};
|
||||
|
||||
/// Largest response body passed through (a long episode at a high bitrate
|
||||
/// stays far below).
|
||||
pub(crate) const MAX_PROXY_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
/// Upstream must answer with headers within this time.
|
||||
pub(crate) const PROXY_HEADER_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
/// Longest a single proxied response may stream.
|
||||
pub(crate) const PROXY_MAX_DURATION: Duration = Duration::from_secs(4 * 60 * 60);
|
||||
/// Redirect hops followed (enclosures often go through tracking redirects).
|
||||
pub(crate) const MAX_PROXY_REDIRECTS: usize = 5;
|
||||
|
||||
/// HTTP client settings of the proxy (no whole-request timeout: bodies are
|
||||
/// streamed; the idle read timeout and [`PROXY_MAX_DURATION`] bound them).
|
||||
pub(crate) const PROXY_HTTP: SafeHttpConfig = SafeHttpConfig {
|
||||
connect_timeout: Duration::from_secs(10),
|
||||
read_timeout: Duration::from_secs(30),
|
||||
total_timeout: None,
|
||||
max_redirects: MAX_PROXY_REDIRECTS,
|
||||
};
|
||||
|
||||
/// Why a proxied request failed.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum ProxyError {
|
||||
/// The stored URL is not a valid URL.
|
||||
InvalidUrl(String),
|
||||
/// Refused or failed in transport (see [`GuardError`]).
|
||||
Upstream(GuardError),
|
||||
/// The publisher answered with an unusable status.
|
||||
Status(u16),
|
||||
/// The publisher's content type is not audio.
|
||||
ContentType(String),
|
||||
/// Declared body larger than [`MAX_PROXY_BYTES`].
|
||||
TooLarge(u64),
|
||||
/// No response headers within [`PROXY_HEADER_TIMEOUT`].
|
||||
Timeout,
|
||||
/// The body stream stopped (limit reached or upstream failure).
|
||||
Body(String),
|
||||
}
|
||||
|
||||
impl ProxyError {
|
||||
/// Status the proxy answers the browser with.
|
||||
pub(crate) fn http_status(&self) -> u16 {
|
||||
match self {
|
||||
Self::Timeout => 504,
|
||||
Self::Upstream(error) if error.is_timeout() => 504,
|
||||
_ => 502,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ProxyError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::InvalidUrl(reason) => write!(f, "stored audio URL is invalid: {reason}"),
|
||||
Self::Upstream(error) => write!(f, "publisher request failed: {error}"),
|
||||
Self::Status(status) => write!(f, "publisher answered HTTP {status}"),
|
||||
Self::ContentType(value) => write!(f, "publisher content type {value:?} is not audio"),
|
||||
Self::TooLarge(len) => write!(
|
||||
f,
|
||||
"publisher response is too large ({len} bytes, limit {MAX_PROXY_BYTES})"
|
||||
),
|
||||
Self::Timeout => write!(
|
||||
f,
|
||||
"publisher did not answer within {} s",
|
||||
PROXY_HEADER_TIMEOUT.as_secs()
|
||||
),
|
||||
Self::Body(reason) => write!(f, "publisher stream failed: {reason}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ProxyError {}
|
||||
|
||||
impl From<GuardError> for ProxyError {
|
||||
fn from(error: GuardError) -> Self {
|
||||
Self::Upstream(error)
|
||||
}
|
||||
}
|
||||
|
||||
/// A proxied response, ready to be sent to the browser.
|
||||
pub(crate) struct ProxiedAudio {
|
||||
/// 200, 206 or 416.
|
||||
pub(crate) status: u16,
|
||||
/// Allow-listed audio type (absent on 416).
|
||||
pub(crate) content_type: Option<String>,
|
||||
pub(crate) content_length: Option<u64>,
|
||||
/// `bytes a-b/n` (206) or `bytes */n` (416).
|
||||
pub(crate) content_range: Option<String>,
|
||||
/// Whether the publisher advertised byte ranges.
|
||||
pub(crate) accept_ranges: bool,
|
||||
/// The body, cut off at [`MAX_PROXY_BYTES`] and [`PROXY_MAX_DURATION`].
|
||||
pub(crate) body: BoxStream<'static, Result<Bytes, ProxyError>>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProxiedAudio {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("ProxiedAudio")
|
||||
.field("status", &self.status)
|
||||
.field("content_type", &self.content_type)
|
||||
.field("content_length", &self.content_length)
|
||||
.field("content_range", &self.content_range)
|
||||
.field("accept_ranges", &self.accept_ranges)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps a browser `Range` header only when it is one well-formed byte
|
||||
/// range (`bytes=a-`, `bytes=a-b`, `bytes=-n`); anything else is dropped
|
||||
/// and the full resource is requested.
|
||||
pub(crate) fn sanitize_range(value: &str) -> Option<String> {
|
||||
let spec = value.trim().strip_prefix("bytes=")?.trim();
|
||||
if spec.len() > 64 {
|
||||
return None;
|
||||
}
|
||||
let (start, end) = spec.split_once('-')?;
|
||||
let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit());
|
||||
let ok = match (start.is_empty(), end.is_empty()) {
|
||||
(false, true) => digits(start) && start.parse::<u64>().is_ok(),
|
||||
(false, false) => {
|
||||
digits(start)
|
||||
&& digits(end)
|
||||
&& matches!((start.parse::<u64>(), end.parse::<u64>()), (Ok(a), Ok(b)) if a <= b)
|
||||
}
|
||||
(true, false) => digits(end) && end.parse::<u64>().is_ok_and(|n| n > 0),
|
||||
(true, true) => false,
|
||||
};
|
||||
ok.then(|| format!("bytes={start}-{end}"))
|
||||
}
|
||||
|
||||
/// `Content-Range` of a 206/416 passed through only when well-formed.
|
||||
fn sanitize_content_range(value: &str) -> Option<String> {
|
||||
let rest = value.trim().strip_prefix("bytes ")?;
|
||||
let (range, total) = rest.split_once('/')?;
|
||||
let total_ok = total == "*" || total.parse::<u64>().is_ok();
|
||||
let range_ok = range == "*"
|
||||
|| range.split_once('-').is_some_and(
|
||||
|(a, b)| matches!((a.parse::<u64>(), b.parse::<u64>()), (Ok(a), Ok(b)) if a <= b),
|
||||
);
|
||||
(total_ok && range_ok && !(range == "*" && total == "*")).then(|| value.trim().to_string())
|
||||
}
|
||||
|
||||
fn is_audio_type(mime: &str) -> bool {
|
||||
let mime = mime.trim().to_ascii_lowercase();
|
||||
let essence = mime.split(';').next().unwrap_or_default().trim();
|
||||
(essence.starts_with("audio/") && essence.len() > "audio/".len())
|
||||
|| matches!(essence, "application/ogg" | "application/x-ogg")
|
||||
}
|
||||
|
||||
/// Audio type implied by the URL path's extension.
|
||||
fn audio_type_from_url(url: &Url) -> Option<&'static str> {
|
||||
let ext = url.path().rsplit('/').next()?.rsplit_once('.')?.1;
|
||||
Some(match ext.to_ascii_lowercase().as_str() {
|
||||
"mp3" => "audio/mpeg",
|
||||
"m4a" | "m4b" | "mp4" => "audio/mp4",
|
||||
"aac" => "audio/aac",
|
||||
"ogg" | "oga" => "audio/ogg",
|
||||
"opus" => "audio/ogg",
|
||||
"flac" => "audio/flac",
|
||||
"wav" => "audio/wav",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// The content type sent to the browser: the publisher's when it is audio;
|
||||
/// for a generic binary type, the stored enclosure type or the URL
|
||||
/// extension when those are audio; otherwise the response is refused.
|
||||
fn audio_content_type(
|
||||
upstream: Option<&str>,
|
||||
stored: Option<&str>,
|
||||
url: &Url,
|
||||
) -> Result<String, ProxyError> {
|
||||
let upstream = upstream.map(str::trim).filter(|value| !value.is_empty());
|
||||
if let Some(value) = upstream
|
||||
&& is_audio_type(value)
|
||||
{
|
||||
return Ok(value.to_string());
|
||||
}
|
||||
let generic = upstream.is_none_or(|value| {
|
||||
let essence = value.split(';').next().unwrap_or_default().trim();
|
||||
essence.eq_ignore_ascii_case("application/octet-stream")
|
||||
|| essence.eq_ignore_ascii_case("binary/octet-stream")
|
||||
});
|
||||
if generic {
|
||||
if let Some(stored) = stored.filter(|value| is_audio_type(value)) {
|
||||
return Ok(stored.trim().to_string());
|
||||
}
|
||||
if let Some(guess) = audio_type_from_url(url) {
|
||||
return Ok(guess.to_string());
|
||||
}
|
||||
}
|
||||
Err(ProxyError::ContentType(
|
||||
upstream.unwrap_or("(none)").to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Streams episode enclosures from publishers.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct AudioProxy {
|
||||
http: SafeHttp,
|
||||
max_bytes: u64,
|
||||
max_duration: Duration,
|
||||
}
|
||||
|
||||
impl AudioProxy {
|
||||
/// The production proxy (public addresses only).
|
||||
pub(crate) fn new() -> anyhow::Result<Self> {
|
||||
Ok(Self::with_http(SafeHttp::public(PROXY_HTTP)?))
|
||||
}
|
||||
|
||||
pub(crate) fn with_http(http: SafeHttp) -> Self {
|
||||
Self {
|
||||
http,
|
||||
max_bytes: MAX_PROXY_BYTES,
|
||||
max_duration: PROXY_MAX_DURATION,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_limits(mut self, max_bytes: u64, max_duration: Duration) -> Self {
|
||||
self.max_bytes = max_bytes;
|
||||
self.max_duration = max_duration;
|
||||
self
|
||||
}
|
||||
|
||||
/// Requests `audio_url` (from the database) with the browser's `Range`
|
||||
/// (sanitized). `stored_mime` is the enclosure type from the feed.
|
||||
pub(crate) async fn fetch(
|
||||
&self,
|
||||
audio_url: &str,
|
||||
stored_mime: Option<&str>,
|
||||
range: Option<&str>,
|
||||
) -> Result<ProxiedAudio, ProxyError> {
|
||||
let url =
|
||||
Url::parse(audio_url).map_err(|error| ProxyError::InvalidUrl(error.to_string()))?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(ACCEPT, HeaderValue::from_static("audio/*, */*;q=0.1"));
|
||||
// Byte ranges must address the stored bytes, not a compressed form.
|
||||
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
|
||||
let range = range.and_then(sanitize_range);
|
||||
if let Some(range) = &range {
|
||||
headers.insert(
|
||||
RANGE,
|
||||
HeaderValue::from_str(range).expect("sanitized range is a valid header"),
|
||||
);
|
||||
}
|
||||
let followed = tokio::time::timeout(PROXY_HEADER_TIMEOUT, self.http.get(url, &headers))
|
||||
.await
|
||||
.map_err(|_| ProxyError::Timeout)??;
|
||||
let response = followed.response;
|
||||
let final_url = followed.final_url;
|
||||
let status = response.status();
|
||||
let header = |name| {
|
||||
response
|
||||
.headers()
|
||||
.get(name)
|
||||
.and_then(|value: &HeaderValue| value.to_str().ok())
|
||||
.map(str::to_string)
|
||||
};
|
||||
let content_range = header(CONTENT_RANGE)
|
||||
.as_deref()
|
||||
.and_then(sanitize_content_range);
|
||||
if status == StatusCode::RANGE_NOT_SATISFIABLE {
|
||||
return Ok(ProxiedAudio {
|
||||
status: 416,
|
||||
content_type: None,
|
||||
content_length: Some(0),
|
||||
content_range,
|
||||
accept_ranges: true,
|
||||
body: futures_util::stream::empty().boxed(),
|
||||
});
|
||||
}
|
||||
let partial = status == StatusCode::PARTIAL_CONTENT;
|
||||
// 206 only answers a range we asked for, and must say which bytes.
|
||||
if !(status == StatusCode::OK || (partial && range.is_some() && content_range.is_some())) {
|
||||
return Err(ProxyError::Status(status.as_u16()));
|
||||
}
|
||||
let content_type =
|
||||
audio_content_type(header(CONTENT_TYPE).as_deref(), stored_mime, &final_url)?;
|
||||
let content_length = header(CONTENT_LENGTH).and_then(|value| value.parse::<u64>().ok());
|
||||
if let Some(len) = content_length
|
||||
&& len > self.max_bytes
|
||||
{
|
||||
return Err(ProxyError::TooLarge(len));
|
||||
}
|
||||
let accept_ranges = partial
|
||||
|| header(ACCEPT_RANGES)
|
||||
.is_some_and(|value| value.trim().eq_ignore_ascii_case("bytes"));
|
||||
Ok(ProxiedAudio {
|
||||
status: status.as_u16(),
|
||||
content_type: Some(content_type),
|
||||
content_length,
|
||||
content_range: if partial { content_range } else { None },
|
||||
accept_ranges,
|
||||
body: limited_body(response, self.max_bytes, self.max_duration),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The upstream body, ending with an error once `max_bytes` would be
|
||||
/// exceeded or `max_duration` has passed. Dropping it (browser gone or
|
||||
/// seeking elsewhere) closes the upstream connection.
|
||||
fn limited_body(
|
||||
response: reqwest::Response,
|
||||
max_bytes: u64,
|
||||
max_duration: Duration,
|
||||
) -> BoxStream<'static, Result<Bytes, ProxyError>> {
|
||||
let deadline = tokio::time::Instant::now() + max_duration;
|
||||
async_stream::stream! {
|
||||
let mut upstream = response.bytes_stream();
|
||||
let mut sent: u64 = 0;
|
||||
loop {
|
||||
let next = match tokio::time::timeout_at(deadline, upstream.next()).await {
|
||||
Ok(next) => next,
|
||||
Err(_) => {
|
||||
yield Err(ProxyError::Body("time limit reached".to_string()));
|
||||
break;
|
||||
}
|
||||
};
|
||||
match next {
|
||||
None => break,
|
||||
Some(Ok(chunk)) => {
|
||||
sent += chunk.len() as u64;
|
||||
if sent > max_bytes {
|
||||
yield Err(ProxyError::Body(format!("size limit of {max_bytes} bytes reached")));
|
||||
break;
|
||||
}
|
||||
yield Ok(chunk);
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
yield Err(ProxyError::Body(GuardError::from(error).to_string()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.boxed()
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
//! Audio proxy: Range pass-through, servers without ranges, redirects,
|
||||
//! content types and limits, against the local test server.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::StreamExt;
|
||||
use futures_util::stream::BoxStream;
|
||||
|
||||
use super::net::{AddressClass, GuardError};
|
||||
use super::proxy::{AudioProxy, ProxyError, sanitize_range};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::test_server::{Route, ScriptedLookup, TestServer, noise, test_http};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audio proxy
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn only_single_byte_ranges_are_forwarded() {
|
||||
assert_eq!(sanitize_range("bytes=0-").as_deref(), Some("bytes=0-"));
|
||||
assert_eq!(
|
||||
sanitize_range("bytes=100-199").as_deref(),
|
||||
Some("bytes=100-199")
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_range(" bytes=-500 ").as_deref(),
|
||||
Some("bytes=-500")
|
||||
);
|
||||
for invalid in [
|
||||
"bytes=1-2,5-6",
|
||||
"items=0-10",
|
||||
"bytes=5-1",
|
||||
"bytes=-",
|
||||
"bytes=-0",
|
||||
"bytes=a-b",
|
||||
"bytes=1-2\r\nCookie: x",
|
||||
"bytes=99999999999999999999999-",
|
||||
"",
|
||||
] {
|
||||
assert_eq!(sanitize_range(invalid), None, "{invalid:?}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect(
|
||||
mut body: BoxStream<'static, Result<bytes::Bytes, ProxyError>>,
|
||||
) -> (Vec<u8>, Option<String>) {
|
||||
let mut out = Vec::new();
|
||||
while let Some(chunk) = body.next().await {
|
||||
match chunk {
|
||||
Ok(chunk) => out.extend_from_slice(&chunk),
|
||||
Err(error) => return (out, Some(error.to_string())),
|
||||
}
|
||||
}
|
||||
(out, None)
|
||||
}
|
||||
|
||||
fn proxy(lookup: Arc<ScriptedLookup>) -> AudioProxy {
|
||||
AudioProxy::with_http(test_http(lookup))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_passes_ranges_through() {
|
||||
let server = TestServer::start();
|
||||
let audio = noise(300_000, 7);
|
||||
server.set("/ep.mp3", Route::new(audio.clone(), "audio/mpeg"));
|
||||
let proxy = proxy(ScriptedLookup::with(&[]));
|
||||
let target = server.url("/ep.mp3");
|
||||
|
||||
let full = proxy.fetch(&target, None, None).await.unwrap();
|
||||
assert_eq!(full.status, 200);
|
||||
assert_eq!(full.content_type.as_deref(), Some("audio/mpeg"));
|
||||
assert_eq!(full.content_length, Some(audio.len() as u64));
|
||||
assert!(full.accept_ranges);
|
||||
assert_eq!(full.content_range, None);
|
||||
assert_eq!(collect(full.body).await, (audio.clone(), None));
|
||||
|
||||
for (range, start, end) in [
|
||||
("bytes=1000-1999", 1000, 2000),
|
||||
("bytes=299000-", 299_000, 300_000),
|
||||
("bytes=-500", 299_500, 300_000),
|
||||
("bytes=0-", 0, 300_000),
|
||||
] {
|
||||
let part = proxy.fetch(&target, None, Some(range)).await.unwrap();
|
||||
assert_eq!(part.status, 206, "{range}");
|
||||
assert_eq!(
|
||||
part.content_range,
|
||||
Some(format!("bytes {start}-{}/{}", end - 1, audio.len())),
|
||||
"{range}"
|
||||
);
|
||||
assert_eq!(part.content_length, Some((end - start) as u64));
|
||||
assert_eq!(collect(part.body).await, (audio[start..end].to_vec(), None));
|
||||
}
|
||||
|
||||
let unsatisfiable = proxy
|
||||
.fetch(&target, None, Some("bytes=400000-"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(unsatisfiable.status, 416);
|
||||
assert_eq!(
|
||||
unsatisfiable.content_range,
|
||||
Some(format!("bytes */{}", audio.len()))
|
||||
);
|
||||
assert_eq!(collect(unsatisfiable.body).await, (Vec::new(), None));
|
||||
|
||||
// Malformed or multi-range requests fetch the whole file.
|
||||
let whole = proxy
|
||||
.fetch(&target, None, Some("bytes=5-6,9-10"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(whole.status, 200);
|
||||
|
||||
let requests = server.requests("/ep.mp3");
|
||||
let ranges: Vec<Option<&str>> = requests
|
||||
.iter()
|
||||
.map(|request| request.get("range").map(String::as_str))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
ranges,
|
||||
vec![
|
||||
None,
|
||||
Some("bytes=1000-1999"),
|
||||
Some("bytes=299000-"),
|
||||
Some("bytes=-500"),
|
||||
Some("bytes=0-"),
|
||||
Some("bytes=400000-"),
|
||||
None,
|
||||
]
|
||||
);
|
||||
for request in &requests {
|
||||
assert_eq!(request["accept-encoding"], "identity");
|
||||
assert!(!request.contains_key("cookie"));
|
||||
assert!(!request.contains_key("authorization"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_handles_servers_without_ranges_and_redirects() {
|
||||
let server = TestServer::start();
|
||||
let audio = noise(50_000, 3);
|
||||
server.set(
|
||||
"/plain.mp3",
|
||||
Route::new(audio.clone(), "audio/mpeg").no_ranges(),
|
||||
);
|
||||
server.set("/track/ep.mp3", Route::redirect(302, "/cdn/ep.mp3"));
|
||||
server.set("/cdn/ep.mp3", Route::new(audio.clone(), "audio/mpeg"));
|
||||
server.set(
|
||||
"/track/private.mp3",
|
||||
Route::redirect(302, "http://172.20.0.3/ep.mp3"),
|
||||
);
|
||||
let proxy = proxy(ScriptedLookup::with(&[]));
|
||||
|
||||
// No range support: the whole file with 200 and no Accept-Ranges, so the
|
||||
// browser knows it cannot seek by range.
|
||||
let plain = proxy
|
||||
.fetch(&server.url("/plain.mp3"), None, Some("bytes=100-"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(plain.status, 200);
|
||||
assert!(!plain.accept_ranges);
|
||||
assert_eq!(plain.content_range, None);
|
||||
assert_eq!(collect(plain.body).await.0, audio);
|
||||
|
||||
// Tracking redirects keep the range on every hop.
|
||||
let part = proxy
|
||||
.fetch(&server.url("/track/ep.mp3"), None, Some("bytes=10-19"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(part.status, 206);
|
||||
assert_eq!(collect(part.body).await.0, audio[10..20].to_vec());
|
||||
assert_eq!(server.requests("/track/ep.mp3")[0]["range"], "bytes=10-19");
|
||||
assert_eq!(server.requests("/cdn/ep.mp3")[0]["range"], "bytes=10-19");
|
||||
|
||||
match proxy
|
||||
.fetch(&server.url("/track/private.mp3"), None, None)
|
||||
.await
|
||||
{
|
||||
Err(ProxyError::Upstream(GuardError::Blocked(blocked))) => {
|
||||
assert_eq!(blocked.class, AddressClass::Private)
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_only_serves_audio() {
|
||||
let server = TestServer::start();
|
||||
let audio = noise(1000, 5);
|
||||
server.set(
|
||||
"/page.mp3",
|
||||
Route::new("<script>alert(1)</script>", "text/html"),
|
||||
);
|
||||
server.set("/video.mp4", Route::new(audio.clone(), "video/mp4"));
|
||||
server.set(
|
||||
"/binary",
|
||||
Route::new(audio.clone(), "application/octet-stream"),
|
||||
);
|
||||
server.set(
|
||||
"/binary.m4a",
|
||||
Route::new(audio.clone(), "binary/octet-stream"),
|
||||
);
|
||||
server.set(
|
||||
"/untyped.ogg",
|
||||
Route::new(audio.clone(), "x").no_content_type(),
|
||||
);
|
||||
server.set("/ogg", Route::new(audio.clone(), "application/ogg"));
|
||||
server.set(
|
||||
"/typed",
|
||||
Route::new(audio.clone(), "audio/mp4; codecs=\"mp4a.40.2\""),
|
||||
);
|
||||
server.set("/missing", Route::new("", "text/plain").status(404));
|
||||
server.set("/error", Route::new("", "audio/mpeg").status(500));
|
||||
let proxy = proxy(ScriptedLookup::with(&[]));
|
||||
let content_type = |path: &'static str, stored: Option<&'static str>| {
|
||||
let proxy = proxy.clone();
|
||||
let target = server.url(path);
|
||||
async move {
|
||||
proxy
|
||||
.fetch(&target, stored, None)
|
||||
.await
|
||||
.map(|audio| audio.content_type.unwrap())
|
||||
}
|
||||
};
|
||||
assert!(matches!(
|
||||
content_type("/page.mp3", Some("audio/mpeg")).await,
|
||||
Err(ProxyError::ContentType(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
content_type("/video.mp4", Some("audio/mpeg")).await,
|
||||
Err(ProxyError::ContentType(_))
|
||||
));
|
||||
assert_eq!(
|
||||
content_type("/binary", Some("audio/mpeg")).await.unwrap(),
|
||||
"audio/mpeg"
|
||||
);
|
||||
assert!(matches!(
|
||||
content_type("/binary", Some("text/html")).await,
|
||||
Err(ProxyError::ContentType(_))
|
||||
));
|
||||
assert_eq!(
|
||||
content_type("/binary.m4a", None).await.unwrap(),
|
||||
"audio/mp4"
|
||||
);
|
||||
assert_eq!(
|
||||
content_type("/untyped.ogg", None).await.unwrap(),
|
||||
"audio/ogg"
|
||||
);
|
||||
assert_eq!(content_type("/ogg", None).await.unwrap(), "application/ogg");
|
||||
assert_eq!(
|
||||
content_type("/typed", None).await.unwrap(),
|
||||
"audio/mp4; codecs=\"mp4a.40.2\""
|
||||
);
|
||||
let missing = content_type("/missing", None).await.unwrap_err();
|
||||
assert!(matches!(missing, ProxyError::Status(404)));
|
||||
assert_eq!(missing.http_status(), 502);
|
||||
assert!(matches!(
|
||||
content_type("/error", None).await,
|
||||
Err(ProxyError::Status(500))
|
||||
));
|
||||
assert!(matches!(
|
||||
proxy.fetch("not a url", None, None).await,
|
||||
Err(ProxyError::InvalidUrl(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_limits_size_and_reports_dropped_streams() {
|
||||
let server = TestServer::start();
|
||||
let audio = noise(100_000, 9);
|
||||
server.set("/sized.mp3", Route::new(audio.clone(), "audio/mpeg"));
|
||||
server.set(
|
||||
"/chunked.mp3",
|
||||
Route::new(audio.clone(), "audio/mpeg").no_length(),
|
||||
);
|
||||
server.set(
|
||||
"/cut.mp3",
|
||||
Route::new(audio.clone(), "audio/mpeg").cut(40_000),
|
||||
);
|
||||
let limited = proxy(ScriptedLookup::with(&[])).with_limits(60_000, Duration::from_secs(60));
|
||||
assert!(matches!(
|
||||
limited.fetch(&server.url("/sized.mp3"), None, None).await,
|
||||
Err(ProxyError::TooLarge(100_000))
|
||||
));
|
||||
// A range below the limit is fine.
|
||||
let part = limited
|
||||
.fetch(&server.url("/sized.mp3"), None, Some("bytes=0-49999"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(collect(part.body).await, (audio[..50_000].to_vec(), None));
|
||||
// Without a declared length the stream stops at the limit.
|
||||
let chunked = limited
|
||||
.fetch(&server.url("/chunked.mp3"), None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
let (received, error) = collect(chunked.body).await;
|
||||
assert!(received.len() <= 60_000);
|
||||
assert!(error.unwrap().contains("size limit"));
|
||||
// A connection dropped mid-body ends the stream with an error instead
|
||||
// of a silently truncated success.
|
||||
let cut = proxy(ScriptedLookup::with(&[]))
|
||||
.fetch(&server.url("/cut.mp3"), None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
let (received, error) = collect(cut.body).await;
|
||||
assert!(received.len() <= 40_000);
|
||||
assert!(error.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_stream_has_a_time_limit() {
|
||||
let server = TestServer::start();
|
||||
server.set(
|
||||
"/slow.mp3",
|
||||
Route::new(noise(10, 1), "audio/mpeg").delay_body(Duration::from_secs(3)),
|
||||
);
|
||||
let limited = proxy(ScriptedLookup::with(&[])).with_limits(1 << 20, Duration::from_millis(300));
|
||||
let audio = limited
|
||||
.fetch(&server.url("/slow.mp3"), None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
let (_, error) = collect(audio.body).await;
|
||||
assert!(error.unwrap().contains("time limit"));
|
||||
}
|
||||
@@ -661,4 +661,25 @@ impl PodcastStore {
|
||||
.await?;
|
||||
Ok(count(total))
|
||||
}
|
||||
|
||||
/// Audio URL and enclosure type of an episode in this account's catalog
|
||||
/// (`None` when the account does not know the episode). The audio proxy
|
||||
/// and the playback source endpoint take URLs only from here.
|
||||
pub(crate) async fn audio_source(
|
||||
&self,
|
||||
user_id: i64,
|
||||
episode_id: &EpisodeId,
|
||||
) -> Result<Option<(String, Option<String>)>> {
|
||||
let row = sqlx::query(
|
||||
"SELECT audio_url, mime FROM furumusic__podcast_episode
|
||||
WHERE user_id = $1 AND episode_id = $2",
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(episode_id.as_str())
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
Ok(row
|
||||
.map(|row| Ok::<_, sqlx::Error>((row.try_get("audio_url")?, row.try_get("mime")?)))
|
||||
.transpose()?)
|
||||
}
|
||||
}
|
||||
@@ -70,7 +70,7 @@ html { overscroll-behavior:none; }
|
||||
.task-form-body { overflow:visible; }
|
||||
.run-log-body { min-height:300px; overflow:visible; }
|
||||
.run-log-output { min-height:50dvh; max-height:70dvh; }
|
||||
.settings-layout { grid-template-columns:minmax(0,1fr); grid-template-areas:"access" "oidc" "agent" "agentstatus" "similarity" "similaritystatus" "downloads" "federation" "lastfm" "developer" "actions"; }
|
||||
.settings-layout { grid-template-columns:minmax(0,1fr); grid-template-areas:"access" "oidc" "agent" "agentstatus" "similarity" "similaritystatus" "downloads" "federation" "podcasts" "lastfm" "developer" "actions"; }
|
||||
.settings-federation-body { grid-template-columns:minmax(0,1fr); }
|
||||
.repair-shell { grid-template-columns:250px minmax(0,1fr); }
|
||||
}
|
||||
|
||||
@@ -859,6 +859,7 @@ tbody tr:hover {
|
||||
"similarity similarity similarity similarity similarity similarity similarity similarity similaritystatus similaritystatus similaritystatus similaritystatus"
|
||||
"downloads downloads downloads downloads downloads downloads downloads downloads downloads downloads downloads downloads"
|
||||
"federation federation federation federation federation federation federation federation federation federation federation federation"
|
||||
"podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts podcasts"
|
||||
"lastfm lastfm lastfm lastfm lastfm lastfm lastfm lastfm developer developer developer developer"
|
||||
"actions actions actions actions actions actions actions actions actions actions actions actions";
|
||||
gap: 16px;
|
||||
@@ -881,6 +882,7 @@ tbody tr:hover {
|
||||
.settings-similarity-status { grid-area: similaritystatus; }
|
||||
.settings-downloads { grid-area: downloads; }
|
||||
.settings-federation { grid-area: federation; }
|
||||
.settings-podcasts { grid-area: podcasts; }
|
||||
.settings-lastfm { grid-area: lastfm; }
|
||||
.settings-developer { grid-area: developer; }
|
||||
.settings-section-narrow { border-left: 2px solid rgba(29, 185, 84, 0.55); }
|
||||
|
||||
+23
-1
@@ -1167,6 +1167,27 @@
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<section class="panel settings-section settings-section-full settings-podcasts" id="settings-podcasts">
|
||||
<div class="panel-head">
|
||||
<div class="panel-title">
|
||||
<strong>Podcasts</strong>
|
||||
<span>Episode audio comes from the publisher</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="settings-grid">
|
||||
<div class="setting-toggle settings-wide">
|
||||
<label>
|
||||
<span>Podcast audio proxy</span>
|
||||
<span class="source-pill" :class="sourceClass('podcast_audio_proxy_enabled')" x-text="settingSource('podcast_audio_proxy_enabled')"></span>
|
||||
</label>
|
||||
<div class="setting-toggle-row">
|
||||
<span x-text="settingsDraft.podcast_audio_proxy_enabled ? 'Stream through this server when needed' : 'Browsers stream from the publisher'"></span>
|
||||
<input type="checkbox" x-model="settingsDraft.podcast_audio_proxy_enabled" />
|
||||
</div>
|
||||
<div class="setting-help">By default the browser loads episode audio directly from the publisher. Enable the proxy for publishers the browser cannot play directly (missing CORS, plain HTTP on an HTTPS site). The server then fetches the audio from the publisher on demand and passes it through without storing it; it only connects to public addresses.</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel settings-section settings-section-narrow settings-similarity-status">
|
||||
<div class="panel-head">
|
||||
@@ -2179,7 +2200,8 @@ function adminV2() {
|
||||
download_proxies: [],
|
||||
torrent_proxy_id: '',
|
||||
youtube_proxy_id: '',
|
||||
youtube_cookie_id: ''
|
||||
youtube_cookie_id: '',
|
||||
podcast_audio_proxy_enabled: false
|
||||
},
|
||||
settingsProbe: { status: 'idle', ok: false },
|
||||
settingsProbeLoading: false,
|
||||
|
||||
Reference in new issue
Block a user