Pass generic proxy responses through only when they start like audio

The enclosure type of a feed or of a client-sent episode reference no
longer makes an application/octet-stream response acceptable.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
sanapachandClaude Opus 5.5 committed 2026-11-20 15:12:38 +02:00
1 parent cfebe1898f
commit fc0befc4a3
4 files changed
+278 -81

No files matched your search

+2 -2
View File
@@ -414,7 +414,7 @@ async fn audio_handler(
};
let store = state.store().await?;
// The URL comes from this account's catalog, never from the request.
let Some((audio_url, mime)) = store
let Some((audio_url, _)) = store
.audio_source(user.id, &episode_id)
.await
.map_err(internal)?
@@ -430,7 +430,7 @@ async fn audio_handler(
return Ok(too_busy("too many podcast audio streams in progress"));
};
let proxy = state.proxy().await?;
let audio = match proxy.fetch(&audio_url, mime.as_deref(), range).await {
let audio = match proxy.fetch(&audio_url, range).await {
Ok(audio) => audio,
Err(error) => {
// Neither the URL (access tokens) nor the account is logged.
+169 -27
View File
@@ -6,7 +6,10 @@
//! authenticated account; a client never supplies a URL. Requests go
//! through [`SafeHttp`] (SSRF guard, every redirect hop checked), carry
//! only `Range`, `Accept`, `Accept-Encoding: identity` and the user agent,
//! and responses are limited in size, time and content type.
//! and responses are limited in size, time and content type. The content
//! type is decided by the publisher's response (and, for generic binary
//! types, the URL extension or the file's first bytes), never by the
//! enclosure type stored with the episode.
use std::fmt;
use std::time::Duration;
@@ -202,36 +205,124 @@ fn audio_type_from_url(url: &Url) -> Option<&'static str> {
})
}
/// Bytes read from the start of a file to recognize its format.
const SNIFF_BYTES: usize = 16;
/// What the declared type and the URL say about a response.
#[derive(Debug, PartialEq, Eq)]
enum TypeCheck {
/// Audio: the publisher's type, or a generic type with an audio file
/// extension.
Audio(String),
/// A generic binary type (or none) without an audio extension: only the
/// file's first bytes can tell.
Content,
}
/// The content type sent to the browser: the publisher's when it is audio;
/// for a generic binary type, the stored enclosure type or the URL
/// extension when those are audio; otherwise the response is refused.
fn audio_content_type(
upstream: Option<&str>,
stored: Option<&str>,
url: &Url,
) -> Result<String, ProxyError> {
/// for a generic binary type, the type implied by the URL extension
/// ([`TypeCheck::Content`] without one); anything else is refused.
///
/// The enclosure type from the feed (or from an episode reference a client
/// sent) is never trusted here: it is chosen by whoever wrote the feed, so
/// it would turn the proxy into a relay for any file served as
/// `application/octet-stream`.
fn audio_content_type(upstream: Option<&str>, url: &Url) -> Result<TypeCheck, ProxyError> {
let upstream = upstream.map(str::trim).filter(|value| !value.is_empty());
if let Some(value) = upstream
&& is_audio_type(value)
{
return Ok(value.to_string());
return Ok(TypeCheck::Audio(value.to_string()));
}
let generic = upstream.is_none_or(|value| {
let essence = value.split(';').next().unwrap_or_default().trim();
essence.eq_ignore_ascii_case("application/octet-stream")
|| essence.eq_ignore_ascii_case("binary/octet-stream")
});
if generic {
if let Some(stored) = stored.filter(|value| is_audio_type(value)) {
return Ok(stored.trim().to_string());
if !generic {
return Err(ProxyError::ContentType(
upstream.unwrap_or("(none)").to_string(),
));
}
Ok(match audio_type_from_url(url) {
Some(guess) => TypeCheck::Audio(guess.to_string()),
None => TypeCheck::Content,
})
}
/// The audio format a file starts with (magic numbers of the formats
/// podcasts use), or `None`.
pub(crate) fn sniff_audio(head: &[u8]) -> Option<&'static str> {
let at = |range: std::ops::Range<usize>| head.get(range);
if head.starts_with(b"ID3") {
return Some("audio/mpeg");
}
if head.starts_with(b"OggS") {
return Some("audio/ogg");
}
if head.starts_with(b"fLaC") {
return Some("audio/flac");
}
if at(0..4) == Some(b"RIFF") && at(8..12) == Some(b"WAVE") {
return Some("audio/wav");
}
// MPEG-4 audio brands only: video files are not passed through.
if at(4..8) == Some(b"ftyp")
&& matches!(
at(8..12),
Some(b"M4A " | b"M4B " | b"M4P " | b"F4A " | b"F4B ")
)
{
return Some("audio/mp4");
}
if let [0xFF, second, ..] = head {
// ADTS AAC: sync, layer 00.
if second & 0xF6 == 0xF0 {
return Some("audio/aac");
}
if let Some(guess) = audio_type_from_url(url) {
return Ok(guess.to_string());
// MPEG audio frame: sync, a defined version and layer.
let version = (second >> 3) & 0b11;
let layer = (second >> 1) & 0b11;
if second & 0xE0 == 0xE0 && version != 0b01 && layer != 0b00 {
return Some("audio/mpeg");
}
}
Err(ProxyError::ContentType(
upstream.unwrap_or("(none)").to_string(),
))
None
}
type Upstream = BoxStream<'static, reqwest::Result<Bytes>>;
/// Reads at least `want` bytes (fewer at the end of the body) from
/// `upstream`; the chunks are sent to the browser afterwards.
async fn read_head(upstream: &mut Upstream, want: usize) -> Result<Vec<Bytes>, ProxyError> {
let mut chunks = Vec::new();
let mut have = 0;
while have < want {
match tokio::time::timeout(PROXY_HEADER_TIMEOUT, upstream.next()).await {
Err(_) => return Err(ProxyError::Timeout),
Ok(None) => break,
Ok(Some(Ok(chunk))) => {
have += chunk.len();
chunks.push(chunk);
}
Ok(Some(Err(error))) => {
return Err(ProxyError::Body(GuardError::from(error).to_string()));
}
}
}
Ok(chunks)
}
fn head_bytes(chunks: &[Bytes]) -> Vec<u8> {
let mut head = Vec::with_capacity(SNIFF_BYTES);
for chunk in chunks {
let take = (SNIFF_BYTES - head.len()).min(chunk.len());
head.extend_from_slice(&chunk[..take]);
if head.len() == SNIFF_BYTES {
break;
}
}
head
}
/// Streams episode enclosures from publishers.
@@ -264,11 +355,13 @@ impl AudioProxy {
}
/// Requests `audio_url` (from the database) with the browser's `Range`
/// (sanitized). `stored_mime` is the enclosure type from the feed.
/// (sanitized). A response with a generic binary type and no audio file
/// extension is passed through only when the file starts like audio
/// (read from this response when it starts at byte 0, otherwise from a
/// separate 16-byte range request).
pub(crate) async fn fetch(
&self,
audio_url: &str,
stored_mime: Option<&str>,
range: Option<&str>,
) -> Result<ProxiedAudio, ProxyError> {
let url =
@@ -315,8 +408,8 @@ impl AudioProxy {
if !(status == StatusCode::OK || (partial && range.is_some() && content_range.is_some())) {
return Err(ProxyError::Status(status.as_u16()));
}
let content_type =
audio_content_type(header(CONTENT_TYPE).as_deref(), stored_mime, &final_url)?;
let declared = header(CONTENT_TYPE);
let check = audio_content_type(declared.as_deref(), &final_url)?;
let content_length = header(CONTENT_LENGTH).and_then(|value| value.parse::<u64>().ok());
if let Some(len) = content_length
&& len > self.max_bytes
@@ -326,28 +419,77 @@ impl AudioProxy {
let accept_ranges = partial
|| header(ACCEPT_RANGES)
.is_some_and(|value| value.trim().eq_ignore_ascii_case("bytes"));
let starts_at_zero = !partial
|| content_range
.as_deref()
.is_some_and(|value| value.starts_with("bytes 0-"));
let mut upstream: Upstream = response.bytes_stream().boxed();
let mut head = Vec::new();
let content_type = match check {
TypeCheck::Audio(content_type) => content_type,
TypeCheck::Content => {
let start = if starts_at_zero {
head = read_head(&mut upstream, SNIFF_BYTES).await?;
head_bytes(&head)
} else {
self.file_head(&final_url).await?
};
sniff_audio(&start)
.ok_or_else(|| {
ProxyError::ContentType(format!(
"{} (not audio content)",
declared.as_deref().unwrap_or("(none)")
))
})?
.to_string()
}
};
Ok(ProxiedAudio {
status: status.as_u16(),
content_type: Some(content_type),
content_length,
content_range: if partial { content_range } else { None },
accept_ranges,
body: limited_body(response, self.max_bytes, self.max_duration),
body: limited_body(head, upstream, self.max_bytes, self.max_duration),
})
}
/// The first [`SNIFF_BYTES`] of the file at `url` (a range request; a
/// server without ranges answers with the whole file, of which only the
/// start is read before the connection is dropped).
async fn file_head(&self, url: &Url) -> Result<Vec<u8>, ProxyError> {
let mut headers = HeaderMap::new();
headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
headers.insert(
RANGE,
HeaderValue::from_str(&format!("bytes=0-{}", SNIFF_BYTES - 1)).expect("valid range"),
);
let followed =
tokio::time::timeout(PROXY_HEADER_TIMEOUT, self.http.get(url.clone(), &headers))
.await
.map_err(|_| ProxyError::Timeout)??;
let status = followed.response.status();
if !(status == StatusCode::OK || status == StatusCode::PARTIAL_CONTENT) {
return Err(ProxyError::Status(status.as_u16()));
}
let mut upstream: Upstream = followed.response.bytes_stream().boxed();
Ok(head_bytes(&read_head(&mut upstream, SNIFF_BYTES).await?))
}
}
/// The upstream body, ending with an error once `max_bytes` would be
/// exceeded or `max_duration` has passed. Dropping it (browser gone or
/// seeking elsewhere) closes the upstream connection.
/// The upstream body (`head`, already read, first), ending with an error
/// once `max_bytes` would be exceeded or `max_duration` has passed.
/// Dropping it (browser gone or seeking elsewhere) closes the upstream
/// connection.
fn limited_body(
response: reqwest::Response,
head: Vec<Bytes>,
upstream: Upstream,
max_bytes: u64,
max_duration: Duration,
) -> BoxStream<'static, Result<Bytes, ProxyError>> {
let deadline = tokio::time::Instant::now() + max_duration;
let mut upstream = futures_util::stream::iter(head.into_iter().map(Ok)).chain(upstream);
async_stream::stream! {
let mut upstream = response.bytes_stream();
let mut sent: u64 = 0;
loop {
let next = match tokio::time::timeout_at(deadline, upstream.next()).await {
+92 -44
View File
@@ -7,7 +7,7 @@ use futures_util::StreamExt;
use futures_util::stream::BoxStream;
use super::net::{AddressClass, GuardError};
use super::proxy::{AudioProxy, ProxyError, sanitize_range};
use super::proxy::{AudioProxy, ProxyError, sanitize_range, sniff_audio};
use std::sync::Arc;
use super::test_server::{Route, ScriptedLookup, TestServer, noise, test_http};
@@ -67,7 +67,7 @@ async fn proxy_passes_ranges_through() {
let proxy = proxy(ScriptedLookup::with(&[]));
let target = server.url("/ep.mp3");
let full = proxy.fetch(&target, None, None).await.unwrap();
let full = proxy.fetch(&target, None).await.unwrap();
assert_eq!(full.status, 200);
assert_eq!(full.content_type.as_deref(), Some("audio/mpeg"));
assert_eq!(full.content_length, Some(audio.len() as u64));
@@ -81,7 +81,7 @@ async fn proxy_passes_ranges_through() {
("bytes=-500", 299_500, 300_000),
("bytes=0-", 0, 300_000),
] {
let part = proxy.fetch(&target, None, Some(range)).await.unwrap();
let part = proxy.fetch(&target, Some(range)).await.unwrap();
assert_eq!(part.status, 206, "{range}");
assert_eq!(
part.content_range,
@@ -92,10 +92,7 @@ async fn proxy_passes_ranges_through() {
assert_eq!(collect(part.body).await, (audio[start..end].to_vec(), None));
}
let unsatisfiable = proxy
.fetch(&target, None, Some("bytes=400000-"))
.await
.unwrap();
let unsatisfiable = proxy.fetch(&target, Some("bytes=400000-")).await.unwrap();
assert_eq!(unsatisfiable.status, 416);
assert_eq!(
unsatisfiable.content_range,
@@ -104,10 +101,7 @@ async fn proxy_passes_ranges_through() {
assert_eq!(collect(unsatisfiable.body).await, (Vec::new(), None));
// Malformed or multi-range requests fetch the whole file.
let whole = proxy
.fetch(&target, None, Some("bytes=5-6,9-10"))
.await
.unwrap();
let whole = proxy.fetch(&target, Some("bytes=5-6,9-10")).await.unwrap();
assert_eq!(whole.status, 200);
let requests = server.requests("/ep.mp3");
@@ -153,7 +147,7 @@ async fn proxy_handles_servers_without_ranges_and_redirects() {
// No range support: the whole file with 200 and no Accept-Ranges, so the
// browser knows it cannot seek by range.
let plain = proxy
.fetch(&server.url("/plain.mp3"), None, Some("bytes=100-"))
.fetch(&server.url("/plain.mp3"), Some("bytes=100-"))
.await
.unwrap();
assert_eq!(plain.status, 200);
@@ -163,7 +157,7 @@ async fn proxy_handles_servers_without_ranges_and_redirects() {
// Tracking redirects keep the range on every hop.
let part = proxy
.fetch(&server.url("/track/ep.mp3"), None, Some("bytes=10-19"))
.fetch(&server.url("/track/ep.mp3"), Some("bytes=10-19"))
.await
.unwrap();
assert_eq!(part.status, 206);
@@ -171,10 +165,7 @@ async fn proxy_handles_servers_without_ranges_and_redirects() {
assert_eq!(server.requests("/track/ep.mp3")[0]["range"], "bytes=10-19");
assert_eq!(server.requests("/cdn/ep.mp3")[0]["range"], "bytes=10-19");
match proxy
.fetch(&server.url("/track/private.mp3"), None, None)
.await
{
match proxy.fetch(&server.url("/track/private.mp3"), None).await {
Err(ProxyError::Upstream(GuardError::Blocked(blocked))) => {
assert_eq!(blocked.class, AddressClass::Private)
}
@@ -182,19 +173,37 @@ async fn proxy_handles_servers_without_ranges_and_redirects() {
}
}
/// `noise` that starts like a file of the given format.
fn audio_file(magic: &[u8], len: usize) -> Vec<u8> {
let mut file = magic.to_vec();
file.extend(noise(len - magic.len(), 3));
file
}
#[tokio::test]
async fn proxy_only_serves_audio() {
let server = TestServer::start();
let audio = noise(1000, 5);
let mp3 = audio_file(b"ID3\x04\0\0", 1000);
server.set(
"/page.mp3",
Route::new("<script>alert(1)</script>", "text/html"),
);
server.set("/video.mp4", Route::new(audio.clone(), "video/mp4"));
// Generic types: the URL extension, or else the file's first bytes.
server.set(
"/binary",
Route::new(audio.clone(), "application/octet-stream"),
);
server.set(
"/zip",
Route::new(audio_file(b"PK\x03\x04", 1000), "application/octet-stream"),
);
server.set(
"/episode",
Route::new(mp3.clone(), "application/octet-stream"),
);
server.set("/untyped", Route::new(mp3.clone(), "x").no_content_type());
server.set(
"/binary.m4a",
Route::new(audio.clone(), "binary/octet-stream"),
@@ -211,32 +220,45 @@ async fn proxy_only_serves_audio() {
server.set("/missing", Route::new("", "text/plain").status(404));
server.set("/error", Route::new("", "audio/mpeg").status(500));
let proxy = proxy(ScriptedLookup::with(&[]));
let content_type = |path: &'static str, stored: Option<&'static str>| {
let content_type = |path: &'static str, range: Option<&'static str>| {
let proxy = proxy.clone();
let target = server.url(path);
async move {
proxy
.fetch(&target, stored, None)
.fetch(&target, range)
.await
.map(|audio| audio.content_type.unwrap())
}
};
assert!(matches!(
content_type("/page.mp3", Some("audio/mpeg")).await,
Err(ProxyError::ContentType(_))
));
assert!(matches!(
content_type("/video.mp4", Some("audio/mpeg")).await,
Err(ProxyError::ContentType(_))
));
assert_eq!(
content_type("/binary", Some("audio/mpeg")).await.unwrap(),
"audio/mpeg"
);
assert!(matches!(
content_type("/binary", Some("text/html")).await,
Err(ProxyError::ContentType(_))
));
for refused in ["/page.mp3", "/video.mp4", "/binary", "/zip"] {
for range in [None, Some("bytes=10-")] {
assert!(
matches!(
content_type(refused, range).await,
Err(ProxyError::ContentType(_))
),
"{refused} {range:?}"
);
}
}
assert_eq!(content_type("/episode", None).await.unwrap(), "audio/mpeg");
assert_eq!(content_type("/untyped", None).await.unwrap(), "audio/mpeg");
// The sniffed bytes are still sent to the browser.
let whole = proxy.fetch(&server.url("/episode"), None).await.unwrap();
assert_eq!(collect(whole.body).await, (mp3.clone(), None));
// A range past the start: the file's head is read with its own request.
let before = server.requests("/episode").len();
let part = proxy
.fetch(&server.url("/episode"), Some("bytes=100-199"))
.await
.unwrap();
assert_eq!(part.content_type.as_deref(), Some("audio/mpeg"));
assert_eq!(collect(part.body).await, (mp3[100..200].to_vec(), None));
let ranges: Vec<String> = server.requests("/episode")[before..]
.iter()
.map(|request| request["range"].clone())
.collect();
assert_eq!(ranges, ["bytes=100-199", "bytes=0-15"]);
assert_eq!(
content_type("/binary.m4a", None).await.unwrap(),
"audio/mp4"
@@ -258,11 +280,40 @@ async fn proxy_only_serves_audio() {
Err(ProxyError::Status(500))
));
assert!(matches!(
proxy.fetch("not a url", None, None).await,
proxy.fetch("not a url", None).await,
Err(ProxyError::InvalidUrl(_))
));
}
#[test]
fn audio_formats_are_recognized_by_their_first_bytes() {
for (head, format) in [
(&b"ID3\x04\0\0\0\0"[..], "audio/mpeg"),
(&[0xFF, 0xFB, 0x90, 0x64][..], "audio/mpeg"),
(&[0xFF, 0xF3, 0x48, 0xC4][..], "audio/mpeg"),
(&[0xFF, 0xF1, 0x50, 0x80][..], "audio/aac"),
(&b"OggS\0\x02"[..], "audio/ogg"),
(&b"fLaC\0\0\0\x22"[..], "audio/flac"),
(&b"RIFF\x24\x08\0\0WAVEfmt "[..], "audio/wav"),
(&b"\0\0\0\x20ftypM4A \0\0\0\0"[..], "audio/mp4"),
] {
assert_eq!(sniff_audio(head), Some(format), "{head:?}");
}
for head in [
&b"\0\0\0\x20ftypisom\0\0\x02\0"[..],
&b"\0\0\0\x18ftypmp42"[..],
&b"PK\x03\x04"[..],
&b"<!DOCTYPE html>"[..],
&b"%PDF-1.7"[..],
&b"RIFF\x24\x08\0\0AVI "[..],
&[0xFF, 0xD8, 0xFF, 0xE0][..],
&[0xFF][..],
&[][..],
] {
assert_eq!(sniff_audio(head), None, "{head:?}");
}
}
#[tokio::test]
async fn proxy_limits_size_and_reports_dropped_streams() {
let server = TestServer::start();
@@ -278,18 +329,18 @@ async fn proxy_limits_size_and_reports_dropped_streams() {
);
let limited = proxy(ScriptedLookup::with(&[])).with_limits(60_000, Duration::from_secs(60));
assert!(matches!(
limited.fetch(&server.url("/sized.mp3"), None, None).await,
limited.fetch(&server.url("/sized.mp3"), None).await,
Err(ProxyError::TooLarge(100_000))
));
// A range below the limit is fine.
let part = limited
.fetch(&server.url("/sized.mp3"), None, Some("bytes=0-49999"))
.fetch(&server.url("/sized.mp3"), Some("bytes=0-49999"))
.await
.unwrap();
assert_eq!(collect(part.body).await, (audio[..50_000].to_vec(), None));
// Without a declared length the stream stops at the limit.
let chunked = limited
.fetch(&server.url("/chunked.mp3"), None, None)
.fetch(&server.url("/chunked.mp3"), None)
.await
.unwrap();
let (received, error) = collect(chunked.body).await;
@@ -298,7 +349,7 @@ async fn proxy_limits_size_and_reports_dropped_streams() {
// A connection dropped mid-body ends the stream with an error instead
// of a silently truncated success.
let cut = proxy(ScriptedLookup::with(&[]))
.fetch(&server.url("/cut.mp3"), None, None)
.fetch(&server.url("/cut.mp3"), None)
.await
.unwrap();
let (received, error) = collect(cut.body).await;
@@ -314,10 +365,7 @@ async fn proxy_stream_has_a_time_limit() {
Route::new(noise(10, 1), "audio/mpeg").delay_body(Duration::from_secs(3)),
);
let limited = proxy(ScriptedLookup::with(&[])).with_limits(1 << 20, Duration::from_millis(300));
let audio = limited
.fetch(&server.url("/slow.mp3"), None, None)
.await
.unwrap();
let audio = limited.fetch(&server.url("/slow.mp3"), None).await.unwrap();
let (_, error) = collect(audio.body).await;
assert!(error.unwrap().contains("time limit"));
}