diff --git a/docs/administration.md b/docs/administration.md index 067d9d1..cf3d333 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -148,14 +148,21 @@ and passes it through without storing it. This uses the server's bandwidth. Security model of the proxy: - It takes an episode id, never a URL. The URL comes from the signed-in - account's own podcast catalog: its feeds, and episode references that its - own trusted devices sent (for example the current item of a handoff). Any - signed-in user can therefore make the server fetch audio URLs of feeds they - subscribe to, as with feed refreshes. -- The same public-address checks as for feeds apply to every connection and - redirect; only audio content types are passed through (no HTML, no video), - with a single byte range, up to 1 GiB and 4 hours per request, and no - cookies or credentials are sent. + account's own podcast catalog: its feeds, and episode references sent by + the account's own clients (its paired devices, or its browser when it plays + a handed-off episode the catalog does not know yet). A signed-in user + controls both (anyone can subscribe to a feed they host), so any signed-in + user can make the server fetch audio from any public URL, as with feed + refreshes. +- What limits that: the same public-address and port checks as for feeds + apply to every connection and redirect, and only audio is passed through. + The publisher's `Content-Type` must be audio; for a generic binary type + (`application/octet-stream` or none), the URL must have an audio file + extension or the file must start like an audio file (MP3, AAC, Ogg, FLAC, + WAV, M4A). The enclosure type written in the feed is never trusted. No + HTML, no video. Requests carry a single byte range, are limited to 1 GiB + and 4 hours each and to four streams per account (64 for the server), and + no cookies or credentials are sent. - Responses are not cached or stored, and are marked `no-store`, `nosniff` and sandboxed. diff --git a/src/podcasts/http.rs b/src/podcasts/http.rs index 45f2fd1..b57ac46 100644 --- a/src/podcasts/http.rs +++ b/src/podcasts/http.rs @@ -414,7 +414,7 @@ async fn audio_handler( }; let store = state.store().await?; // The URL comes from this account's catalog, never from the request. - let Some((audio_url, mime)) = store + let Some((audio_url, _)) = store .audio_source(user.id, &episode_id) .await .map_err(internal)? @@ -430,7 +430,7 @@ async fn audio_handler( return Ok(too_busy("too many podcast audio streams in progress")); }; let proxy = state.proxy().await?; - let audio = match proxy.fetch(&audio_url, mime.as_deref(), range).await { + let audio = match proxy.fetch(&audio_url, range).await { Ok(audio) => audio, Err(error) => { // Neither the URL (access tokens) nor the account is logged. diff --git a/src/podcasts/proxy.rs b/src/podcasts/proxy.rs index ddf63c6..04fd90f 100644 --- a/src/podcasts/proxy.rs +++ b/src/podcasts/proxy.rs @@ -6,7 +6,10 @@ //! authenticated account; a client never supplies a URL. Requests go //! through [`SafeHttp`] (SSRF guard, every redirect hop checked), carry //! only `Range`, `Accept`, `Accept-Encoding: identity` and the user agent, -//! and responses are limited in size, time and content type. +//! and responses are limited in size, time and content type. The content +//! type is decided by the publisher's response (and, for generic binary +//! types, the URL extension or the file's first bytes), never by the +//! enclosure type stored with the episode. use std::fmt; use std::time::Duration; @@ -202,36 +205,124 @@ fn audio_type_from_url(url: &Url) -> Option<&'static str> { }) } +/// Bytes read from the start of a file to recognize its format. +const SNIFF_BYTES: usize = 16; + +/// What the declared type and the URL say about a response. +#[derive(Debug, PartialEq, Eq)] +enum TypeCheck { + /// Audio: the publisher's type, or a generic type with an audio file + /// extension. + Audio(String), + /// A generic binary type (or none) without an audio extension: only the + /// file's first bytes can tell. + Content, +} + /// The content type sent to the browser: the publisher's when it is audio; -/// for a generic binary type, the stored enclosure type or the URL -/// extension when those are audio; otherwise the response is refused. -fn audio_content_type( - upstream: Option<&str>, - stored: Option<&str>, - url: &Url, -) -> Result { +/// for a generic binary type, the type implied by the URL extension +/// ([`TypeCheck::Content`] without one); anything else is refused. +/// +/// The enclosure type from the feed (or from an episode reference a client +/// sent) is never trusted here: it is chosen by whoever wrote the feed, so +/// it would turn the proxy into a relay for any file served as +/// `application/octet-stream`. +fn audio_content_type(upstream: Option<&str>, url: &Url) -> Result { let upstream = upstream.map(str::trim).filter(|value| !value.is_empty()); if let Some(value) = upstream && is_audio_type(value) { - return Ok(value.to_string()); + return Ok(TypeCheck::Audio(value.to_string())); } let generic = upstream.is_none_or(|value| { let essence = value.split(';').next().unwrap_or_default().trim(); essence.eq_ignore_ascii_case("application/octet-stream") || essence.eq_ignore_ascii_case("binary/octet-stream") }); - if generic { - if let Some(stored) = stored.filter(|value| is_audio_type(value)) { - return Ok(stored.trim().to_string()); + if !generic { + return Err(ProxyError::ContentType( + upstream.unwrap_or("(none)").to_string(), + )); + } + Ok(match audio_type_from_url(url) { + Some(guess) => TypeCheck::Audio(guess.to_string()), + None => TypeCheck::Content, + }) +} + +/// The audio format a file starts with (magic numbers of the formats +/// podcasts use), or `None`. +pub(crate) fn sniff_audio(head: &[u8]) -> Option<&'static str> { + let at = |range: std::ops::Range| head.get(range); + if head.starts_with(b"ID3") { + return Some("audio/mpeg"); + } + if head.starts_with(b"OggS") { + return Some("audio/ogg"); + } + if head.starts_with(b"fLaC") { + return Some("audio/flac"); + } + if at(0..4) == Some(b"RIFF") && at(8..12) == Some(b"WAVE") { + return Some("audio/wav"); + } + // MPEG-4 audio brands only: video files are not passed through. + if at(4..8) == Some(b"ftyp") + && matches!( + at(8..12), + Some(b"M4A " | b"M4B " | b"M4P " | b"F4A " | b"F4B ") + ) + { + return Some("audio/mp4"); + } + if let [0xFF, second, ..] = head { + // ADTS AAC: sync, layer 00. + if second & 0xF6 == 0xF0 { + return Some("audio/aac"); } - if let Some(guess) = audio_type_from_url(url) { - return Ok(guess.to_string()); + // MPEG audio frame: sync, a defined version and layer. + let version = (second >> 3) & 0b11; + let layer = (second >> 1) & 0b11; + if second & 0xE0 == 0xE0 && version != 0b01 && layer != 0b00 { + return Some("audio/mpeg"); } } - Err(ProxyError::ContentType( - upstream.unwrap_or("(none)").to_string(), - )) + None +} + +type Upstream = BoxStream<'static, reqwest::Result>; + +/// Reads at least `want` bytes (fewer at the end of the body) from +/// `upstream`; the chunks are sent to the browser afterwards. +async fn read_head(upstream: &mut Upstream, want: usize) -> Result, ProxyError> { + let mut chunks = Vec::new(); + let mut have = 0; + while have < want { + match tokio::time::timeout(PROXY_HEADER_TIMEOUT, upstream.next()).await { + Err(_) => return Err(ProxyError::Timeout), + Ok(None) => break, + Ok(Some(Ok(chunk))) => { + have += chunk.len(); + chunks.push(chunk); + } + Ok(Some(Err(error))) => { + return Err(ProxyError::Body(GuardError::from(error).to_string())); + } + } + } + Ok(chunks) +} + +fn head_bytes(chunks: &[Bytes]) -> Vec { + let mut head = Vec::with_capacity(SNIFF_BYTES); + for chunk in chunks { + let take = (SNIFF_BYTES - head.len()).min(chunk.len()); + head.extend_from_slice(&chunk[..take]); + if head.len() == SNIFF_BYTES { + break; + } + } + head } /// Streams episode enclosures from publishers. @@ -264,11 +355,13 @@ impl AudioProxy { } /// Requests `audio_url` (from the database) with the browser's `Range` - /// (sanitized). `stored_mime` is the enclosure type from the feed. + /// (sanitized). A response with a generic binary type and no audio file + /// extension is passed through only when the file starts like audio + /// (read from this response when it starts at byte 0, otherwise from a + /// separate 16-byte range request). pub(crate) async fn fetch( &self, audio_url: &str, - stored_mime: Option<&str>, range: Option<&str>, ) -> Result { let url = @@ -315,8 +408,8 @@ impl AudioProxy { if !(status == StatusCode::OK || (partial && range.is_some() && content_range.is_some())) { return Err(ProxyError::Status(status.as_u16())); } - let content_type = - audio_content_type(header(CONTENT_TYPE).as_deref(), stored_mime, &final_url)?; + let declared = header(CONTENT_TYPE); + let check = audio_content_type(declared.as_deref(), &final_url)?; let content_length = header(CONTENT_LENGTH).and_then(|value| value.parse::().ok()); if let Some(len) = content_length && len > self.max_bytes @@ -326,28 +419,77 @@ impl AudioProxy { let accept_ranges = partial || header(ACCEPT_RANGES) .is_some_and(|value| value.trim().eq_ignore_ascii_case("bytes")); + let starts_at_zero = !partial + || content_range + .as_deref() + .is_some_and(|value| value.starts_with("bytes 0-")); + let mut upstream: Upstream = response.bytes_stream().boxed(); + let mut head = Vec::new(); + let content_type = match check { + TypeCheck::Audio(content_type) => content_type, + TypeCheck::Content => { + let start = if starts_at_zero { + head = read_head(&mut upstream, SNIFF_BYTES).await?; + head_bytes(&head) + } else { + self.file_head(&final_url).await? + }; + sniff_audio(&start) + .ok_or_else(|| { + ProxyError::ContentType(format!( + "{} (not audio content)", + declared.as_deref().unwrap_or("(none)") + )) + })? + .to_string() + } + }; Ok(ProxiedAudio { status: status.as_u16(), content_type: Some(content_type), content_length, content_range: if partial { content_range } else { None }, accept_ranges, - body: limited_body(response, self.max_bytes, self.max_duration), + body: limited_body(head, upstream, self.max_bytes, self.max_duration), }) } + + /// The first [`SNIFF_BYTES`] of the file at `url` (a range request; a + /// server without ranges answers with the whole file, of which only the + /// start is read before the connection is dropped). + async fn file_head(&self, url: &Url) -> Result, ProxyError> { + let mut headers = HeaderMap::new(); + headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); + headers.insert( + RANGE, + HeaderValue::from_str(&format!("bytes=0-{}", SNIFF_BYTES - 1)).expect("valid range"), + ); + let followed = + tokio::time::timeout(PROXY_HEADER_TIMEOUT, self.http.get(url.clone(), &headers)) + .await + .map_err(|_| ProxyError::Timeout)??; + let status = followed.response.status(); + if !(status == StatusCode::OK || status == StatusCode::PARTIAL_CONTENT) { + return Err(ProxyError::Status(status.as_u16())); + } + let mut upstream: Upstream = followed.response.bytes_stream().boxed(); + Ok(head_bytes(&read_head(&mut upstream, SNIFF_BYTES).await?)) + } } -/// The upstream body, ending with an error once `max_bytes` would be -/// exceeded or `max_duration` has passed. Dropping it (browser gone or -/// seeking elsewhere) closes the upstream connection. +/// The upstream body (`head`, already read, first), ending with an error +/// once `max_bytes` would be exceeded or `max_duration` has passed. +/// Dropping it (browser gone or seeking elsewhere) closes the upstream +/// connection. fn limited_body( - response: reqwest::Response, + head: Vec, + upstream: Upstream, max_bytes: u64, max_duration: Duration, ) -> BoxStream<'static, Result> { let deadline = tokio::time::Instant::now() + max_duration; + let mut upstream = futures_util::stream::iter(head.into_iter().map(Ok)).chain(upstream); async_stream::stream! { - let mut upstream = response.bytes_stream(); let mut sent: u64 = 0; loop { let next = match tokio::time::timeout_at(deadline, upstream.next()).await { diff --git a/src/podcasts/proxy_tests.rs b/src/podcasts/proxy_tests.rs index bf9c173..9eaacc0 100644 --- a/src/podcasts/proxy_tests.rs +++ b/src/podcasts/proxy_tests.rs @@ -7,7 +7,7 @@ use futures_util::StreamExt; use futures_util::stream::BoxStream; use super::net::{AddressClass, GuardError}; -use super::proxy::{AudioProxy, ProxyError, sanitize_range}; +use super::proxy::{AudioProxy, ProxyError, sanitize_range, sniff_audio}; use std::sync::Arc; use super::test_server::{Route, ScriptedLookup, TestServer, noise, test_http}; @@ -67,7 +67,7 @@ async fn proxy_passes_ranges_through() { let proxy = proxy(ScriptedLookup::with(&[])); let target = server.url("/ep.mp3"); - let full = proxy.fetch(&target, None, None).await.unwrap(); + let full = proxy.fetch(&target, None).await.unwrap(); assert_eq!(full.status, 200); assert_eq!(full.content_type.as_deref(), Some("audio/mpeg")); assert_eq!(full.content_length, Some(audio.len() as u64)); @@ -81,7 +81,7 @@ async fn proxy_passes_ranges_through() { ("bytes=-500", 299_500, 300_000), ("bytes=0-", 0, 300_000), ] { - let part = proxy.fetch(&target, None, Some(range)).await.unwrap(); + let part = proxy.fetch(&target, Some(range)).await.unwrap(); assert_eq!(part.status, 206, "{range}"); assert_eq!( part.content_range, @@ -92,10 +92,7 @@ async fn proxy_passes_ranges_through() { assert_eq!(collect(part.body).await, (audio[start..end].to_vec(), None)); } - let unsatisfiable = proxy - .fetch(&target, None, Some("bytes=400000-")) - .await - .unwrap(); + let unsatisfiable = proxy.fetch(&target, Some("bytes=400000-")).await.unwrap(); assert_eq!(unsatisfiable.status, 416); assert_eq!( unsatisfiable.content_range, @@ -104,10 +101,7 @@ async fn proxy_passes_ranges_through() { assert_eq!(collect(unsatisfiable.body).await, (Vec::new(), None)); // Malformed or multi-range requests fetch the whole file. - let whole = proxy - .fetch(&target, None, Some("bytes=5-6,9-10")) - .await - .unwrap(); + let whole = proxy.fetch(&target, Some("bytes=5-6,9-10")).await.unwrap(); assert_eq!(whole.status, 200); let requests = server.requests("/ep.mp3"); @@ -153,7 +147,7 @@ async fn proxy_handles_servers_without_ranges_and_redirects() { // No range support: the whole file with 200 and no Accept-Ranges, so the // browser knows it cannot seek by range. let plain = proxy - .fetch(&server.url("/plain.mp3"), None, Some("bytes=100-")) + .fetch(&server.url("/plain.mp3"), Some("bytes=100-")) .await .unwrap(); assert_eq!(plain.status, 200); @@ -163,7 +157,7 @@ async fn proxy_handles_servers_without_ranges_and_redirects() { // Tracking redirects keep the range on every hop. let part = proxy - .fetch(&server.url("/track/ep.mp3"), None, Some("bytes=10-19")) + .fetch(&server.url("/track/ep.mp3"), Some("bytes=10-19")) .await .unwrap(); assert_eq!(part.status, 206); @@ -171,10 +165,7 @@ async fn proxy_handles_servers_without_ranges_and_redirects() { assert_eq!(server.requests("/track/ep.mp3")[0]["range"], "bytes=10-19"); assert_eq!(server.requests("/cdn/ep.mp3")[0]["range"], "bytes=10-19"); - match proxy - .fetch(&server.url("/track/private.mp3"), None, None) - .await - { + match proxy.fetch(&server.url("/track/private.mp3"), None).await { Err(ProxyError::Upstream(GuardError::Blocked(blocked))) => { assert_eq!(blocked.class, AddressClass::Private) } @@ -182,19 +173,37 @@ async fn proxy_handles_servers_without_ranges_and_redirects() { } } +/// `noise` that starts like a file of the given format. +fn audio_file(magic: &[u8], len: usize) -> Vec { + let mut file = magic.to_vec(); + file.extend(noise(len - magic.len(), 3)); + file +} + #[tokio::test] async fn proxy_only_serves_audio() { let server = TestServer::start(); let audio = noise(1000, 5); + let mp3 = audio_file(b"ID3\x04\0\0", 1000); server.set( "/page.mp3", Route::new("", "text/html"), ); server.set("/video.mp4", Route::new(audio.clone(), "video/mp4")); + // Generic types: the URL extension, or else the file's first bytes. server.set( "/binary", Route::new(audio.clone(), "application/octet-stream"), ); + server.set( + "/zip", + Route::new(audio_file(b"PK\x03\x04", 1000), "application/octet-stream"), + ); + server.set( + "/episode", + Route::new(mp3.clone(), "application/octet-stream"), + ); + server.set("/untyped", Route::new(mp3.clone(), "x").no_content_type()); server.set( "/binary.m4a", Route::new(audio.clone(), "binary/octet-stream"), @@ -211,32 +220,45 @@ async fn proxy_only_serves_audio() { server.set("/missing", Route::new("", "text/plain").status(404)); server.set("/error", Route::new("", "audio/mpeg").status(500)); let proxy = proxy(ScriptedLookup::with(&[])); - let content_type = |path: &'static str, stored: Option<&'static str>| { + let content_type = |path: &'static str, range: Option<&'static str>| { let proxy = proxy.clone(); let target = server.url(path); async move { proxy - .fetch(&target, stored, None) + .fetch(&target, range) .await .map(|audio| audio.content_type.unwrap()) } }; - assert!(matches!( - content_type("/page.mp3", Some("audio/mpeg")).await, - Err(ProxyError::ContentType(_)) - )); - assert!(matches!( - content_type("/video.mp4", Some("audio/mpeg")).await, - Err(ProxyError::ContentType(_)) - )); - assert_eq!( - content_type("/binary", Some("audio/mpeg")).await.unwrap(), - "audio/mpeg" - ); - assert!(matches!( - content_type("/binary", Some("text/html")).await, - Err(ProxyError::ContentType(_)) - )); + for refused in ["/page.mp3", "/video.mp4", "/binary", "/zip"] { + for range in [None, Some("bytes=10-")] { + assert!( + matches!( + content_type(refused, range).await, + Err(ProxyError::ContentType(_)) + ), + "{refused} {range:?}" + ); + } + } + assert_eq!(content_type("/episode", None).await.unwrap(), "audio/mpeg"); + assert_eq!(content_type("/untyped", None).await.unwrap(), "audio/mpeg"); + // The sniffed bytes are still sent to the browser. + let whole = proxy.fetch(&server.url("/episode"), None).await.unwrap(); + assert_eq!(collect(whole.body).await, (mp3.clone(), None)); + // A range past the start: the file's head is read with its own request. + let before = server.requests("/episode").len(); + let part = proxy + .fetch(&server.url("/episode"), Some("bytes=100-199")) + .await + .unwrap(); + assert_eq!(part.content_type.as_deref(), Some("audio/mpeg")); + assert_eq!(collect(part.body).await, (mp3[100..200].to_vec(), None)); + let ranges: Vec = server.requests("/episode")[before..] + .iter() + .map(|request| request["range"].clone()) + .collect(); + assert_eq!(ranges, ["bytes=100-199", "bytes=0-15"]); assert_eq!( content_type("/binary.m4a", None).await.unwrap(), "audio/mp4" @@ -258,11 +280,40 @@ async fn proxy_only_serves_audio() { Err(ProxyError::Status(500)) )); assert!(matches!( - proxy.fetch("not a url", None, None).await, + proxy.fetch("not a url", None).await, Err(ProxyError::InvalidUrl(_)) )); } +#[test] +fn audio_formats_are_recognized_by_their_first_bytes() { + for (head, format) in [ + (&b"ID3\x04\0\0\0\0"[..], "audio/mpeg"), + (&[0xFF, 0xFB, 0x90, 0x64][..], "audio/mpeg"), + (&[0xFF, 0xF3, 0x48, 0xC4][..], "audio/mpeg"), + (&[0xFF, 0xF1, 0x50, 0x80][..], "audio/aac"), + (&b"OggS\0\x02"[..], "audio/ogg"), + (&b"fLaC\0\0\0\x22"[..], "audio/flac"), + (&b"RIFF\x24\x08\0\0WAVEfmt "[..], "audio/wav"), + (&b"\0\0\0\x20ftypM4A \0\0\0\0"[..], "audio/mp4"), + ] { + assert_eq!(sniff_audio(head), Some(format), "{head:?}"); + } + for head in [ + &b"\0\0\0\x20ftypisom\0\0\x02\0"[..], + &b"\0\0\0\x18ftypmp42"[..], + &b"PK\x03\x04"[..], + &b""[..], + &b"%PDF-1.7"[..], + &b"RIFF\x24\x08\0\0AVI "[..], + &[0xFF, 0xD8, 0xFF, 0xE0][..], + &[0xFF][..], + &[][..], + ] { + assert_eq!(sniff_audio(head), None, "{head:?}"); + } +} + #[tokio::test] async fn proxy_limits_size_and_reports_dropped_streams() { let server = TestServer::start(); @@ -278,18 +329,18 @@ async fn proxy_limits_size_and_reports_dropped_streams() { ); let limited = proxy(ScriptedLookup::with(&[])).with_limits(60_000, Duration::from_secs(60)); assert!(matches!( - limited.fetch(&server.url("/sized.mp3"), None, None).await, + limited.fetch(&server.url("/sized.mp3"), None).await, Err(ProxyError::TooLarge(100_000)) )); // A range below the limit is fine. let part = limited - .fetch(&server.url("/sized.mp3"), None, Some("bytes=0-49999")) + .fetch(&server.url("/sized.mp3"), Some("bytes=0-49999")) .await .unwrap(); assert_eq!(collect(part.body).await, (audio[..50_000].to_vec(), None)); // Without a declared length the stream stops at the limit. let chunked = limited - .fetch(&server.url("/chunked.mp3"), None, None) + .fetch(&server.url("/chunked.mp3"), None) .await .unwrap(); let (received, error) = collect(chunked.body).await; @@ -298,7 +349,7 @@ async fn proxy_limits_size_and_reports_dropped_streams() { // A connection dropped mid-body ends the stream with an error instead // of a silently truncated success. let cut = proxy(ScriptedLookup::with(&[])) - .fetch(&server.url("/cut.mp3"), None, None) + .fetch(&server.url("/cut.mp3"), None) .await .unwrap(); let (received, error) = collect(cut.body).await; @@ -314,10 +365,7 @@ async fn proxy_stream_has_a_time_limit() { Route::new(noise(10, 1), "audio/mpeg").delay_body(Duration::from_secs(3)), ); let limited = proxy(ScriptedLookup::with(&[])).with_limits(1 << 20, Duration::from_millis(300)); - let audio = limited - .fetch(&server.url("/slow.mp3"), None, None) - .await - .unwrap(); + let audio = limited.fetch(&server.url("/slow.mp3"), None).await.unwrap(); let (_, error) = collect(audio.body).await; assert!(error.unwrap().contains("time limit")); }