From 39d75b07f6c4f022f40f1916ed7741347346a5a9 Mon Sep 17 00:00:00 2001 From: Ultradesu Date: Tue, 1 Sep 2026 12:47:01 +0100 Subject: [PATCH] Added yt-dlp cookies --- src/admin/mod.rs | 45 ++++ src/admin/v2.rs | 186 +++++++++++++++- src/config.rs | 19 +- src/metrics.rs | 2 + src/player/mod.rs | 37 +++- src/youtube.rs | 458 ++++++++++++++++++++++++++++++++++++++-- templates/admin/v2.html | 187 +++++++++++++++- 7 files changed, 903 insertions(+), 31 deletions(-) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 7dd7c31..1284539 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -81,6 +81,11 @@ struct PathId { id: i64, } +#[derive(Debug, Deserialize)] +struct PathStringId { + id: String, +} + #[derive(Debug, Deserialize)] struct PathName { name: String, @@ -415,6 +420,43 @@ impl App for AdminApp { }), "admin_v2_settings_probe", ), + Route::with_handler_and_name( + "/v2/api/settings/youtube-cookies", + { + let pool = Arc::clone(&pool); + let pool_config = Arc::clone(&pool_config); + cot::router::method::post( + move |session: Session, + db: Database, + json: Json| { + let pool = Arc::clone(&pool); + let pool_config = Arc::clone(&pool_config); + async move { + let pg_pool = pool + .get_or_init(|| async { + sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .connect(&pool_config.database_url) + .await + .expect("admin pool") + }) + .await; + v2::upload_youtube_cookie_file(session, db, pg_pool, json).await + } + }, + ) + }, + "admin_v2_youtube_cookie_upload", + ), + Route::with_handler_and_name( + "/v2/api/settings/youtube-cookies/{id}", + cot::router::method::delete( + move |session: Session, db: Database, path: Path| async move { + v2::delete_youtube_cookie_file(session, db, &path.0.id).await + }, + ), + "admin_v2_youtube_cookie_delete", + ), Route::with_handler_and_name( "/v2/api/federation", get(move |session: Session, db: Database| async move { @@ -1442,6 +1484,9 @@ impl App for AdminApp { all.extend(cot::db::migrations::wrap_migrations( crate::auth::db_migrations::MIGRATIONS, )); + all.extend(cot::db::migrations::wrap_migrations( + crate::youtube::db_migrations::MIGRATIONS, + )); all } } diff --git a/src/admin/v2.rs b/src/admin/v2.rs index 432e0f5..c9e8b5f 100644 --- a/src/admin/v2.rs +++ b/src/admin/v2.rs @@ -16,7 +16,7 @@ use sqlx::{PgPool, Postgres, QueryBuilder}; use super::BUILD_INFO; use crate::agent; use crate::auth::{self, AuthenticatedUser, Role}; -use crate::config::{AppConfig, ConfigEntry, ConfigSources, DownloadProxy}; +use crate::config::{AppConfig, ConfigEntry, ConfigSource, ConfigSources, DownloadProxy}; use crate::i18n::{I18n, Translations}; use crate::scheduler::{self, JobRegistry, JobRun, ScheduledJob}; @@ -165,6 +165,12 @@ pub(super) struct UploadLibraryImageRequest { mime_type: String, } +#[derive(Debug, Deserialize)] +pub(super) struct UploadYoutubeCookieFileRequest { + filename: String, + data: String, +} + #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] struct ReviewFilter { status: Option, @@ -444,11 +450,42 @@ struct MergeReleasesResponse { struct AdminSettingsDto { values: AdminSettingsValues, sources: AdminSettingsSources, + youtube_cookie_files: Vec, lastfm_api_key_configured: bool, lastfm_shared_secret_configured: bool, lastfm_scrobbling_configured: bool, } +#[derive(Debug, Clone, Serialize, JsonSchema)] +struct AdminYoutubeCookieFileDto { + id: String, + filename: String, + cookie_count: u64, + uploaded_at: String, +} + +impl From for AdminYoutubeCookieFileDto { + fn from(file: crate::youtube::YoutubeCookieFile) -> Self { + Self { + id: file.id_str().to_owned(), + filename: file.filename().to_owned(), + cookie_count: file.cookie_count(), + uploaded_at: file.uploaded_at().to_owned(), + } + } +} + +impl From for AdminYoutubeCookieFileDto { + fn from(file: crate::youtube::YoutubeCookieFileMetadata) -> Self { + Self { + id: file.id_str().to_owned(), + filename: file.filename().to_owned(), + cookie_count: file.cookie_count(), + uploaded_at: file.uploaded_at().to_owned(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] struct AdminSettingsValues { auth_password_enabled: bool, @@ -497,6 +534,8 @@ struct AdminSettingsValues { torrent_proxy_id: String, #[serde(default)] youtube_proxy_id: String, + #[serde(default)] + youtube_cookie_id: String, } #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] @@ -566,6 +605,7 @@ struct AdminSettingsSources { download_proxies: &'static str, torrent_proxy_id: &'static str, youtube_proxy_id: &'static str, + youtube_cookie_id: &'static str, } #[derive(Debug, Deserialize)] @@ -616,6 +656,8 @@ pub(super) struct UpdateSettingsRequest { torrent_proxy_id: String, #[serde(default)] youtube_proxy_id: String, + #[serde(default)] + youtube_cookie_id: String, } fn default_similarity_model() -> String { @@ -1055,7 +1097,10 @@ pub async fn settings(session: Session, db: Database) -> cot::Result, +) -> cot::Result { + if let Err(response) = require_admin_json(&session, &db).await { + return Ok(response); + } + let saved_count = match crate::youtube::cookie_file_count(&db).await { + Ok(count) => count, + Err(error) => { + tracing::error!(%error, "could not count saved YouTube cookie files"); + return Ok(json_error( + StatusCode::INTERNAL_SERVER_ERROR, + "YouTube cookie storage is unavailable; restart the server to apply database migrations", + )); + } + }; + if saved_count >= crate::youtube::MAX_COOKIE_FILES { + return Ok(json_error( + StatusCode::BAD_REQUEST, + "at most 32 YouTube cookie files can be saved", + )); + } + let filename = Path::new(body.filename.trim()) + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or_default() + .trim() + .chars() + .filter(|character| !character.is_control()) + .take(255) + .collect::(); + if filename.is_empty() { + return Ok(json_error( + StatusCode::BAD_REQUEST, + "cookie filename is empty", + )); + } + use base64::Engine; + let max_encoded_len = crate::youtube::MAX_COOKIE_FILE_BYTES.div_ceil(3) * 4; + if body.data.trim().len() > max_encoded_len { + return Ok(json_error( + StatusCode::BAD_REQUEST, + "cookie file is larger than 2 MiB", + )); + } + let data = match base64::engine::general_purpose::STANDARD.decode(body.data.trim()) { + Ok(data) => data, + Err(_) => { + return Ok(json_error( + StatusCode::BAD_REQUEST, + "cookie file payload is invalid", + )); + } + }; + let (contents, cookie_count) = match crate::youtube::parse_cookie_file(&data) { + Ok(parsed) => parsed, + Err(error) => return Ok(json_error(StatusCode::BAD_REQUEST, &error.to_string())), + }; + let file = + match crate::youtube::store_cookie_file(pool, &filename, cookie_count, &contents).await { + Ok(file) => file, + Err(error) => { + tracing::error!(%error, "could not save YouTube cookie file"); + return Ok(json_error( + StatusCode::INTERNAL_SERVER_ERROR, + "could not save YouTube cookie file; check the server log for details", + )); + } + }; + Json(AdminYoutubeCookieFileDto::from(file)).into_response() +} + +pub async fn delete_youtube_cookie_file( + session: Session, + db: Database, + id: &str, +) -> cot::Result { + if let Err(response) = require_admin_json(&session, &db).await { + return Ok(response); + } + let id = id.trim(); + if !crate::youtube::cookie_file_exists(&db, id) + .await + .map_err(|error| cot::Error::internal(error.to_string()))? + { + return Ok(json_error( + StatusCode::NOT_FOUND, + "YouTube cookie file not found", + )); + } + let (config, sources) = AppConfig::load_with_db(&db).await; + if config.youtube_cookie_id == id { + if sources.youtube_cookie_id == ConfigSource::Env { + return Ok(json_error( + StatusCode::BAD_REQUEST, + "this cookie file is selected by FURU_YOUTUBE_COOKIE_ID and cannot be deleted", + )); + } + let mut entry = ConfigEntry::new("youtube_cookie_id".to_owned(), String::new()); + entry + .save(&db) + .await + .map_err(|error| cot::Error::internal(error.to_string()))?; + } + crate::youtube::YoutubeCookieFile::delete_by_id(&db, id) + .await + .map_err(|error| cot::Error::internal(error.to_string()))?; + Json(serde_json::json!({ "ok": true })).into_response() +} + // --------------------------------------------------------------------------- // Federation (status + manual controls) // --------------------------------------------------------------------------- @@ -1373,7 +1543,11 @@ pub async fn settings_probe( .into_response() } -fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto { +fn settings_dto( + config: AppConfig, + sources: ConfigSources, + cookie_files: Vec, +) -> AdminSettingsDto { let download_proxies = config .parsed_download_proxies() .unwrap_or_else(|error| { @@ -1384,6 +1558,10 @@ fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto { .map(AdminDownloadProxy::from) .collect(); AdminSettingsDto { + youtube_cookie_files: cookie_files + .into_iter() + .map(AdminYoutubeCookieFileDto::from) + .collect(), lastfm_api_key_configured: !config.lastfm_api_key.trim().is_empty(), lastfm_shared_secret_configured: !config.lastfm_shared_secret.trim().is_empty(), lastfm_scrobbling_configured: !config.lastfm_api_key.trim().is_empty() @@ -1422,6 +1600,7 @@ fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto { download_proxies, torrent_proxy_id: config.torrent_proxy_id, youtube_proxy_id: config.youtube_proxy_id, + youtube_cookie_id: config.youtube_cookie_id, }, sources: AdminSettingsSources { auth_password_enabled: sources.auth_password_enabled.code(), @@ -1457,6 +1636,7 @@ fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto { download_proxies: sources.download_proxies.code(), torrent_proxy_id: sources.torrent_proxy_id.code(), youtube_proxy_id: sources.youtube_proxy_id.code(), + youtube_cookie_id: sources.youtube_cookie_id.code(), }, } } diff --git a/src/config.rs b/src/config.rs index ac93016..e554667 100644 --- a/src/config.rs +++ b/src/config.rs @@ -148,6 +148,7 @@ pub struct ConfigSources { pub download_proxies: ConfigSource, pub torrent_proxy_id: ConfigSource, pub youtube_proxy_id: ConfigSource, + pub youtube_cookie_id: ConfigSource, } impl Default for ConfigSources { @@ -188,6 +189,7 @@ impl Default for ConfigSources { download_proxies: ConfigSource::Default, torrent_proxy_id: ConfigSource::Default, youtube_proxy_id: ConfigSource::Default, + youtube_cookie_id: ConfigSource::Default, } } } @@ -403,6 +405,8 @@ pub struct AppConfig { pub torrent_proxy_id: String, /// Saved proxy id used for YouTube downloads; empty means a direct connection. pub youtube_proxy_id: String, + /// Saved cookie-file id used by yt-dlp; empty means no cookies. + pub youtube_cookie_id: String, } impl Default for AppConfig { @@ -446,6 +450,7 @@ impl Default for AppConfig { download_proxies: "[]".into(), torrent_proxy_id: String::new(), youtube_proxy_id: String::new(), + youtube_cookie_id: String::new(), } } } @@ -487,6 +492,7 @@ impl_env_overrides!( download_proxies, torrent_proxy_id, youtube_proxy_id, + youtube_cookie_id, ); impl AppConfig { @@ -581,11 +587,13 @@ impl AppConfig { sources.$field = ConfigSource::Database; } Err(_) => { - tracing::warn!( - "ignoring invalid DB config value for {}: {:?}", - stringify!($field), - val, - ); + if !val.trim().is_empty() { + tracing::warn!( + "ignoring invalid DB config value for {}: {:?}", + stringify!($field), + val, + ); + } } } } @@ -627,6 +635,7 @@ impl AppConfig { apply_db_field!(download_proxies); apply_db_field!(torrent_proxy_id); apply_db_field!(youtube_proxy_id); + apply_db_field!(youtube_cookie_id); } pub fn parsed_download_proxies(&self) -> anyhow::Result> { diff --git a/src/metrics.rs b/src/metrics.rs index 6c36cc9..407f40e 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -832,6 +832,8 @@ const KNOWN_HTTP_ROUTES: &[&str] = &[ "/admin/v2/api/jobs/{name}/run", "/admin/v2/api/settings", "/admin/v2/api/settings/probe", + "/admin/v2/api/settings/youtube-cookies", + "/admin/v2/api/settings/youtube-cookies/{id}", "/admin/v2/api/jobs/{name}/toggle", "/admin/v2/api/jobs/{name}/runs", "/admin/v2/api/jobs/{name}/runs/{run_id}", diff --git a/src/player/mod.rs b/src/player/mod.rs index 22ef7ea..523f7ea 100644 --- a/src/player/mod.rs +++ b/src/player/mod.rs @@ -93,6 +93,15 @@ fn download_proxy_for( result.map_err(|error| json_error(StatusCode::BAD_REQUEST, &error.to_string())) } +async fn youtube_cookie_contents( + config: &AppConfig, + db: &Database, +) -> Result, cot::response::Response> { + crate::youtube::selected_cookie_contents(db, &config.youtube_cookie_id) + .await + .map_err(|error| json_error(StatusCode::BAD_REQUEST, &error.to_string())) +} + #[derive(serde::Serialize)] struct LocalUploadResponse { ok: bool, @@ -8542,12 +8551,23 @@ impl App for PlayerApp { Ok(proxy_url) => proxy_url, Err(response) => return Ok(response), }; + let cookie_contents = match youtube_cookie_contents(&live_config, &db).await { + Ok(cookie_contents) => cookie_contents, + Err(response) => return Ok(response), + }; let service = youtube_service .get_or_init(|| async { Arc::new(YouTubeService::new(Arc::clone(&scheduler_handle))) }) .await; - match service.preview(json.0, proxy_url.as_deref()).await { + match service + .preview( + json.0, + proxy_url.as_deref(), + cookie_contents.as_deref(), + ) + .await + { Ok(preview) => Json(preview).into_response(), Err(err) => { Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string())) @@ -8603,12 +8623,17 @@ impl App for PlayerApp { Ok(proxy_url) => proxy_url, Err(response) => return Ok(response), }; + let cookie_contents = match youtube_cookie_contents(&live_config, &db).await { + Ok(cookie_contents) => cookie_contents, + Err(response) => return Ok(response), + }; match service .list( pg_pool, user.id, &live_config.agent_inbox_dir, proxy_url, + cookie_contents, ) .await { @@ -8670,6 +8695,10 @@ impl App for PlayerApp { Ok(proxy_url) => proxy_url, Err(response) => return Ok(response), }; + let cookie_contents = match youtube_cookie_contents(&live_config, &db).await { + Ok(cookie_contents) => cookie_contents, + Err(response) => return Ok(response), + }; match service .start( pg_pool, @@ -8677,6 +8706,7 @@ impl App for PlayerApp { json.0, &live_config.agent_inbox_dir, proxy_url, + cookie_contents, ) .await { @@ -8738,6 +8768,10 @@ impl App for PlayerApp { Ok(proxy_url) => proxy_url, Err(response) => return Ok(response), }; + let cookie_contents = match youtube_cookie_contents(&live_config, &db).await { + Ok(cookie_contents) => cookie_contents, + Err(response) => return Ok(response), + }; match service .retry( pg_pool, @@ -8745,6 +8779,7 @@ impl App for PlayerApp { &path.0.id, &live_config.agent_inbox_dir, proxy_url, + cookie_contents, ) .await { diff --git a/src/youtube.rs b/src/youtube.rs index 40b5dec..b603c18 100644 --- a/src/youtube.rs +++ b/src/youtube.rs @@ -1,10 +1,13 @@ use std::collections::{HashMap, HashSet, VecDeque}; +use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::Arc; use std::time::Duration; use anyhow::{Context, bail}; +use cot::db::migrations::{self, Operation, SyncDynMigration}; +use cot::db::{Database, LimitedString, Model}; use image::codecs::jpeg::JpegEncoder; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -22,11 +25,216 @@ const RESOLVE_TIMEOUT: Duration = Duration::from_secs(180); const HTTP_403_RETRY_DELAY: Duration = Duration::from_secs(30); const HTTP_403_MAX_RETRIES: usize = 3; const MAX_ERROR_LEN: usize = 4_000; +pub const MAX_COOKIE_FILE_BYTES: usize = 2 * 1024 * 1024; +pub const MAX_COOKIE_FILES: u64 = 32; const AUDIO_EXTENSIONS: &[&str] = &[ "mp3", "flac", "ogg", "opus", "aac", "m4a", "wav", "ape", "wv", "wma", "tta", "aiff", "aif", ]; const IMAGE_EXTENSIONS: &[&str] = &["jpg", "jpeg", "png", "webp", "bmp", "gif"]; +#[derive(Debug, Clone)] +#[cot::db::model(table_name = "youtube_cookie_file")] +pub struct YoutubeCookieFile { + #[model(primary_key)] + id: String, + filename: String, + cookie_count: i64, + uploaded_at: LimitedString<32>, + contents: String, +} + +#[derive(Debug, Clone)] +#[cot::db::model(table_name = "youtube_cookie_file")] +pub struct YoutubeCookieFileMetadata { + #[model(primary_key)] + id: String, + filename: String, + cookie_count: i64, + uploaded_at: LimitedString<32>, +} + +impl YoutubeCookieFile { + pub async fn get_by_id(db: &Database, id: &str) -> cot::db::Result> { + if id.len() > 36 { + return Ok(None); + } + let id = id.to_owned(); + cot::db::query!(YoutubeCookieFile, $id == id).get(db).await + } + + pub async fn delete_by_id(db: &Database, id: &str) -> cot::db::Result<()> { + if id.len() > 36 { + return Ok(()); + } + let id = id.to_owned(); + cot::db::query!(YoutubeCookieFile, $id == id) + .delete(db) + .await?; + Ok(()) + } + + pub fn id_str(&self) -> &str { + &self.id + } + + pub fn filename(&self) -> &str { + &self.filename + } + + pub fn cookie_count(&self) -> u64 { + u64::try_from(self.cookie_count).unwrap_or(0) + } + + pub fn uploaded_at(&self) -> &str { + &self.uploaded_at + } + + fn contents(&self) -> &str { + &self.contents + } +} + +pub async fn store_cookie_file( + pool: &PgPool, + filename: &str, + cookie_count: usize, + contents: &str, +) -> anyhow::Result { + let id = Uuid::new_v4().to_string(); + let uploaded_at = now_string(); + let cookie_count = i64::try_from(cookie_count).unwrap_or(i64::MAX); + sqlx::query( + r#"INSERT INTO furumusic__youtube_cookie_file + (id, filename, cookie_count, uploaded_at, contents) + VALUES ($1, $2, $3, $4, $5)"#, + ) + .bind(&id) + .bind(filename) + .bind(cookie_count) + .bind(&uploaded_at) + .bind(contents) + .execute(pool) + .await + .context("could not insert YouTube cookie file")?; + Ok(YoutubeCookieFileMetadata { + id, + filename: filename.to_owned(), + cookie_count, + uploaded_at: LimitedString::new(uploaded_at).expect("timestamp fits cookie metadata"), + }) +} + +impl YoutubeCookieFileMetadata { + pub async fn get_by_id(db: &Database, id: &str) -> cot::db::Result> { + if id.len() > 36 { + return Ok(None); + } + let id = id.to_owned(); + cot::db::query!(YoutubeCookieFileMetadata, $id == id) + .get(db) + .await + } + + pub fn id_str(&self) -> &str { + &self.id + } + + pub fn filename(&self) -> &str { + &self.filename + } + + pub fn cookie_count(&self) -> u64 { + u64::try_from(self.cookie_count).unwrap_or(0) + } + + pub fn uploaded_at(&self) -> &str { + &self.uploaded_at + } +} + +pub async fn cookie_file_count(db: &Database) -> cot::db::Result { + YoutubeCookieFile::objects().count(db).await +} + +pub async fn cookie_file_exists(db: &Database, id: &str) -> cot::db::Result { + Ok(YoutubeCookieFileMetadata::get_by_id(db, id) + .await? + .is_some()) +} + +pub async fn list_cookie_files(db: &Database) -> cot::db::Result> { + let mut files = YoutubeCookieFileMetadata::objects().all(db).await?; + files.sort_by(|left, right| { + right + .uploaded_at() + .cmp(left.uploaded_at()) + .then_with(|| left.filename().cmp(right.filename())) + }); + Ok(files) +} + +pub async fn selected_cookie_contents( + db: &Database, + cookie_file_id: &str, +) -> anyhow::Result> { + let cookie_file_id = cookie_file_id.trim(); + if cookie_file_id.is_empty() { + return Ok(None); + } + let file = YoutubeCookieFile::get_by_id(db, cookie_file_id) + .await? + .context("selected YouTube cookie file no longer exists")?; + Ok(Some(file.contents().to_owned())) +} + +pub fn parse_cookie_file(data: &[u8]) -> anyhow::Result<(String, usize)> { + if data.is_empty() { + bail!("cookie file is empty"); + } + if data.len() > MAX_COOKIE_FILE_BYTES { + bail!("cookie file is larger than 2 MiB"); + } + let text = std::str::from_utf8(data).context("cookie file must be UTF-8 text")?; + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let mut normalized = Vec::new(); + + for (index, raw_line) in text.lines().enumerate() { + let line = raw_line.trim_end_matches('\r'); + if line.trim().is_empty() || (line.starts_with('#') && !line.starts_with("#HttpOnly_")) { + continue; + } + let fields = line.splitn(7, '\t').collect::>(); + if fields.len() != 7 { + bail!( + "invalid Netscape cookie on line {}: expected 7 tab-separated fields", + index + 1 + ); + } + let domain = fields[0].strip_prefix("#HttpOnly_").unwrap_or(fields[0]); + if domain.is_empty() + || domain.chars().any(char::is_whitespace) + || !matches!(fields[1], "TRUE" | "FALSE") + || fields[2].is_empty() + || !matches!(fields[3], "TRUE" | "FALSE") + || fields[4].parse::().is_err() + || fields[5].is_empty() + { + bail!("invalid Netscape cookie on line {}", index + 1); + } + normalized.push(fields.join("\t")); + } + + if normalized.is_empty() { + bail!("cookie file contains no Netscape-format cookies"); + } + let cookie_count = normalized.len(); + let contents = format!( + "# Netscape HTTP Cookie File\n# Generated by furumusic\n\n{}\n", + normalized.join("\n") + ); + Ok((contents, cookie_count)) +} + #[derive(Debug, Deserialize)] pub struct YouTubePreviewRequest { pub url: String, @@ -216,6 +424,52 @@ impl std::fmt::Display for YtDlpDownloadFailure { impl std::error::Error for YtDlpDownloadFailure {} +struct TemporaryCookieFile { + path: PathBuf, +} + +impl TemporaryCookieFile { + fn create(contents: &str) -> anyhow::Result { + let path = + std::env::temp_dir().join(format!("furumusic-ytdlp-cookies-{}.txt", Uuid::new_v4())); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(&path) + .context("could not create temporary yt-dlp cookie file")?; + if let Err(error) = file.write_all(contents.as_bytes()) { + let _ = std::fs::remove_file(&path); + return Err(error).context("could not write temporary yt-dlp cookie file"); + } + Ok(Self { path }) + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TemporaryCookieFile { + fn drop(&mut self) { + if let Err(error) = std::fs::remove_file(&self.path) + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!(path = %self.path.display(), %error, "could not remove temporary yt-dlp cookie file"); + } + } +} + +#[derive(Clone, Copy)] +struct YtDlpOptions<'a> { + proxy_url: Option<&'a str>, + cookie_contents: Option<&'a str>, +} + pub struct YouTubeService { running_jobs: Mutex>, cancellations: Mutex>, @@ -237,9 +491,10 @@ impl YouTubeService { &self, request: YouTubePreviewRequest, proxy_url: Option<&str>, + cookie_contents: Option<&str>, ) -> anyhow::Result { let url = validate_youtube_url(&request.url)?; - let resolved = resolve_source(&url, proxy_url).await?; + let resolved = resolve_source(&url, proxy_url, cookie_contents).await?; let requested_video_id = requested_video_id(&url); let select_requested_only = resolved.kind == "playlist" && requested_video_id.is_some(); Ok(YouTubePreviewDto { @@ -270,6 +525,7 @@ impl YouTubeService { request: YouTubeStartRequest, inbox_dir: &str, proxy_url: Option, + cookie_contents: Option, ) -> anyhow::Result { let url = validate_youtube_url(&request.url)?; validate_inbox_dir(inbox_dir)?; @@ -286,7 +542,8 @@ impl YouTubeService { bail!("YouTube selection contains an invalid video ID"); } - let resolved = resolve_source(&url, proxy_url.as_deref()).await?; + let resolved = + resolve_source(&url, proxy_url.as_deref(), cookie_contents.as_deref()).await?; let selected_items: Vec = resolved .items .into_iter() @@ -357,8 +614,14 @@ impl YouTubeService { } transaction.commit().await?; - self.spawn_job(pool.clone(), id.clone(), inbox_dir.to_string(), proxy_url) - .await; + self.spawn_job( + pool.clone(), + id.clone(), + inbox_dir.to_string(), + proxy_url, + cookie_contents, + ) + .await; load_job_dto(pool, user_id, &id).await } @@ -368,6 +631,7 @@ impl YouTubeService { user_id: i64, inbox_dir: &str, proxy_url: Option, + cookie_contents: Option, ) -> anyhow::Result> { validate_inbox_dir(inbox_dir)?; sync_ai_statuses(pool, user_id).await?; @@ -383,8 +647,14 @@ impl YouTubeService { .fetch_all(pool) .await?; for (id, _) in resumable { - self.spawn_job(pool.clone(), id, inbox_dir.to_string(), proxy_url.clone()) - .await; + self.spawn_job( + pool.clone(), + id, + inbox_dir.to_string(), + proxy_url.clone(), + cookie_contents.clone(), + ) + .await; } let ids: Vec = sqlx::query_scalar( @@ -413,6 +683,7 @@ impl YouTubeService { id: &str, inbox_dir: &str, proxy_url: Option, + cookie_contents: Option, ) -> anyhow::Result { validate_inbox_dir(inbox_dir)?; let job = load_job_row(pool, user_id, id).await?; @@ -448,6 +719,7 @@ impl YouTubeService { id.to_string(), inbox_dir.to_string(), proxy_url, + cookie_contents, ) .await; load_job_dto(pool, user_id, id).await @@ -538,6 +810,7 @@ impl YouTubeService { id: String, inbox_dir: String, proxy_url: Option, + cookie_contents: Option, ) { { let mut running = self.running_jobs.lock().await; @@ -559,7 +832,14 @@ impl YouTubeService { }; let result = if let Some(permit) = permit { let result = service - .run_job(&pool, &id, &inbox_dir, proxy_url.as_deref(), &cancel) + .run_job( + &pool, + &id, + &inbox_dir, + proxy_url.as_deref(), + cookie_contents.as_deref(), + &cancel, + ) .await; drop(permit); result @@ -588,8 +868,13 @@ impl YouTubeService { id: &str, inbox_dir: &str, proxy_url: Option<&str>, + cookie_contents: Option<&str>, cancel: &CancellationToken, ) -> anyhow::Result<()> { + let ytdlp = YtDlpOptions { + proxy_url, + cookie_contents, + }; if cancel.is_cancelled() { return Ok(()); } @@ -610,7 +895,8 @@ impl YouTubeService { return Ok(()); } set_parent_status(pool, id, "resolving", None).await?; - let resolved = resolve_source(&job.source_url, proxy_url).await?; + let resolved = + resolve_source(&job.source_url, ytdlp.proxy_url, ytdlp.cookie_contents).await?; if cancel.is_cancelled() { return Ok(()); } @@ -677,7 +963,7 @@ impl YouTubeService { continue; } if let Err(err) = self - .process_item(pool, &job, &item, &inbox_root, proxy_url, cancel) + .process_item(pool, &job, &item, &inbox_root, ytdlp, cancel) .await { if cancel.is_cancelled() { @@ -707,7 +993,7 @@ impl YouTubeService { job: &YouTubeJobRow, item: &YouTubeItemRow, inbox_root: &Path, - proxy_url: Option<&str>, + ytdlp: YtDlpOptions<'_>, cancel: &CancellationToken, ) -> anyhow::Result<()> { if cancel.is_cancelled() { @@ -720,8 +1006,16 @@ impl YouTubeService { tokio::fs::create_dir_all(&stage).await?; let mut forbidden_retries = 0; loop { - match run_ytdlp_download(pool, &item.id, &item.source_url, &stage, proxy_url, cancel) - .await + match run_ytdlp_download( + pool, + &item.id, + &item.source_url, + &stage, + ytdlp.proxy_url, + ytdlp.cookie_contents, + cancel, + ) + .await { Ok(()) => break, Err(error) @@ -815,8 +1109,15 @@ impl YouTubeService { } } -async fn resolve_source(url: &str, proxy_url: Option<&str>) -> anyhow::Result { - let mut command = base_ytdlp_command(proxy_url); +async fn resolve_source( + url: &str, + proxy_url: Option<&str>, + cookie_contents: Option<&str>, +) -> anyhow::Result { + let cookie_file = cookie_contents + .map(TemporaryCookieFile::create) + .transpose()?; + let mut command = base_ytdlp_command(proxy_url, cookie_file.as_ref().map(|file| file.path())); command .arg("--flat-playlist") .arg("--dump-single-json") @@ -887,15 +1188,19 @@ async fn resolve_source(url: &str, proxy_url: Option<&str>) -> anyhow::Result) -> Command { +fn base_ytdlp_command(proxy_url: Option<&str>, cookie_file: Option<&Path>) -> Command { let mut command = Command::new("yt-dlp"); command .arg("--no-config") - .arg("--no-cookies") .arg("--no-cookies-from-browser") .arg("--js-runtimes") .arg("deno") .stdin(Stdio::null()); + if let Some(cookie_file) = cookie_file { + command.arg("--cookies").arg(cookie_file); + } else { + command.arg("--no-cookies"); + } if let Some(proxy_url) = proxy_url { command.arg("--proxy").arg(proxy_url); } @@ -908,9 +1213,13 @@ async fn run_ytdlp_download( url: &str, stage: &Path, proxy_url: Option<&str>, + cookie_contents: Option<&str>, cancel: &CancellationToken, ) -> anyhow::Result<()> { - let mut command = base_ytdlp_command(proxy_url); + let cookie_file = cookie_contents + .map(TemporaryCookieFile::create) + .transpose()?; + let mut command = base_ytdlp_command(proxy_url, cookie_file.as_ref().map(|file| file.path())); command .arg("--no-playlist") .arg("--continue") @@ -1964,10 +2273,61 @@ fn now_string() -> String { chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string() } +pub mod db_migrations { + use super::*; + + #[cot::db::migrations::migration_op] + async fn create_youtube_cookie_files( + ctx: migrations::MigrationContext<'_>, + ) -> cot::db::Result<()> { + ctx.db + .raw( + "CREATE TABLE IF NOT EXISTS furumusic__youtube_cookie_file ( + id VARCHAR(36) PRIMARY KEY, + filename TEXT NOT NULL, + cookie_count BIGINT NOT NULL, + uploaded_at VARCHAR(32) NOT NULL, + contents TEXT NOT NULL + )", + ) + .await?; + Ok(()) + } + + #[derive(Debug, Copy, Clone)] + pub struct M0048CreateYoutubeCookieFiles; + + impl migrations::Migration for M0048CreateYoutubeCookieFiles { + const APP_NAME: &'static str = "furumusic"; + const MIGRATION_NAME: &'static str = "m_0048_create_youtube_cookie_files"; + const DEPENDENCIES: &'static [migrations::MigrationDependency] = + &[migrations::MigrationDependency::migration( + "furumusic", + "m_0047_create_youtube_import_media_links", + )]; + const OPERATIONS: &'static [Operation] = + &[Operation::custom(create_youtube_cookie_files).build()]; + } + + pub const MIGRATIONS: &[&SyncDynMigration] = &[&M0048CreateYoutubeCookieFiles]; +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn cookie_models_use_exact_migrated_table_name() { + assert_eq!( + ::TABLE_NAME.as_str(), + "furumusic__youtube_cookie_file" + ); + assert_eq!( + ::TABLE_NAME.as_str(), + "furumusic__youtube_cookie_file" + ); + } + #[test] fn validates_supported_youtube_hosts() { assert!(validate_youtube_url("https://youtube.com/watch?v=abc").is_ok()); @@ -2049,7 +2409,7 @@ mod tests { #[test] fn ytdlp_command_receives_selected_proxy() { - let command = base_ytdlp_command(Some("socks5://user:pass@127.0.0.1:1080/")); + let command = base_ytdlp_command(Some("socks5://user:pass@127.0.0.1:1080/"), None); let args: Vec = command .as_std() .get_args() @@ -2062,13 +2422,73 @@ mod tests { ] })); - let direct = base_ytdlp_command(None); + let direct = base_ytdlp_command(None, None); assert!( direct .as_std() .get_args() .all(|argument| argument != "--proxy") ); + assert!( + direct + .as_std() + .get_args() + .any(|argument| argument == "--no-cookies") + ); + } + + #[test] + fn parses_and_normalizes_netscape_cookie_files() { + let source = b"# Netscape HTTP Cookie File\r\n\r\n.youtube.com\tTRUE\t/\tTRUE\t1893456000\tSID\tsecret\r\n#HttpOnly_.youtube.com\tTRUE\t/\tTRUE\t0\tLOGIN_INFO\tvalue\r\n"; + let (contents, count) = parse_cookie_file(source).unwrap(); + assert_eq!(count, 2); + assert!(contents.starts_with("# Netscape HTTP Cookie File\n")); + assert!(contents.contains("#HttpOnly_.youtube.com\tTRUE")); + assert!(!contents.contains('\r')); + } + + #[test] + fn rejects_non_netscape_cookie_files() { + assert!(parse_cookie_file(br#"[{"name":"SID","value":"secret"}]"#).is_err()); + assert!(parse_cookie_file(b"# Netscape HTTP Cookie File\n").is_err()); + assert!(parse_cookie_file(b"youtube.com TRUE / TRUE 0 SID secret\n").is_err()); + } + + #[test] + fn ytdlp_command_receives_cookie_file_when_selected() { + let path = Path::new("/tmp/furumusic-test-cookies.txt"); + let command = base_ytdlp_command(None, Some(path)); + let args = command + .as_std() + .get_args() + .map(|argument| argument.to_string_lossy().into_owned()) + .collect::>(); + assert!(args.windows(2).any(|pair| { + pair == ["--cookies".to_string(), path.to_string_lossy().into_owned()] + })); + assert!(!args.iter().any(|argument| argument == "--no-cookies")); + } + + #[test] + fn temporary_cookie_file_is_private_and_removed() { + let path; + { + let file = TemporaryCookieFile::create( + "# Netscape HTTP Cookie File\n.youtube.com\tTRUE\t/\tTRUE\t0\tSID\tsecret\n", + ) + .unwrap(); + path = file.path().to_owned(); + assert_eq!(std::fs::read_to_string(&path).unwrap().lines().count(), 2); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + } + assert!(!path.exists()); } #[test] diff --git a/templates/admin/v2.html b/templates/admin/v2.html index 1b331a2..3cb466a 100644 --- a/templates/admin/v2.html +++ b/templates/admin/v2.html @@ -959,6 +959,89 @@ tbody tr:hover { outline: none; } +.youtube-cookie-settings { + display: grid; + gap: 9px; + margin-top: 11px; + padding-top: 11px; + border-top: 1px solid var(--border-color); +} + +.youtube-cookie-picker, +.youtube-cookie-upload, +.youtube-cookie-row { + display: flex; + align-items: center; + gap: 9px; +} + +.youtube-cookie-picker > span { + flex: 0 0 auto; + color: var(--text-subdued); + font-size: 11px; + font-weight: 750; +} + +.youtube-cookie-picker select { + min-width: 0; + flex: 1; + height: 34px; + padding: 0 10px; + border: 1px solid var(--border-color); + border-radius: 6px; + background: var(--bg-primary); + color: var(--text-primary); + outline: none; +} + +.youtube-cookie-upload { + justify-content: space-between; +} + +.youtube-cookie-upload span, +.youtube-cookie-empty { + color: var(--text-subdued); + font-size: 11px; +} + +.youtube-cookie-list { + display: grid; + gap: 6px; +} + +.youtube-cookie-row { + justify-content: space-between; + min-width: 0; + padding: 8px 9px; + border: 1px solid var(--border-color); + border-radius: 6px; + background: var(--bg-primary); +} + +.youtube-cookie-row.active { + border-color: rgba(29, 185, 84, 0.55); + background: rgba(29, 185, 84, 0.08); +} + +.youtube-cookie-info { + display: grid; + min-width: 0; + gap: 3px; +} + +.youtube-cookie-info strong { + overflow: hidden; + color: var(--text-primary); + font-size: 11px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.youtube-cookie-info span { + color: var(--text-subdued); + font-size: 10px; +} + .proxy-editor { grid-column: 1 / -1; min-width: 0; @@ -1148,6 +1231,8 @@ tbody tr:hover { .setting-field { max-width: none; } .download-method-controls, .proxy-row { grid-template-columns: 1fr; } + .youtube-cookie-picker, + .youtube-cookie-upload { align-items: stretch; flex-direction: column; } } .settings-note { @@ -2725,7 +2810,43 @@ tbody tr:hover { -
The selected proxy is passed to metadata lookup and downloads through yt-dlp's --proxy option.
+
+
+ + yt-dlp cookies + + + +
+ +
+ Netscape cookies.txt; maximum size 2 MiB. + +
+
No cookie files saved. yt-dlp will run without cookies.
+
+ +
+
+
The selected proxy and cookie file are passed to both metadata lookup and downloads through yt-dlp.
@@ -3718,7 +3839,7 @@ function adminV2() { mergeDetails: [], mergeArtistSearch: '', mergeDraft: { release_ids: [], target_release_id: null, title: '', release_type: 'album', year: '', hidden: 'false', cover_file_id: null, artist_ids: [], tracks: [] }, - settings: { values: {}, sources: {}, lastfm_api_key_configured: false, lastfm_shared_secret_configured: false, lastfm_scrobbling_configured: false }, + settings: { values: {}, sources: {}, youtube_cookie_files: [], lastfm_api_key_configured: false, lastfm_shared_secret_configured: false, lastfm_scrobbling_configured: false }, settingsDraft: { auth_password_enabled: false, auth_sso_enabled: false, @@ -3752,7 +3873,8 @@ function adminV2() { youtube_downloads_enabled: true, download_proxies: [], torrent_proxy_id: '', - youtube_proxy_id: '' + youtube_proxy_id: '', + youtube_cookie_id: '' }, settingsProbe: { status: 'idle', ok: false }, settingsProbeLoading: false, @@ -3763,6 +3885,7 @@ function adminV2() { similarityStatus: { status: { phase: 'disabled' }, models: [], profiles: [] }, similarityLoading: false, settingsSaving: false, + youtubeCookieUploading: false, routeReady: false, poller: null, @@ -4070,6 +4193,64 @@ function adminV2() { return address || 'New proxy'; }, + youtubeCookieLabel(file) { + const filename = String(file?.filename || '').trim() || 'Cookie file'; + return `${filename} (${this.fmt(file?.cookie_count)} cookies)`; + }, + + async uploadYoutubeCookieFile(event) { + const input = event?.target; + const file = input?.files && input.files.length ? input.files[0] : null; + if (!file || this.youtubeCookieUploading) return; + if (file.size > 2 * 1024 * 1024) { + this.showToast('Cookie file is larger than 2 MiB'); + input.value = ''; + return; + } + this.youtubeCookieUploading = true; + try { + const dataUrl = await this.readFileAsDataUrl(file); + const data = String(dataUrl).split(',')[1] || ''; + const uploaded = await this.request(`${this.apiBase}/settings/youtube-cookies`, { + method: 'POST', + body: JSON.stringify({ filename: file.name, data }) + }); + this.settings.youtube_cookie_files = [ + uploaded, + ...(this.settings.youtube_cookie_files || []).filter(item => item.id !== uploaded.id) + ]; + this.settingsDraft.youtube_cookie_id = uploaded.id; + this.showToast('Cookie file uploaded; save settings to activate it'); + } catch (error) { + this.showToast(error.message); + } finally { + this.youtubeCookieUploading = false; + if (input) input.value = ''; + this.icons(); + } + }, + + async deleteYoutubeCookieFile(file) { + if (!file || this.youtubeCookieUploading) return; + if (!window.confirm(`Delete cookie file "${file.filename}"? This cannot be undone.`)) return; + this.youtubeCookieUploading = true; + try { + await this.request(`${this.apiBase}/settings/youtube-cookies/${encodeURIComponent(file.id)}`, { + method: 'DELETE' + }); + this.settings.youtube_cookie_files = (this.settings.youtube_cookie_files || []) + .filter(item => item.id !== file.id); + if (this.settingsDraft.youtube_cookie_id === file.id) this.settingsDraft.youtube_cookie_id = ''; + if (this.settings.values?.youtube_cookie_id === file.id) this.settings.values.youtube_cookie_id = ''; + this.showToast('Cookie file deleted'); + } catch (error) { + this.showToast(error.message); + } finally { + this.youtubeCookieUploading = false; + this.icons(); + } + }, + async saveSettings() { if (this.settingsSaving) return; const networkSimilarityWasEnabled = Boolean(