Furumi/furumi-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Added yt-dlp cookies

Browse Source
This commit is contained in:
Ultradesu
2026-09-01 12:47:01 +01:00
parent 34e90b33f6
commit 39d75b07f6
7 changed files with 903 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/admin/mod.rs
+45
View File
@@ -81,6 +81,11 @@ struct PathId {
id: i64,
}
#[derive(Debug, Deserialize)]
struct PathStringId {
id: String,
}
#[derive(Debug, Deserialize)]
struct PathName {
name: String,
@@ -415,6 +420,43 @@ impl App for AdminApp {
}),
"admin_v2_settings_probe",
),
Route::with_handler_and_name(
"/v2/api/settings/youtube-cookies",
{
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
cot::router::method::post(
move |session: Session,
db: Database,
json: Json<v2::UploadYoutubeCookieFileRequest>| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
async move {
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("admin pool")
})
.await;
v2::upload_youtube_cookie_file(session, db, pg_pool, json).await
}
},
)
},
"admin_v2_youtube_cookie_upload",
),
Route::with_handler_and_name(
"/v2/api/settings/youtube-cookies/{id}",
cot::router::method::delete(
move |session: Session, db: Database, path: Path<PathStringId>| async move {
v2::delete_youtube_cookie_file(session, db, &path.0.id).await
},
),
"admin_v2_youtube_cookie_delete",
),
Route::with_handler_and_name(
"/v2/api/federation",
get(move |session: Session, db: Database| async move {
@@ -1442,6 +1484,9 @@ impl App for AdminApp {
all.extend(cot::db::migrations::wrap_migrations(
crate::auth::db_migrations::MIGRATIONS,
));
all.extend(cot::db::migrations::wrap_migrations(
crate::youtube::db_migrations::MIGRATIONS,
));
all
}
}
src/admin/v2.rs
+183 -3
View File
@@ -16,7 +16,7 @@ use sqlx::{PgPool, Postgres, QueryBuilder};
use super::BUILD_INFO;
use crate::agent;
use crate::auth::{self, AuthenticatedUser, Role};
use crate::config::{AppConfig, ConfigEntry, ConfigSources, DownloadProxy};
use crate::config::{AppConfig, ConfigEntry, ConfigSource, ConfigSources, DownloadProxy};
use crate::i18n::{I18n, Translations};
use crate::scheduler::{self, JobRegistry, JobRun, ScheduledJob};
@@ -165,6 +165,12 @@ pub(super) struct UploadLibraryImageRequest {
mime_type: String,
}
#[derive(Debug, Deserialize)]
pub(super) struct UploadYoutubeCookieFileRequest {
filename: String,
data: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct ReviewFilter {
status: Option<String>,
@@ -444,11 +450,42 @@ struct MergeReleasesResponse {
struct AdminSettingsDto {
values: AdminSettingsValues,
sources: AdminSettingsSources,
youtube_cookie_files: Vec<AdminYoutubeCookieFileDto>,
lastfm_api_key_configured: bool,
lastfm_shared_secret_configured: bool,
lastfm_scrobbling_configured: bool,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct AdminYoutubeCookieFileDto {
id: String,
filename: String,
cookie_count: u64,
uploaded_at: String,
}
impl From<crate::youtube::YoutubeCookieFile> for AdminYoutubeCookieFileDto {
fn from(file: crate::youtube::YoutubeCookieFile) -> Self {
Self {
id: file.id_str().to_owned(),
filename: file.filename().to_owned(),
cookie_count: file.cookie_count(),
uploaded_at: file.uploaded_at().to_owned(),
}
}
}
impl From<crate::youtube::YoutubeCookieFileMetadata> for AdminYoutubeCookieFileDto {
fn from(file: crate::youtube::YoutubeCookieFileMetadata) -> Self {
Self {
id: file.id_str().to_owned(),
filename: file.filename().to_owned(),
cookie_count: file.cookie_count(),
uploaded_at: file.uploaded_at().to_owned(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
struct AdminSettingsValues {
auth_password_enabled: bool,
@@ -497,6 +534,8 @@ struct AdminSettingsValues {
torrent_proxy_id: String,
#[serde(default)]
youtube_proxy_id: String,
#[serde(default)]
youtube_cookie_id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
@@ -566,6 +605,7 @@ struct AdminSettingsSources {
download_proxies: &'static str,
torrent_proxy_id: &'static str,
youtube_proxy_id: &'static str,
youtube_cookie_id: &'static str,
}
#[derive(Debug, Deserialize)]
@@ -616,6 +656,8 @@ pub(super) struct UpdateSettingsRequest {
torrent_proxy_id: String,
#[serde(default)]
youtube_proxy_id: String,
#[serde(default)]
youtube_cookie_id: String,
}
fn default_similarity_model() -> String {
@@ -1055,7 +1097,10 @@ pub async fn settings(session: Session, db: Database) -> cot::Result<cot::respon
return Ok(response);
}
let (config, sources) = AppConfig::load_with_db(&db).await;
Json(settings_dto(config, sources)).into_response()
let cookie_files = crate::youtube::list_cookie_files(&db)
.await
.map_err(|error| cot::Error::internal(error.to_string()))?;
Json(settings_dto(config, sources, cookie_files)).into_response()
}
pub async fn update_settings(
@@ -1117,6 +1162,7 @@ pub async fn update_settings(
}
let torrent_proxy_id = body.torrent_proxy_id.trim().to_string();
let youtube_proxy_id = body.youtube_proxy_id.trim().to_string();
let youtube_cookie_id = body.youtube_cookie_id.trim().to_string();
for (method, proxy_id) in [
("torrent", torrent_proxy_id.as_str()),
("YouTube", youtube_proxy_id.as_str()),
@@ -1128,6 +1174,16 @@ pub async fn update_settings(
));
}
}
if !youtube_cookie_id.is_empty()
&& !crate::youtube::cookie_file_exists(&db, &youtube_cookie_id)
.await
.map_err(|error| cot::Error::internal(error.to_string()))?
{
return Ok(json_error(
StatusCode::BAD_REQUEST,
"selected YouTube cookie file is not in the saved cookie file list",
));
}
let download_proxies_json = serde_json::to_string(&download_proxies)
.map_err(|error| cot::Error::internal(error.to_string()))?;
let fields = [
@@ -1200,6 +1256,7 @@ pub async fn update_settings(
("download_proxies", download_proxies_json),
("torrent_proxy_id", torrent_proxy_id),
("youtube_proxy_id", youtube_proxy_id),
("youtube_cookie_id", youtube_cookie_id),
];
for (key, value) in fields {
let mut entry = ConfigEntry::new(key.to_string(), value);
@@ -1218,6 +1275,119 @@ pub async fn update_settings(
Json(serde_json::json!({ "ok": true })).into_response()
}
pub async fn upload_youtube_cookie_file(
session: Session,
db: Database,
pool: &PgPool,
Json(body): Json<UploadYoutubeCookieFileRequest>,
) -> cot::Result<cot::response::Response> {
if let Err(response) = require_admin_json(&session, &db).await {
return Ok(response);
}
let saved_count = match crate::youtube::cookie_file_count(&db).await {
Ok(count) => count,
Err(error) => {
tracing::error!(%error, "could not count saved YouTube cookie files");
return Ok(json_error(
StatusCode::INTERNAL_SERVER_ERROR,
"YouTube cookie storage is unavailable; restart the server to apply database migrations",
));
}
};
if saved_count >= crate::youtube::MAX_COOKIE_FILES {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"at most 32 YouTube cookie files can be saved",
));
}
let filename = Path::new(body.filename.trim())
.file_name()
.and_then(|value| value.to_str())
.unwrap_or_default()
.trim()
.chars()
.filter(|character| !character.is_control())
.take(255)
.collect::<String>();
if filename.is_empty() {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"cookie filename is empty",
));
}
use base64::Engine;
let max_encoded_len = crate::youtube::MAX_COOKIE_FILE_BYTES.div_ceil(3) * 4;
if body.data.trim().len() > max_encoded_len {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"cookie file is larger than 2 MiB",
));
}
let data = match base64::engine::general_purpose::STANDARD.decode(body.data.trim()) {
Ok(data) => data,
Err(_) => {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"cookie file payload is invalid",
));
}
};
let (contents, cookie_count) = match crate::youtube::parse_cookie_file(&data) {
Ok(parsed) => parsed,
Err(error) => return Ok(json_error(StatusCode::BAD_REQUEST, &error.to_string())),
};
let file =
match crate::youtube::store_cookie_file(pool, &filename, cookie_count, &contents).await {
Ok(file) => file,
Err(error) => {
tracing::error!(%error, "could not save YouTube cookie file");
return Ok(json_error(
StatusCode::INTERNAL_SERVER_ERROR,
"could not save YouTube cookie file; check the server log for details",
));
}
};
Json(AdminYoutubeCookieFileDto::from(file)).into_response()
}
pub async fn delete_youtube_cookie_file(
session: Session,
db: Database,
id: &str,
) -> cot::Result<cot::response::Response> {
if let Err(response) = require_admin_json(&session, &db).await {
return Ok(response);
}
let id = id.trim();
if !crate::youtube::cookie_file_exists(&db, id)
.await
.map_err(|error| cot::Error::internal(error.to_string()))?
{
return Ok(json_error(
StatusCode::NOT_FOUND,
"YouTube cookie file not found",
));
}
let (config, sources) = AppConfig::load_with_db(&db).await;
if config.youtube_cookie_id == id {
if sources.youtube_cookie_id == ConfigSource::Env {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"this cookie file is selected by FURU_YOUTUBE_COOKIE_ID and cannot be deleted",
));
}
let mut entry = ConfigEntry::new("youtube_cookie_id".to_owned(), String::new());
entry
.save(&db)
.await
.map_err(|error| cot::Error::internal(error.to_string()))?;
}
crate::youtube::YoutubeCookieFile::delete_by_id(&db, id)
.await
.map_err(|error| cot::Error::internal(error.to_string()))?;
Json(serde_json::json!({ "ok": true })).into_response()
}
// ---------------------------------------------------------------------------
// Federation (status + manual controls)
// ---------------------------------------------------------------------------
@@ -1373,7 +1543,11 @@ pub async fn settings_probe(
.into_response()
}
fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto {
fn settings_dto(
config: AppConfig,
sources: ConfigSources,
cookie_files: Vec<crate::youtube::YoutubeCookieFileMetadata>,
) -> AdminSettingsDto {
let download_proxies = config
.parsed_download_proxies()
.unwrap_or_else(|error| {
@@ -1384,6 +1558,10 @@ fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto {
.map(AdminDownloadProxy::from)
.collect();
AdminSettingsDto {
youtube_cookie_files: cookie_files
.into_iter()
.map(AdminYoutubeCookieFileDto::from)
.collect(),
lastfm_api_key_configured: !config.lastfm_api_key.trim().is_empty(),
lastfm_shared_secret_configured: !config.lastfm_shared_secret.trim().is_empty(),
lastfm_scrobbling_configured: !config.lastfm_api_key.trim().is_empty()
@@ -1422,6 +1600,7 @@ fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto {
download_proxies,
torrent_proxy_id: config.torrent_proxy_id,
youtube_proxy_id: config.youtube_proxy_id,
youtube_cookie_id: config.youtube_cookie_id,
},
sources: AdminSettingsSources {
auth_password_enabled: sources.auth_password_enabled.code(),
@@ -1457,6 +1636,7 @@ fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto {
download_proxies: sources.download_proxies.code(),
torrent_proxy_id: sources.torrent_proxy_id.code(),
youtube_proxy_id: sources.youtube_proxy_id.code(),
youtube_cookie_id: sources.youtube_cookie_id.code(),
},
}
}
src/config.rs
+14 -5
View File
@@ -148,6 +148,7 @@ pub struct ConfigSources {
pub download_proxies: ConfigSource,
pub torrent_proxy_id: ConfigSource,
pub youtube_proxy_id: ConfigSource,
pub youtube_cookie_id: ConfigSource,
}
impl Default for ConfigSources {
@@ -188,6 +189,7 @@ impl Default for ConfigSources {
download_proxies: ConfigSource::Default,
torrent_proxy_id: ConfigSource::Default,
youtube_proxy_id: ConfigSource::Default,
youtube_cookie_id: ConfigSource::Default,
}
}
}
@@ -403,6 +405,8 @@ pub struct AppConfig {
pub torrent_proxy_id: String,
/// Saved proxy id used for YouTube downloads; empty means a direct connection.
pub youtube_proxy_id: String,
/// Saved cookie-file id used by yt-dlp; empty means no cookies.
pub youtube_cookie_id: String,
}
impl Default for AppConfig {
@@ -446,6 +450,7 @@ impl Default for AppConfig {
download_proxies: "[]".into(),
torrent_proxy_id: String::new(),
youtube_proxy_id: String::new(),
youtube_cookie_id: String::new(),
}
}
}
@@ -487,6 +492,7 @@ impl_env_overrides!(
download_proxies,
torrent_proxy_id,
youtube_proxy_id,
youtube_cookie_id,
);
impl AppConfig {
@@ -581,11 +587,13 @@ impl AppConfig {
sources.$field = ConfigSource::Database;
}
Err(_) => {
tracing::warn!(
"ignoring invalid DB config value for {}: {:?}",
stringify!($field),
val,
);
if !val.trim().is_empty() {
tracing::warn!(
"ignoring invalid DB config value for {}: {:?}",
stringify!($field),
val,
);
}
}
}
}
@@ -627,6 +635,7 @@ impl AppConfig {
apply_db_field!(download_proxies);
apply_db_field!(torrent_proxy_id);
apply_db_field!(youtube_proxy_id);
apply_db_field!(youtube_cookie_id);
}
pub fn parsed_download_proxies(&self) -> anyhow::Result<Vec<DownloadProxy>> {
src/metrics.rs
+2
View File
@@ -832,6 +832,8 @@ const KNOWN_HTTP_ROUTES: &[&str] = &[
"/admin/v2/api/jobs/{name}/run",
"/admin/v2/api/settings",
"/admin/v2/api/settings/probe",
"/admin/v2/api/settings/youtube-cookies",
"/admin/v2/api/settings/youtube-cookies/{id}",
"/admin/v2/api/jobs/{name}/toggle",
"/admin/v2/api/jobs/{name}/runs",
"/admin/v2/api/jobs/{name}/runs/{run_id}",
src/player/mod.rs
+36 -1
View File
@@ -93,6 +93,15 @@ fn download_proxy_for(
result.map_err(|error| json_error(StatusCode::BAD_REQUEST, &error.to_string()))
}
async fn youtube_cookie_contents(
config: &AppConfig,
db: &Database,
) -> Result<Option<String>, cot::response::Response> {
crate::youtube::selected_cookie_contents(db, &config.youtube_cookie_id)
.await
.map_err(|error| json_error(StatusCode::BAD_REQUEST, &error.to_string()))
}
#[derive(serde::Serialize)]
struct LocalUploadResponse {
ok: bool,
@@ -8542,12 +8551,23 @@ impl App for PlayerApp {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
let cookie_contents = match youtube_cookie_contents(&live_config, &db).await {
Ok(cookie_contents) => cookie_contents,
Err(response) => return Ok(response),
};
let service = youtube_service
.get_or_init(|| async {
Arc::new(YouTubeService::new(Arc::clone(&scheduler_handle)))
})
.await;
match service.preview(json.0, proxy_url.as_deref()).await {
match service
.preview(
json.0,
proxy_url.as_deref(),
cookie_contents.as_deref(),
)
.await
{
Ok(preview) => Json(preview).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
@@ -8603,12 +8623,17 @@ impl App for PlayerApp {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
let cookie_contents = match youtube_cookie_contents(&live_config, &db).await {
Ok(cookie_contents) => cookie_contents,
Err(response) => return Ok(response),
};
match service
.list(
pg_pool,
user.id,
&live_config.agent_inbox_dir,
proxy_url,
cookie_contents,
)
.await
{
@@ -8670,6 +8695,10 @@ impl App for PlayerApp {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
let cookie_contents = match youtube_cookie_contents(&live_config, &db).await {
Ok(cookie_contents) => cookie_contents,
Err(response) => return Ok(response),
};
match service
.start(
pg_pool,
@@ -8677,6 +8706,7 @@ impl App for PlayerApp {
json.0,
&live_config.agent_inbox_dir,
proxy_url,
cookie_contents,
)
.await
{
@@ -8738,6 +8768,10 @@ impl App for PlayerApp {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
let cookie_contents = match youtube_cookie_contents(&live_config, &db).await {
Ok(cookie_contents) => cookie_contents,
Err(response) => return Ok(response),
};
match service
.retry(
pg_pool,
@@ -8745,6 +8779,7 @@ impl App for PlayerApp {
&path.0.id,
&live_config.agent_inbox_dir,
proxy_url,
cookie_contents,
)
.await
{
src/youtube.rs
+439 -19
View File
@@ -1,10 +1,13 @@
use std::collections::{HashMap, HashSet, VecDeque};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, bail};
use cot::db::migrations::{self, Operation, SyncDynMigration};
use cot::db::{Database, LimitedString, Model};
use image::codecs::jpeg::JpegEncoder;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
@@ -22,11 +25,216 @@ const RESOLVE_TIMEOUT: Duration = Duration::from_secs(180);
const HTTP_403_RETRY_DELAY: Duration = Duration::from_secs(30);
const HTTP_403_MAX_RETRIES: usize = 3;
const MAX_ERROR_LEN: usize = 4_000;
pub const MAX_COOKIE_FILE_BYTES: usize = 2 * 1024 * 1024;
pub const MAX_COOKIE_FILES: u64 = 32;
const AUDIO_EXTENSIONS: &[&str] = &[
"mp3", "flac", "ogg", "opus", "aac", "m4a", "wav", "ape", "wv", "wma", "tta", "aiff", "aif",
];
const IMAGE_EXTENSIONS: &[&str] = &["jpg", "jpeg", "png", "webp", "bmp", "gif"];
#[derive(Debug, Clone)]
#[cot::db::model(table_name = "youtube_cookie_file")]
pub struct YoutubeCookieFile {
#[model(primary_key)]
id: String,
filename: String,
cookie_count: i64,
uploaded_at: LimitedString<32>,
contents: String,
}
#[derive(Debug, Clone)]
#[cot::db::model(table_name = "youtube_cookie_file")]
pub struct YoutubeCookieFileMetadata {
#[model(primary_key)]
id: String,
filename: String,
cookie_count: i64,
uploaded_at: LimitedString<32>,
}
impl YoutubeCookieFile {
pub async fn get_by_id(db: &Database, id: &str) -> cot::db::Result<Option<Self>> {
if id.len() > 36 {
return Ok(None);
}
let id = id.to_owned();
cot::db::query!(YoutubeCookieFile, $id == id).get(db).await
}
pub async fn delete_by_id(db: &Database, id: &str) -> cot::db::Result<()> {
if id.len() > 36 {
return Ok(());
}
let id = id.to_owned();
cot::db::query!(YoutubeCookieFile, $id == id)
.delete(db)
.await?;
Ok(())
}
pub fn id_str(&self) -> &str {
&self.id
}
pub fn filename(&self) -> &str {
&self.filename
}
pub fn cookie_count(&self) -> u64 {
u64::try_from(self.cookie_count).unwrap_or(0)
}
pub fn uploaded_at(&self) -> &str {
&self.uploaded_at
}
fn contents(&self) -> &str {
&self.contents
}
}
pub async fn store_cookie_file(
pool: &PgPool,
filename: &str,
cookie_count: usize,
contents: &str,
) -> anyhow::Result<YoutubeCookieFileMetadata> {
let id = Uuid::new_v4().to_string();
let uploaded_at = now_string();
let cookie_count = i64::try_from(cookie_count).unwrap_or(i64::MAX);
sqlx::query(
r#"INSERT INTO furumusic__youtube_cookie_file
(id, filename, cookie_count, uploaded_at, contents)
VALUES ($1, $2, $3, $4, $5)"#,
)
.bind(&id)
.bind(filename)
.bind(cookie_count)
.bind(&uploaded_at)
.bind(contents)
.execute(pool)
.await
.context("could not insert YouTube cookie file")?;
Ok(YoutubeCookieFileMetadata {
id,
filename: filename.to_owned(),
cookie_count,
uploaded_at: LimitedString::new(uploaded_at).expect("timestamp fits cookie metadata"),
})
}
impl YoutubeCookieFileMetadata {
pub async fn get_by_id(db: &Database, id: &str) -> cot::db::Result<Option<Self>> {
if id.len() > 36 {
return Ok(None);
}
let id = id.to_owned();
cot::db::query!(YoutubeCookieFileMetadata, $id == id)
.get(db)
.await
}
pub fn id_str(&self) -> &str {
&self.id
}
pub fn filename(&self) -> &str {
&self.filename
}
pub fn cookie_count(&self) -> u64 {
u64::try_from(self.cookie_count).unwrap_or(0)
}
pub fn uploaded_at(&self) -> &str {
&self.uploaded_at
}
}
pub async fn cookie_file_count(db: &Database) -> cot::db::Result<u64> {
YoutubeCookieFile::objects().count(db).await
}
pub async fn cookie_file_exists(db: &Database, id: &str) -> cot::db::Result<bool> {
Ok(YoutubeCookieFileMetadata::get_by_id(db, id)
.await?
.is_some())
}
pub async fn list_cookie_files(db: &Database) -> cot::db::Result<Vec<YoutubeCookieFileMetadata>> {
let mut files = YoutubeCookieFileMetadata::objects().all(db).await?;
files.sort_by(|left, right| {
right
.uploaded_at()
.cmp(left.uploaded_at())
.then_with(|| left.filename().cmp(right.filename()))
});
Ok(files)
}
pub async fn selected_cookie_contents(
db: &Database,
cookie_file_id: &str,
) -> anyhow::Result<Option<String>> {
let cookie_file_id = cookie_file_id.trim();
if cookie_file_id.is_empty() {
return Ok(None);
}
let file = YoutubeCookieFile::get_by_id(db, cookie_file_id)
.await?
.context("selected YouTube cookie file no longer exists")?;
Ok(Some(file.contents().to_owned()))
}
pub fn parse_cookie_file(data: &[u8]) -> anyhow::Result<(String, usize)> {
if data.is_empty() {
bail!("cookie file is empty");
}
if data.len() > MAX_COOKIE_FILE_BYTES {
bail!("cookie file is larger than 2 MiB");
}
let text = std::str::from_utf8(data).context("cookie file must be UTF-8 text")?;
let text = text.strip_prefix('\u{feff}').unwrap_or(text);
let mut normalized = Vec::new();
for (index, raw_line) in text.lines().enumerate() {
let line = raw_line.trim_end_matches('\r');
if line.trim().is_empty() || (line.starts_with('#') && !line.starts_with("#HttpOnly_")) {
continue;
}
let fields = line.splitn(7, '\t').collect::<Vec<_>>();
if fields.len() != 7 {
bail!(
"invalid Netscape cookie on line {}: expected 7 tab-separated fields",
index + 1
);
}
let domain = fields[0].strip_prefix("#HttpOnly_").unwrap_or(fields[0]);
if domain.is_empty()
|| domain.chars().any(char::is_whitespace)
|| !matches!(fields[1], "TRUE" | "FALSE")
|| fields[2].is_empty()
|| !matches!(fields[3], "TRUE" | "FALSE")
|| fields[4].parse::<i64>().is_err()
|| fields[5].is_empty()
{
bail!("invalid Netscape cookie on line {}", index + 1);
}
normalized.push(fields.join("\t"));
}
if normalized.is_empty() {
bail!("cookie file contains no Netscape-format cookies");
}
let cookie_count = normalized.len();
let contents = format!(
"# Netscape HTTP Cookie File\n# Generated by furumusic\n\n{}\n",
normalized.join("\n")
);
Ok((contents, cookie_count))
}
#[derive(Debug, Deserialize)]
pub struct YouTubePreviewRequest {
pub url: String,
@@ -216,6 +424,52 @@ impl std::fmt::Display for YtDlpDownloadFailure {
impl std::error::Error for YtDlpDownloadFailure {}
struct TemporaryCookieFile {
path: PathBuf,
}
impl TemporaryCookieFile {
fn create(contents: &str) -> anyhow::Result<Self> {
let path =
std::env::temp_dir().join(format!("furumusic-ytdlp-cookies-{}.txt", Uuid::new_v4()));
let mut options = std::fs::OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let mut file = options
.open(&path)
.context("could not create temporary yt-dlp cookie file")?;
if let Err(error) = file.write_all(contents.as_bytes()) {
let _ = std::fs::remove_file(&path);
return Err(error).context("could not write temporary yt-dlp cookie file");
}
Ok(Self { path })
}
fn path(&self) -> &Path {
&self.path
}
}
impl Drop for TemporaryCookieFile {
fn drop(&mut self) {
if let Err(error) = std::fs::remove_file(&self.path)
&& error.kind() != std::io::ErrorKind::NotFound
{
tracing::warn!(path = %self.path.display(), %error, "could not remove temporary yt-dlp cookie file");
}
}
}
#[derive(Clone, Copy)]
struct YtDlpOptions<'a> {
proxy_url: Option<&'a str>,
cookie_contents: Option<&'a str>,
}
pub struct YouTubeService {
running_jobs: Mutex<HashSet<String>>,
cancellations: Mutex<HashMap<String, CancellationToken>>,
@@ -237,9 +491,10 @@ impl YouTubeService {
&self,
request: YouTubePreviewRequest,
proxy_url: Option<&str>,
cookie_contents: Option<&str>,
) -> anyhow::Result<YouTubePreviewDto> {
let url = validate_youtube_url(&request.url)?;
let resolved = resolve_source(&url, proxy_url).await?;
let resolved = resolve_source(&url, proxy_url, cookie_contents).await?;
let requested_video_id = requested_video_id(&url);
let select_requested_only = resolved.kind == "playlist" && requested_video_id.is_some();
Ok(YouTubePreviewDto {
@@ -270,6 +525,7 @@ impl YouTubeService {
request: YouTubeStartRequest,
inbox_dir: &str,
proxy_url: Option<String>,
cookie_contents: Option<String>,
) -> anyhow::Result<YouTubeJobDto> {
let url = validate_youtube_url(&request.url)?;
validate_inbox_dir(inbox_dir)?;
@@ -286,7 +542,8 @@ impl YouTubeService {
bail!("YouTube selection contains an invalid video ID");
}
let resolved = resolve_source(&url, proxy_url.as_deref()).await?;
let resolved =
resolve_source(&url, proxy_url.as_deref(), cookie_contents.as_deref()).await?;
let selected_items: Vec<ResolvedItem> = resolved
.items
.into_iter()
@@ -357,8 +614,14 @@ impl YouTubeService {
}
transaction.commit().await?;
self.spawn_job(pool.clone(), id.clone(), inbox_dir.to_string(), proxy_url)
.await;
self.spawn_job(
pool.clone(),
id.clone(),
inbox_dir.to_string(),
proxy_url,
cookie_contents,
)
.await;
load_job_dto(pool, user_id, &id).await
}
@@ -368,6 +631,7 @@ impl YouTubeService {
user_id: i64,
inbox_dir: &str,
proxy_url: Option<String>,
cookie_contents: Option<String>,
) -> anyhow::Result<Vec<YouTubeJobDto>> {
validate_inbox_dir(inbox_dir)?;
sync_ai_statuses(pool, user_id).await?;
@@ -383,8 +647,14 @@ impl YouTubeService {
.fetch_all(pool)
.await?;
for (id, _) in resumable {
self.spawn_job(pool.clone(), id, inbox_dir.to_string(), proxy_url.clone())
.await;
self.spawn_job(
pool.clone(),
id,
inbox_dir.to_string(),
proxy_url.clone(),
cookie_contents.clone(),
)
.await;
}
let ids: Vec<String> = sqlx::query_scalar(
@@ -413,6 +683,7 @@ impl YouTubeService {
id: &str,
inbox_dir: &str,
proxy_url: Option<String>,
cookie_contents: Option<String>,
) -> anyhow::Result<YouTubeJobDto> {
validate_inbox_dir(inbox_dir)?;
let job = load_job_row(pool, user_id, id).await?;
@@ -448,6 +719,7 @@ impl YouTubeService {
id.to_string(),
inbox_dir.to_string(),
proxy_url,
cookie_contents,
)
.await;
load_job_dto(pool, user_id, id).await
@@ -538,6 +810,7 @@ impl YouTubeService {
id: String,
inbox_dir: String,
proxy_url: Option<String>,
cookie_contents: Option<String>,
) {
{
let mut running = self.running_jobs.lock().await;
@@ -559,7 +832,14 @@ impl YouTubeService {
};
let result = if let Some(permit) = permit {
let result = service
.run_job(&pool, &id, &inbox_dir, proxy_url.as_deref(), &cancel)
.run_job(
&pool,
&id,
&inbox_dir,
proxy_url.as_deref(),
cookie_contents.as_deref(),
&cancel,
)
.await;
drop(permit);
result
@@ -588,8 +868,13 @@ impl YouTubeService {
id: &str,
inbox_dir: &str,
proxy_url: Option<&str>,
cookie_contents: Option<&str>,
cancel: &CancellationToken,
) -> anyhow::Result<()> {
let ytdlp = YtDlpOptions {
proxy_url,
cookie_contents,
};
if cancel.is_cancelled() {
return Ok(());
}
@@ -610,7 +895,8 @@ impl YouTubeService {
return Ok(());
}
set_parent_status(pool, id, "resolving", None).await?;
let resolved = resolve_source(&job.source_url, proxy_url).await?;
let resolved =
resolve_source(&job.source_url, ytdlp.proxy_url, ytdlp.cookie_contents).await?;
if cancel.is_cancelled() {
return Ok(());
}
@@ -677,7 +963,7 @@ impl YouTubeService {
continue;
}
if let Err(err) = self
.process_item(pool, &job, &item, &inbox_root, proxy_url, cancel)
.process_item(pool, &job, &item, &inbox_root, ytdlp, cancel)
.await
{
if cancel.is_cancelled() {
@@ -707,7 +993,7 @@ impl YouTubeService {
job: &YouTubeJobRow,
item: &YouTubeItemRow,
inbox_root: &Path,
proxy_url: Option<&str>,
ytdlp: YtDlpOptions<'_>,
cancel: &CancellationToken,
) -> anyhow::Result<()> {
if cancel.is_cancelled() {
@@ -720,8 +1006,16 @@ impl YouTubeService {
tokio::fs::create_dir_all(&stage).await?;
let mut forbidden_retries = 0;
loop {
match run_ytdlp_download(pool, &item.id, &item.source_url, &stage, proxy_url, cancel)
.await
match run_ytdlp_download(
pool,
&item.id,
&item.source_url,
&stage,
ytdlp.proxy_url,
ytdlp.cookie_contents,
cancel,
)
.await
{
Ok(()) => break,
Err(error)
@@ -815,8 +1109,15 @@ impl YouTubeService {
}
}
async fn resolve_source(url: &str, proxy_url: Option<&str>) -> anyhow::Result<ResolvedSource> {
let mut command = base_ytdlp_command(proxy_url);
async fn resolve_source(
url: &str,
proxy_url: Option<&str>,
cookie_contents: Option<&str>,
) -> anyhow::Result<ResolvedSource> {
let cookie_file = cookie_contents
.map(TemporaryCookieFile::create)
.transpose()?;
let mut command = base_ytdlp_command(proxy_url, cookie_file.as_ref().map(|file| file.path()));
command
.arg("--flat-playlist")
.arg("--dump-single-json")
@@ -887,15 +1188,19 @@ async fn resolve_source(url: &str, proxy_url: Option<&str>) -> anyhow::Result<Re
})
}
fn base_ytdlp_command(proxy_url: Option<&str>) -> Command {
fn base_ytdlp_command(proxy_url: Option<&str>, cookie_file: Option<&Path>) -> Command {
let mut command = Command::new("yt-dlp");
command
.arg("--no-config")
.arg("--no-cookies")
.arg("--no-cookies-from-browser")
.arg("--js-runtimes")
.arg("deno")
.stdin(Stdio::null());
if let Some(cookie_file) = cookie_file {
command.arg("--cookies").arg(cookie_file);
} else {
command.arg("--no-cookies");
}
if let Some(proxy_url) = proxy_url {
command.arg("--proxy").arg(proxy_url);
}
@@ -908,9 +1213,13 @@ async fn run_ytdlp_download(
url: &str,
stage: &Path,
proxy_url: Option<&str>,
cookie_contents: Option<&str>,
cancel: &CancellationToken,
) -> anyhow::Result<()> {
let mut command = base_ytdlp_command(proxy_url);
let cookie_file = cookie_contents
.map(TemporaryCookieFile::create)
.transpose()?;
let mut command = base_ytdlp_command(proxy_url, cookie_file.as_ref().map(|file| file.path()));
command
.arg("--no-playlist")
.arg("--continue")
@@ -1964,10 +2273,61 @@ fn now_string() -> String {
chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string()
}
pub mod db_migrations {
use super::*;
#[cot::db::migrations::migration_op]
async fn create_youtube_cookie_files(
ctx: migrations::MigrationContext<'_>,
) -> cot::db::Result<()> {
ctx.db
.raw(
"CREATE TABLE IF NOT EXISTS furumusic__youtube_cookie_file (
id VARCHAR(36) PRIMARY KEY,
filename TEXT NOT NULL,
cookie_count BIGINT NOT NULL,
uploaded_at VARCHAR(32) NOT NULL,
contents TEXT NOT NULL
)",
)
.await?;
Ok(())
}
#[derive(Debug, Copy, Clone)]
pub struct M0048CreateYoutubeCookieFiles;
impl migrations::Migration for M0048CreateYoutubeCookieFiles {
const APP_NAME: &'static str = "furumusic";
const MIGRATION_NAME: &'static str = "m_0048_create_youtube_cookie_files";
const DEPENDENCIES: &'static [migrations::MigrationDependency] =
&[migrations::MigrationDependency::migration(
"furumusic",
"m_0047_create_youtube_import_media_links",
)];
const OPERATIONS: &'static [Operation] =
&[Operation::custom(create_youtube_cookie_files).build()];
}
pub const MIGRATIONS: &[&SyncDynMigration] = &[&M0048CreateYoutubeCookieFiles];
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cookie_models_use_exact_migrated_table_name() {
assert_eq!(
<YoutubeCookieFile as Model>::TABLE_NAME.as_str(),
"furumusic__youtube_cookie_file"
);
assert_eq!(
<YoutubeCookieFileMetadata as Model>::TABLE_NAME.as_str(),
"furumusic__youtube_cookie_file"
);
}
#[test]
fn validates_supported_youtube_hosts() {
assert!(validate_youtube_url("https://youtube.com/watch?v=abc").is_ok());
@@ -2049,7 +2409,7 @@ mod tests {
#[test]
fn ytdlp_command_receives_selected_proxy() {
let command = base_ytdlp_command(Some("socks5://user:pass@127.0.0.1:1080/"));
let command = base_ytdlp_command(Some("socks5://user:pass@127.0.0.1:1080/"), None);
let args: Vec<String> = command
.as_std()
.get_args()
@@ -2062,13 +2422,73 @@ mod tests {
]
}));
let direct = base_ytdlp_command(None);
let direct = base_ytdlp_command(None, None);
assert!(
direct
.as_std()
.get_args()
.all(|argument| argument != "--proxy")
);
assert!(
direct
.as_std()
.get_args()
.any(|argument| argument == "--no-cookies")
);
}
#[test]
fn parses_and_normalizes_netscape_cookie_files() {
let source = b"# Netscape HTTP Cookie File\r\n\r\n.youtube.com\tTRUE\t/\tTRUE\t1893456000\tSID\tsecret\r\n#HttpOnly_.youtube.com\tTRUE\t/\tTRUE\t0\tLOGIN_INFO\tvalue\r\n";
let (contents, count) = parse_cookie_file(source).unwrap();
assert_eq!(count, 2);
assert!(contents.starts_with("# Netscape HTTP Cookie File\n"));
assert!(contents.contains("#HttpOnly_.youtube.com\tTRUE"));
assert!(!contents.contains('\r'));
}
#[test]
fn rejects_non_netscape_cookie_files() {
assert!(parse_cookie_file(br#"[{"name":"SID","value":"secret"}]"#).is_err());
assert!(parse_cookie_file(b"# Netscape HTTP Cookie File\n").is_err());
assert!(parse_cookie_file(b"youtube.com TRUE / TRUE 0 SID secret\n").is_err());
}
#[test]
fn ytdlp_command_receives_cookie_file_when_selected() {
let path = Path::new("/tmp/furumusic-test-cookies.txt");
let command = base_ytdlp_command(None, Some(path));
let args = command
.as_std()
.get_args()
.map(|argument| argument.to_string_lossy().into_owned())
.collect::<Vec<_>>();
assert!(args.windows(2).any(|pair| {
pair == ["--cookies".to_string(), path.to_string_lossy().into_owned()]
}));
assert!(!args.iter().any(|argument| argument == "--no-cookies"));
}
#[test]
fn temporary_cookie_file_is_private_and_removed() {
let path;
{
let file = TemporaryCookieFile::create(
"# Netscape HTTP Cookie File\n.youtube.com\tTRUE\t/\tTRUE\t0\tSID\tsecret\n",
)
.unwrap();
path = file.path().to_owned();
assert_eq!(std::fs::read_to_string(&path).unwrap().lines().count(), 2);
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
assert_eq!(
std::fs::metadata(&path).unwrap().permissions().mode() & 0o777,
0o600
);
}
}
assert!(!path.exists());
}
#[test]