Furumi/furumi-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Added youtube import. Reworked Download Manager
Build and Publish / Build and Publish Docker Image (push) Successful in 3m52s
Details

Browse Source
This commit is contained in:
Aleksandr Bogomiakov
2026-08-14 01:21:51 +01:00
parent 2b254f417f
commit 943191a0ff
16 changed files with 4167 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/i18n/phrases.rs
+67 -9
View File
@@ -389,9 +389,54 @@ translations! {
player_live_releases: "Live releases" , "Концертные релизы";
player_soundtracks: "Soundtracks" , "Саундтреки";
// Player torrent/history UI
player_torrent_manager: "Torrent manager" , "Торрент-менеджер";
player_import_torrent: "Import torrent" , "Импортировать торрент";
// Player download/history UI
player_torrent_manager: "Download Manager" , "Менеджер загрузок";
player_import_torrent: "Open Download Manager" , "Открыть менеджер загрузок";
player_youtube: "YouTube" , "YouTube";
player_torrents: "Torrents" , "Торренты";
player_files: "Files" , "Файлы";
player_youtube_url: "Public video or playlist URL" , "Ссылка на открытое видео или плейлист";
player_youtube_url_hint: "Public youtube.com, music.youtube.com and youtu.be links are supported. Each video is imported as one AI batch." , "Поддерживаются открытые ссылки youtube.com, music.youtube.com и youtu.be. Каждое видео импортируется одним ИИ-батчем.";
player_youtube_downloads: "YouTube downloads" , "Загрузки YouTube";
player_no_youtube_downloads: "No YouTube downloads yet" , "Загрузок YouTube пока нет";
player_youtube_parse: "Check link" , "Проверить ссылку";
player_youtube_parsing: "Reading YouTube link..." , "Читаю ссылку YouTube...";
player_youtube_preview_failed: "Could not read YouTube link" , "Не удалось прочитать ссылку YouTube";
player_youtube_preview_title: "Choose videos to import" , "Выберите видео для импорта";
player_youtube_select_all: "Select all" , "Отметить все";
player_youtube_clear_selection: "Clear selection" , "Снять все";
player_youtube_selected_count: "selected" , "выбрано";
player_youtube_start_import: "Start import" , "Начать импорт";
player_start_download: "Start download" , "Начать загрузку";
player_retry_failed: "Retry failed" , "Повторить ошибки";
player_download_steps: "Processing steps" , "Этапы обработки";
player_chapters: "chapters" , "глав";
player_youtube_video: "video" , "видео";
player_youtube_playlist: "playlist" , "плейлист";
player_youtube_items: "videos" , "видео";
player_youtube_errors: "errors" , "ошибок";
player_youtube_queued: "Queued" , "В очереди";
player_youtube_resolving: "Reading link" , "Чтение ссылки";
player_youtube_postprocessing: "FFmpeg processing" , "Обработка FFmpeg";
player_youtube_awaiting_ai: "Waiting for AI" , "Ожидание ИИ";
player_youtube_ai_processing: "AI processing" , "Обработка ИИ";
player_youtube_needs_review: "Needs review" , "Требует проверки";
player_youtube_complete_with_errors: "Completed with errors" , "Завершено с ошибками";
player_youtube_skipped: "Already imported" , "Уже импортировано";
player_youtube_cancelled: "Stopped" , "Остановлено";
player_youtube_stop: "Stop import" , "Остановить импорт";
player_youtube_stop_confirm: "Stop this YouTube import? Completed and already published audio will remain." , "Остановить этот импорт из YouTube? Готовое и уже переданное на обработку аудио останется.";
player_youtube_stopping: "Stopping YouTube import..." , "Останавливаю импорт из YouTube...";
player_youtube_stopped: "YouTube import stopped." , "Импорт из YouTube остановлен.";
player_youtube_stop_failed: "Could not stop YouTube import" , "Не удалось остановить импорт из YouTube";
player_youtube_starting: "Adding YouTube download..." , "Добавляю загрузку YouTube...";
player_youtube_started: "YouTube download added." , "Загрузка YouTube добавлена.";
player_youtube_load_failed: "Could not load YouTube downloads" , "Не удалось загрузить список YouTube";
player_youtube_start_failed: "Could not start YouTube download" , "Не удалось начать загрузку YouTube";
player_youtube_retry_failed: "Could not retry YouTube download" , "Не удалось повторить загрузку YouTube";
player_youtube_delete_failed: "Could not remove YouTube download" , "Не удалось удалить загрузку YouTube";
player_youtube_delete_confirm: "Remove this YouTube download from history? Imported audio will remain." , "Удалить эту загрузку YouTube из истории? Импортированное аудио останется.";
player_remove_from_history: "Remove from history" , "Удалить из истории";
player_client_idle: "Client idle" , "Клиент простаивает";
player_active: "active" , "активно";
player_ai_idle: "AI idle" , "ИИ простаивает";
@@ -467,11 +512,24 @@ translations! {
player_track_approved_imported: "Track approved and imported" , "Трек подтверждён и импортирован";
player_failed_update_selected_tracks: "Failed to update selected tracks" , "Не удалось обновить выбранные треки";
player_selected_tracks_updated: "Selected tracks updated" , "Выбранные треки обновлены";
player_choose_saved_or_add_torrent: "Choose a saved item or upload new files." , "Выберите сохранённый элемент или загрузите новые файлы.";
player_choose_saved_or_add_torrent: "Choose a saved torrent or add a new one." , "Выберите сохранённый торрент или добавьте новый.";
player_local_files: "Local audio files" , "Локальные аудиофайлы";
player_file_uploads: "File uploads" , "Загрузки файлов";
player_drop_audio_title: "Drop audio files here" , "Перетащите аудиофайлы сюда";
player_drop_audio_hint: "or click to choose files" , "или нажмите, чтобы выбрать файлы";
player_drop_audio_formats: "MP3, FLAC, WAV, M4A, OGG, Opus and AAC" , "MP3, FLAC, WAV, M4A, OGG, Opus и AAC";
player_upload_selected_files: "Upload selected files" , "Загрузить выбранные файлы";
player_upload_history: "Upload history" , "История загрузок";
player_no_file_uploads: "No file uploads yet" , "Загрузок файлов пока нет";
player_file_upload_load_failed: "Could not load file upload history" , "Не удалось загрузить историю файлов";
player_remove_file_upload_confirm: "Remove this file upload from history? Imported audio will remain." , "Удалить эту загрузку файла из истории? Импортированное аудио останется.";
player_file_upload_history_removed: "File upload removed from history." , "Загрузка файла удалена из истории.";
player_file_upload_history_remove_failed: "Could not remove file upload from history" , "Не удалось удалить загрузку файла из истории";
player_no_supported_audio_files: "Choose at least one supported audio file." , "Выберите хотя бы один поддерживаемый аудиофайл.";
player_torrent_file: "Torrent file" , "Torrent-файл";
player_magnet_link: "Magnet link" , "Magnet-ссылка";
player_upload_content: "Upload" , "Загрузить";
player_upload_content: "Preview torrent" , "Проверить торрент";
player_add_torrent: "Add torrent" , "Добавить торрент";
player_download_selected: "Download selected" , "Скачать выбранное";
player_pause_download: "Pause download" , "Поставить на паузу";
player_expand_all: "Expand all" , "Развернуть всё";
@@ -501,7 +559,7 @@ translations! {
player_no_plays_yet: "No plays yet" , "Прослушиваний пока нет";
player_page: "Page" , "Страница";
player_of: "of" , "из";
player_choose_torrent: "Choose local files, paste a magnet link, or choose a .torrent file." , "Выберите локальные файлы, вставьте magnet-ссылку или выберите .torrent файл.";
player_choose_torrent: "Paste a magnet link or choose a .torrent file." , "Вставьте magnet-ссылку или выберите .torrent файл.";
player_uploading_files: "Uploading files..." , "Загружаю файлы...";
player_upload_complete: "Upload complete. Files are queued for processing." , "Загрузка завершена. Файлы поставлены в обработку.";
player_upload_failed: "Upload failed" , "Загрузка не удалась";
@@ -511,8 +569,8 @@ translations! {
player_all_files_selected: "All files are selected by default. Clear or adjust the tree before download." , "Все файлы выбраны по умолчанию. Перед скачиванием можно очистить или изменить выбор.";
player_opening_saved_torrent: "Opening saved torrent..." , "Открываю сохранённый торрент...";
player_saved_torrent_opened: "Saved torrent opened. Adjust files or resume download." , "Сохранённый торрент открыт. Можно изменить файлы или продолжить скачивание.";
player_remove_torrent_confirm: "Remove this torrent from the client list? Downloaded files will stay on disk." , "Удалить этот торрент из списка клиента? Скачанные файлы останутся на диске.";
player_torrent_removed: "Torrent removed from the client list." , "Торрент удалён из списка клиента.";
player_remove_torrent_confirm: "Remove this torrent from history? Downloaded files will stay on disk." , "Удалить этот торрент из истории? Скачанные файлы останутся на диске.";
player_torrent_removed: "Torrent removed from history." , "Торрент удалён из истории.";
player_select_one_file: "Select at least one file." , "Выберите хотя бы один файл.";
player_starting_download: "Starting download..." , "Запускаю скачивание...";
player_download_started: "Download started. Files will move to inbox when complete." , "Скачивание началось. После завершения файлы будут перенесены во входящие.";
@@ -523,6 +581,6 @@ translations! {
player_pause_failed: "Pause failed" , "Не удалось поставить на паузу";
player_load_torrents_failed: "Could not load torrents" , "Не удалось загрузить торренты";
player_open_torrent_failed: "Could not open torrent" , "Не удалось открыть торрент";
player_delete_torrent_failed: "Could not delete torrent" , "Не удалось удалить торрент";
player_delete_torrent_failed: "Could not remove torrent from history" , "Не удалось удалить торрент из истории";
player_load_ai_queue_failed: "Could not load AI queue" , "Не удалось загрузить очередь ИИ";
}
src/local_uploads.rs
+259
View File
@@ -0,0 +1,259 @@
use std::collections::HashMap;
use anyhow::{Context, bail};
use serde::Serialize;
use sqlx::{FromRow, PgPool};
const LOCAL_UPLOAD_LIST_LIMIT: i64 = 100;
#[derive(Debug, Clone, Serialize)]
pub struct LocalUploadDto {
pub id: String,
pub filename: String,
pub size_bytes: u64,
pub status: String,
pub error: Option<String>,
pub created_at: String,
pub updated_at: String,
pub completed_at: Option<String>,
}
#[derive(Debug, Clone, FromRow)]
struct LocalUploadRow {
id: String,
user_id: i64,
filename: String,
size_bytes: i64,
status: String,
inbox_path: String,
error: Option<String>,
created_at: String,
updated_at: String,
completed_at: Option<String>,
}
impl LocalUploadRow {
fn dto(&self) -> LocalUploadDto {
LocalUploadDto {
id: self.id.clone(),
filename: self.filename.clone(),
size_bytes: u64::try_from(self.size_bytes).unwrap_or(0),
status: self.status.clone(),
error: self.error.clone(),
created_at: self.created_at.clone(),
updated_at: self.updated_at.clone(),
completed_at: self.completed_at.clone(),
}
}
}
pub async fn create(
pool: &PgPool,
id: &str,
user_id: i64,
filename: &str,
size_bytes: u64,
inbox_path: &str,
) -> anyhow::Result<()> {
let now = now_string();
sqlx::query(
r#"INSERT INTO furumusic__local_upload
(id, user_id, filename, size_bytes, status, inbox_path, error,
created_at, updated_at, completed_at)
VALUES ($1, $2, $3, $4, 'uploading', $5, NULL, $6, $6, NULL)"#,
)
.bind(id)
.bind(user_id)
.bind(filename)
.bind(i64::try_from(size_bytes).unwrap_or(i64::MAX))
.bind(inbox_path)
.bind(now)
.execute(pool)
.await?;
Ok(())
}
pub async fn mark_queued(pool: &PgPool, id: &str, user_id: i64) -> anyhow::Result<LocalUploadDto> {
update_status(pool, id, user_id, "queued", None).await?;
load(pool, user_id, id).await.map(|row| row.dto())
}
pub async fn mark_failed(pool: &PgPool, id: &str, user_id: i64, error: &str) -> anyhow::Result<()> {
update_status(pool, id, user_id, "failed", Some(error)).await
}
pub async fn list(
pool: &PgPool,
user_id: i64,
inbox_dir: &str,
) -> anyhow::Result<Vec<LocalUploadDto>> {
sync_statuses(pool, user_id, inbox_dir).await?;
let rows: Vec<LocalUploadRow> = sqlx::query_as(
r#"SELECT id, user_id, filename, size_bytes, status, inbox_path, error,
created_at, updated_at, completed_at
FROM furumusic__local_upload
WHERE user_id = $1
ORDER BY created_at DESC, id DESC
LIMIT $2"#,
)
.bind(user_id)
.bind(LOCAL_UPLOAD_LIST_LIMIT)
.fetch_all(pool)
.await?;
Ok(rows.iter().map(LocalUploadRow::dto).collect())
}
pub async fn remove(pool: &PgPool, user_id: i64, id: &str) -> anyhow::Result<()> {
let result = sqlx::query("DELETE FROM furumusic__local_upload WHERE id = $1 AND user_id = $2")
.bind(id)
.bind(user_id)
.execute(pool)
.await?;
if result.rows_affected() == 0 {
bail!("file upload history entry not found");
}
Ok(())
}
async fn sync_statuses(pool: &PgPool, user_id: i64, inbox_dir: &str) -> anyhow::Result<()> {
let inbox_dir = inbox_dir.trim();
if inbox_dir.is_empty() {
bail!("agent_inbox_dir is not configured");
}
let inbox_root = crate::media_paths::resolve_config_path_buf(inbox_dir);
if !inbox_root.is_absolute() {
bail!("agent_inbox_dir must be an absolute path");
}
let rows: Vec<LocalUploadRow> = sqlx::query_as(
r#"SELECT id, user_id, filename, size_bytes, status, inbox_path, error,
created_at, updated_at, completed_at
FROM furumusic__local_upload
WHERE user_id = $1 AND status <> 'complete'"#,
)
.bind(user_id)
.fetch_all(pool)
.await?;
if rows.is_empty() {
return Ok(());
}
let inbox_paths: Vec<String> = rows.iter().map(|row| row.inbox_path.clone()).collect();
let state_rows: Vec<(String, String, i64)> = sqlx::query_as(
r#"SELECT input_path, status::text, COUNT(*)
FROM furumusic__pending_review
WHERE input_path = ANY($1)
GROUP BY input_path, status"#,
)
.bind(&inbox_paths)
.fetch_all(pool)
.await?;
let mut states_by_path: HashMap<String, HashMap<String, i64>> = HashMap::new();
for (input_path, status, total) in state_rows {
states_by_path
.entry(input_path)
.or_default()
.insert(status, total);
}
let error_rows: Vec<(String, String)> = sqlx::query_as(
r#"SELECT DISTINCT ON (input_path) input_path, error_message
FROM furumusic__pending_review
WHERE input_path = ANY($1) AND status = 'failed'
AND error_message IS NOT NULL
ORDER BY input_path, id DESC"#,
)
.bind(&inbox_paths)
.fetch_all(pool)
.await?;
let errors_by_path: HashMap<String, String> = error_rows.into_iter().collect();
for row in rows {
let counts = states_by_path
.get(&row.inbox_path)
.cloned()
.unwrap_or_default();
let total: i64 = counts.values().sum();
let mut terminal_error = None;
let next = if total == 0 {
if matches!(row.status.as_str(), "uploading" | "failed" | "needs_review") {
continue;
}
let full_path = crate::media_paths::resolve_path_from_root(inbox_dir, &row.inbox_path);
if tokio::fs::try_exists(full_path).await.unwrap_or(false) {
"queued"
} else {
"complete"
}
} else if count(&counts, "processing") > 0 {
"ai_processing"
} else if count(&counts, "queued") > 0 {
"queued"
} else if count(&counts, "failed") > 0 {
terminal_error = errors_by_path.get(&row.inbox_path).cloned();
"failed"
} else if count(&counts, "pending") > 0 || count(&counts, "rejected") > 0 {
"needs_review"
} else if count(&counts, "approved") > 0 || count(&counts, "auto_approved") > 0 {
"complete"
} else {
"queued"
};
if row.status != next || row.error != terminal_error {
update_status(pool, &row.id, row.user_id, next, terminal_error.as_deref()).await?;
}
}
Ok(())
}
async fn update_status(
pool: &PgPool,
id: &str,
user_id: i64,
status: &str,
error: Option<&str>,
) -> anyhow::Result<()> {
let now = now_string();
let completed_at =
matches!(status, "complete" | "failed" | "needs_review").then(|| now.clone());
sqlx::query(
r#"UPDATE furumusic__local_upload
SET status = $3, error = $4, updated_at = $5, completed_at = $6
WHERE id = $1 AND user_id = $2"#,
)
.bind(id)
.bind(user_id)
.bind(status)
.bind(error.map(trim_error))
.bind(&now)
.bind(completed_at)
.execute(pool)
.await?;
Ok(())
}
async fn load(pool: &PgPool, user_id: i64, id: &str) -> anyhow::Result<LocalUploadRow> {
sqlx::query_as(
r#"SELECT id, user_id, filename, size_bytes, status, inbox_path, error,
created_at, updated_at, completed_at
FROM furumusic__local_upload WHERE id = $1 AND user_id = $2"#,
)
.bind(id)
.bind(user_id)
.fetch_optional(pool)
.await?
.context("file upload history entry not found")
}
fn count(counts: &HashMap<String, i64>, status: &str) -> i64 {
counts.get(status).copied().unwrap_or(0)
}
fn trim_error(value: &str) -> String {
value.chars().take(4_000).collect()
}
fn now_string() -> String {
chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string()
}
src/main.rs
+2
View File
@@ -7,6 +7,7 @@ mod federation;
mod i18n;
mod jobs;
mod lastfm;
mod local_uploads;
mod media_paths;
mod metrics;
mod music;
@@ -16,6 +17,7 @@ mod scheduler;
mod similarity;
mod torrents;
mod user;
mod youtube;
use std::sync::Arc;
src/metrics.rs
+25 -1
View File
@@ -884,10 +884,18 @@ const KNOWN_HTTP_ROUTES: &[&str] = &[
"/api/player/lastfm/scrobble",
"/api/player/agent-queue",
"/api/player/offline/manifest",
"/api/player/youtube",
"/api/player/youtube/preview",
"/api/player/youtube/start",
"/api/player/youtube/{id}/retry",
"/api/player/youtube/{id}/cancel",
"/api/player/youtube/{id}",
"/api/player/uploads/local",
"/api/player/uploads/local/history",
"/api/player/uploads/local/history/{id}",
"/api/player/torrents",
"/api/player/torrents/session/{id}",
"/api/player/torrents/preview",
"/api/player/uploads/local",
"/api/player/uploads/tracks",
"/api/player/uploads/tracks/{track_id}",
"/api/player/uploads/bulk-tracks",
@@ -951,6 +959,22 @@ mod tests {
known_http_route("/share/release/42"),
Some("/share/release/{id}")
);
assert_eq!(
known_http_route("/api/player/youtube/start"),
Some("/api/player/youtube/start")
);
assert_eq!(
known_http_route("/api/player/youtube/job-42/retry"),
Some("/api/player/youtube/{id}/retry")
);
assert_eq!(
known_http_route("/api/player/youtube/job-42/cancel"),
Some("/api/player/youtube/{id}/cancel")
);
assert_eq!(
known_http_route("/api/player/uploads/local/history/upload-42"),
Some("/api/player/uploads/local/history/{id}")
);
}
#[test]
src/music/mod.rs
+148
View File
@@ -2569,6 +2569,152 @@ pub mod db_migrations {
&[Operation::custom(add_similarity_routing_signature).build()];
}
// -- M0045: persistent YouTube download jobs ----------------------------
#[cot::db::migrations::migration_op]
async fn create_youtube_downloads(
ctx: migrations::MigrationContext<'_>,
) -> cot::db::Result<()> {
ctx.db
.raw(
"CREATE TABLE IF NOT EXISTS furumusic__youtube_download (
id VARCHAR(36) PRIMARY KEY,
user_id BIGINT NOT NULL,
source_url TEXT NOT NULL,
title TEXT NOT NULL,
source_kind VARCHAR(32) NOT NULL,
status VARCHAR(32) NOT NULL,
total_items INTEGER NOT NULL DEFAULT 0,
completed_items INTEGER NOT NULL DEFAULT 0,
failed_items INTEGER NOT NULL DEFAULT 0,
review_items INTEGER NOT NULL DEFAULT 0,
error TEXT,
created_at VARCHAR(32) NOT NULL,
updated_at VARCHAR(32) NOT NULL,
completed_at VARCHAR(32)
)",
)
.await?;
ctx.db
.raw(
"CREATE TABLE IF NOT EXISTS furumusic__youtube_download_item (
id VARCHAR(36) PRIMARY KEY,
job_id VARCHAR(36) NOT NULL REFERENCES furumusic__youtube_download(id) ON DELETE CASCADE,
source_id VARCHAR(128) NOT NULL,
source_url TEXT NOT NULL,
title TEXT NOT NULL,
playlist_index INTEGER NOT NULL,
status VARCHAR(32) NOT NULL,
progress_percent DOUBLE PRECISION NOT NULL DEFAULT 0,
downloaded_bytes BIGINT NOT NULL DEFAULT 0,
total_bytes BIGINT,
speed_bytes_per_sec BIGINT,
eta_seconds BIGINT,
chapter_count INTEGER NOT NULL DEFAULT 0,
audio_file_count INTEGER NOT NULL DEFAULT 0,
inbox_path TEXT,
error TEXT,
created_at VARCHAR(32) NOT NULL,
updated_at VARCHAR(32) NOT NULL,
completed_at VARCHAR(32),
UNIQUE(job_id, source_id)
)",
)
.await?;
ctx.db
.raw(
"CREATE INDEX IF NOT EXISTS idx_youtube_download_user_updated
ON furumusic__youtube_download (user_id, updated_at DESC)",
)
.await?;
ctx.db
.raw(
"CREATE INDEX IF NOT EXISTS idx_youtube_download_user_status
ON furumusic__youtube_download (user_id, status)",
)
.await?;
ctx.db
.raw(
"CREATE INDEX IF NOT EXISTS idx_youtube_download_item_job_status
ON furumusic__youtube_download_item (job_id, status, playlist_index)",
)
.await?;
ctx.db
.raw(
"CREATE INDEX IF NOT EXISTS idx_youtube_download_item_source
ON furumusic__youtube_download_item (source_id)",
)
.await?;
Ok(())
}
#[derive(Debug, Copy, Clone)]
pub struct M0045CreateYouTubeDownloads;
impl migrations::Migration for M0045CreateYouTubeDownloads {
const APP_NAME: &'static str = "furumusic";
const MIGRATION_NAME: &'static str = "m_0045_create_youtube_downloads";
const DEPENDENCIES: &'static [migrations::MigrationDependency] =
&[migrations::MigrationDependency::migration(
"furumusic",
"m_0044_add_similarity_routing_signature",
)];
const OPERATIONS: &'static [Operation] =
&[Operation::custom(create_youtube_downloads).build()];
}
// -- M0046: persistent direct-file upload history -----------------------
#[cot::db::migrations::migration_op]
async fn create_local_upload_history(
ctx: migrations::MigrationContext<'_>,
) -> cot::db::Result<()> {
ctx.db
.raw(
"CREATE TABLE IF NOT EXISTS furumusic__local_upload (
id VARCHAR(36) PRIMARY KEY,
user_id BIGINT NOT NULL,
filename TEXT NOT NULL,
size_bytes BIGINT NOT NULL DEFAULT 0,
status VARCHAR(32) NOT NULL,
inbox_path TEXT NOT NULL,
error TEXT,
created_at VARCHAR(32) NOT NULL,
updated_at VARCHAR(32) NOT NULL,
completed_at VARCHAR(32)
)",
)
.await?;
ctx.db
.raw(
"CREATE INDEX IF NOT EXISTS idx_local_upload_user_updated
ON furumusic__local_upload (user_id, updated_at DESC)",
)
.await?;
ctx.db
.raw(
"CREATE INDEX IF NOT EXISTS idx_local_upload_user_status
ON furumusic__local_upload (user_id, status)",
)
.await?;
Ok(())
}
#[derive(Debug, Copy, Clone)]
pub struct M0046CreateLocalUploadHistory;
impl migrations::Migration for M0046CreateLocalUploadHistory {
const APP_NAME: &'static str = "furumusic";
const MIGRATION_NAME: &'static str = "m_0046_create_local_upload_history";
const DEPENDENCIES: &'static [migrations::MigrationDependency] =
&[migrations::MigrationDependency::migration(
"furumusic",
"m_0045_create_youtube_downloads",
)];
const OPERATIONS: &'static [Operation] =
&[Operation::custom(create_local_upload_history).build()];
}
pub const MIGRATIONS: &[&SyncDynMigration] = &[
&M0006CreateMediaFile,
&M0007CreateArtist,
@@ -2604,5 +2750,7 @@ pub mod db_migrations {
&M0042RepairLegacyListenQualification,
&M0043CreateSimilarityEmbeddings,
&M0044AddSimilarityRoutingSignature,
&M0045CreateYouTubeDownloads,
&M0046CreateLocalUploadHistory,
];
}
src/player/mod.rs
+484 -15
View File
@@ -21,8 +21,10 @@ use crate::auth;
use crate::config::AppConfig;
use crate::i18n::Translations;
use crate::lastfm::{LastfmClient, LastfmCredentials, LastfmTrackPayload};
use crate::local_uploads::LocalUploadDto;
use crate::scheduler::SchedulerHandle;
use crate::torrents::{TorrentPreviewRequest, TorrentService, TorrentStartRequest};
use crate::youtube::{YouTubePreviewRequest, YouTubeService, YouTubeStartRequest};
mod dto;
mod helpers;
@@ -50,8 +52,7 @@ fn json_error(status: StatusCode, message: &str) -> cot::response::Response {
#[derive(serde::Serialize)]
struct LocalUploadResponse {
ok: bool,
filename: String,
size: u64,
upload: LocalUploadDto,
}
const PLAYER_DEVICE_TTL_MS: i64 = 30_000;
@@ -4830,6 +4831,7 @@ async fn local_upload_handler(
session: Session,
db: Database,
config: AppConfig,
pool: &sqlx::PgPool,
scheduler_handle: Arc<tokio::sync::OnceCell<Arc<SchedulerHandle>>>,
request: cot::request::Request,
) -> cot::Result<cot::http::Response<Body>> {
@@ -4862,6 +4864,14 @@ async fn local_upload_handler(
.filter(|value| !value.is_empty())
.unwrap_or_else(|| "upload.mp3".to_string());
let filename = sanitize_upload_filename(&original_name);
let upload_id_header = HeaderName::from_static("x-furumusic-upload-id");
let upload_id = request
.headers()
.get(upload_id_header)
.and_then(|value| value.to_str().ok())
.and_then(|value| uuid::Uuid::parse_str(value.trim()).ok())
.unwrap_or_else(uuid::Uuid::new_v4)
.to_string();
let bytes = request
.into_body()
@@ -4878,14 +4888,53 @@ async fn local_upload_handler(
let upload_dir = inbox_root
.join("user_uploads")
.join(user.id.to_string())
.join(format!("local-{}", uuid::Uuid::new_v4()));
tokio::fs::create_dir_all(&upload_dir)
.await
.map_err(|err| cot::Error::internal(err.to_string()))?;
.join(format!("local-{upload_id}"));
let destination = upload_dir.join(&filename);
tokio::fs::write(&destination, &bytes)
.await
.map_err(|err| cot::Error::internal(err.to_string()))?;
let Some(inbox_path) =
crate::media_paths::path_for_root(&inbox_root.to_string_lossy(), &destination)
else {
return Ok(json_error(
StatusCode::INTERNAL_SERVER_ERROR,
"upload destination escaped agent_inbox_dir",
));
};
if let Err(err) = crate::local_uploads::create(
pool,
&upload_id,
user.id,
&filename,
bytes.len() as u64,
&inbox_path,
)
.await
{
return Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()));
}
let temporary = upload_dir.join(format!(".{upload_id}.uploading"));
let write_result = async {
tokio::fs::create_dir_all(&upload_dir).await?;
tokio::fs::write(&temporary, &bytes).await?;
tokio::fs::rename(&temporary, &destination).await?;
Ok::<(), std::io::Error>(())
}
.await;
if let Err(err) = write_result {
let message = format!("could not save uploaded file: {err}");
let _ = tokio::fs::remove_dir_all(&upload_dir).await;
let _ = crate::local_uploads::mark_failed(pool, &upload_id, user.id, &message).await;
return Ok(json_error(StatusCode::INTERNAL_SERVER_ERROR, &message));
}
let upload = match crate::local_uploads::mark_queued(pool, &upload_id, user.id).await {
Ok(upload) => upload,
Err(err) => {
return Ok(json_error(
StatusCode::INTERNAL_SERVER_ERROR,
&err.to_string(),
));
}
};
if let Some(handle) = scheduler_handle.get() {
let handle = Arc::clone(handle);
@@ -4896,12 +4945,39 @@ async fn local_upload_handler(
});
}
Json(LocalUploadResponse {
ok: true,
filename,
size: bytes.len() as u64,
})
.into_response()
Json(LocalUploadResponse { ok: true, upload }).into_response()
}
async fn local_upload_history_handler(
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
pool: &sqlx::PgPool,
) -> cot::Result<cot::response::Response> {
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
let (config, _) = AppConfig::load_with_db(&db).await;
match crate::local_uploads::list(pool, user.id, &config.agent_inbox_dir).await {
Ok(items) => Json(items).into_response(),
Err(err) => Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string())),
}
}
async fn local_upload_history_remove_handler(
auth_ctx: auth::AuthContext,
session: Session,
db: Database,
pool: &sqlx::PgPool,
path: Path<PathStringId>,
) -> cot::Result<cot::response::Response> {
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
match crate::local_uploads::remove(pool, user.id, &path.0.id).await {
Ok(()) => Json(serde_json::json!({ "ok": true })).into_response(),
Err(err) => Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string())),
}
}
fn sanitize_upload_filename(value: &str) -> String {
@@ -8107,6 +8183,8 @@ impl App for PlayerApp {
let pool: Arc<tokio::sync::OnceCell<sqlx::PgPool>> = Arc::new(tokio::sync::OnceCell::new());
let torrent_service: Arc<tokio::sync::OnceCell<Arc<TorrentService>>> =
Arc::new(tokio::sync::OnceCell::new());
let youtube_service: Arc<tokio::sync::OnceCell<Arc<YouTubeService>>> =
Arc::new(tokio::sync::OnceCell::new());
let device_hub = Arc::clone(&self.device_hub);
Router::with_urls([
@@ -8345,6 +8423,329 @@ impl App for PlayerApp {
},
"player_agent_queue",
),
// -- YouTube downloads --
Route::with_handler_and_name(
"/youtube/preview",
{
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&self.scheduler_handle);
post(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
json: Json<YouTubePreviewRequest>| {
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&scheduler_handle);
async move {
if auth::get_request_user(&auth_ctx, &session, &db)
.await
.is_none()
{
return Ok(json_error(
StatusCode::UNAUTHORIZED,
"not authenticated",
));
}
let service = youtube_service
.get_or_init(|| async {
Arc::new(YouTubeService::new(Arc::clone(&scheduler_handle)))
})
.await;
match service.preview(json.0).await {
Ok(preview) => Json(preview).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
}
}
}
},
)
},
"player_youtube_preview",
),
Route::with_handler_and_name(
"/youtube",
{
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&self.scheduler_handle);
get(
move |auth_ctx: auth::AuthContext, session: Session, db: Database| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&scheduler_handle);
async move {
let Some(user) =
auth::get_request_user(&auth_ctx, &session, &db).await
else {
return Ok(json_error(
StatusCode::UNAUTHORIZED,
"not authenticated",
));
};
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
let service = youtube_service
.get_or_init(|| async {
Arc::new(YouTubeService::new(Arc::clone(&scheduler_handle)))
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
match service
.list(pg_pool, user.id, &live_config.agent_inbox_dir)
.await
{
Ok(items) => Json(items).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
}
}
}
},
)
},
"player_youtube_list",
),
Route::with_handler_and_name(
"/youtube/start",
{
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&self.scheduler_handle);
post(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
json: Json<YouTubeStartRequest>| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&scheduler_handle);
async move {
let Some(user) =
auth::get_request_user(&auth_ctx, &session, &db).await
else {
return Ok(json_error(
StatusCode::UNAUTHORIZED,
"not authenticated",
));
};
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
let service = youtube_service
.get_or_init(|| async {
Arc::new(YouTubeService::new(Arc::clone(&scheduler_handle)))
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
match service
.start(
pg_pool,
user.id,
json.0,
&live_config.agent_inbox_dir,
)
.await
{
Ok(job) => Json(job).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
}
}
}
},
)
},
"player_youtube_start",
),
Route::with_handler_and_name(
"/youtube/{id}/retry",
{
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&self.scheduler_handle);
post(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PathStringId>| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&scheduler_handle);
async move {
let Some(user) =
auth::get_request_user(&auth_ctx, &session, &db).await
else {
return Ok(json_error(
StatusCode::UNAUTHORIZED,
"not authenticated",
));
};
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
let service = youtube_service
.get_or_init(|| async {
Arc::new(YouTubeService::new(Arc::clone(&scheduler_handle)))
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
match service
.retry(
pg_pool,
user.id,
&path.0.id,
&live_config.agent_inbox_dir,
)
.await
{
Ok(job) => Json(job).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
}
}
}
},
)
},
"player_youtube_retry",
),
Route::with_handler_and_name(
"/youtube/{id}/cancel",
{
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&self.scheduler_handle);
post(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PathStringId>| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&scheduler_handle);
async move {
let Some(user) =
auth::get_request_user(&auth_ctx, &session, &db).await
else {
return Ok(json_error(
StatusCode::UNAUTHORIZED,
"not authenticated",
));
};
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
let service = youtube_service
.get_or_init(|| async {
Arc::new(YouTubeService::new(Arc::clone(&scheduler_handle)))
})
.await;
match service.cancel(pg_pool, user.id, &path.0.id).await {
Ok(job) => Json(job).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
}
}
}
},
)
},
"player_youtube_cancel",
),
Route::with_handler_and_name(
"/youtube/{id}",
{
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&self.scheduler_handle);
delete(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PathStringId>| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let youtube_service = Arc::clone(&youtube_service);
let scheduler_handle = Arc::clone(&scheduler_handle);
async move {
let Some(user) =
auth::get_request_user(&auth_ctx, &session, &db).await
else {
return Ok(json_error(
StatusCode::UNAUTHORIZED,
"not authenticated",
));
};
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
let service = youtube_service
.get_or_init(|| async {
Arc::new(YouTubeService::new(Arc::clone(&scheduler_handle)))
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
match service
.remove(
pg_pool,
user.id,
&path.0.id,
&live_config.agent_inbox_dir,
)
.await
{
Ok(()) => Json(serde_json::json!({ "ok": true })).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
}
}
}
},
)
},
"player_youtube_remove",
),
// -- Torrent import widget --
Route::with_handler_and_name(
"/torrents",
@@ -8546,20 +8947,34 @@ impl App for PlayerApp {
Route::with_handler_and_name(
"/uploads/local",
{
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let scheduler_handle = Arc::clone(&self.scheduler_handle);
post(
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
request: cot::request::Request| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
let scheduler_handle = Arc::clone(&scheduler_handle);
async move {
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
local_upload_handler(
auth_ctx,
session,
db,
live_config,
pg_pool,
scheduler_handle,
request,
)
@@ -8570,6 +8985,60 @@ impl App for PlayerApp {
},
"player_local_upload",
),
Route::with_handler_and_name(
"/uploads/local/history",
get({
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
move |auth_ctx: auth::AuthContext, session: Session, db: Database| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
async move {
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
local_upload_history_handler(auth_ctx, session, db, pg_pool).await
}
}
}),
"player_local_upload_history",
),
Route::with_handler_and_name(
"/uploads/local/history/{id}",
delete({
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
move |auth_ctx: auth::AuthContext,
session: Session,
db: Database,
path: Path<PathStringId>| {
let pool = Arc::clone(&pool);
let pool_config = Arc::clone(&pool_config);
async move {
let pg_pool = pool
.get_or_init(|| async {
sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&pool_config.database_url)
.await
.expect("player pool")
})
.await;
local_upload_history_remove_handler(
auth_ctx, session, db, pg_pool, path,
)
.await
}
}
}),
"player_local_upload_history_remove",
),
Route::with_handler_and_name(
"/uploads/tracks",
get({
src/similarity.rs
+108
View File
@@ -7,6 +7,7 @@
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, OnceLock, RwLock};
use std::time::{Duration, Instant};
@@ -1194,6 +1195,23 @@ fn decode_mono_window(
path: &Path,
start_seconds: f64,
length_seconds: Option<f64>,
) -> Result<Vec<f32>> {
match decode_mono_window_native(path, start_seconds, length_seconds) {
Ok(samples) => Ok(samples),
Err(native_error) => decode_mono_window_ffmpeg(path, start_seconds, length_seconds)
.with_context(|| {
format!(
"native decoder failed for {} ({native_error:#}); FFmpeg fallback failed",
path.display()
)
}),
}
}
fn decode_mono_window_native(
path: &Path,
start_seconds: f64,
length_seconds: Option<f64>,
) -> Result<Vec<f32>> {
let file = File::open(path).with_context(|| format!("opening {}", path.display()))?;
let mut decoder =
@@ -1229,6 +1247,64 @@ fn decode_mono_window(
Ok(resample_sinc(&mono, source_rate, SAMPLE_RATE))
}
fn decode_mono_window_ffmpeg(
path: &Path,
start_seconds: f64,
length_seconds: Option<f64>,
) -> Result<Vec<f32>> {
let mut command = Command::new("ffmpeg");
command.arg("-v").arg("error").arg("-nostdin");
if start_seconds > 0.0 {
command.arg("-ss").arg(format!("{start_seconds:.6}"));
}
command.arg("-i").arg(path);
if let Some(length_seconds) = length_seconds {
command.arg("-t").arg(format!("{length_seconds:.6}"));
}
let output = command
.arg("-map")
.arg("0:a:0")
.arg("-vn")
.arg("-sn")
.arg("-dn")
.arg("-ac")
.arg("1")
.arg("-ar")
.arg(SAMPLE_RATE.to_string())
.arg("-f")
.arg("f32le")
.arg("pipe:1")
.output()
.with_context(|| "starting FFmpeg; install FFmpeg to decode this audio format")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
let detail = stderr
.lines()
.map(str::trim)
.rfind(|line| !line.is_empty())
.unwrap_or("unknown FFmpeg error");
anyhow::bail!("FFmpeg exited with {}: {detail}", output.status);
}
anyhow::ensure!(
output
.stdout
.len()
.is_multiple_of(std::mem::size_of::<f32>()),
"FFmpeg returned a truncated f32le stream"
);
let samples: Vec<f32> = output
.stdout
.chunks_exact(std::mem::size_of::<f32>())
.map(|bytes| f32::from_le_bytes(bytes.try_into().expect("four-byte sample")))
.collect();
anyhow::ensure!(!samples.is_empty(), "FFmpeg decoded track is empty");
anyhow::ensure!(
samples.iter().all(|sample| sample.is_finite()),
"FFmpeg decoded non-finite samples"
);
Ok(samples)
}
fn resample_sinc(input: &[f32], source_rate: usize, target_rate: usize) -> Vec<f32> {
if input.len() < 2 || source_rate == 0 {
return input.to_vec();
@@ -1458,4 +1534,36 @@ mod tests {
assert_eq!(output.len(), 160);
assert!(output.iter().all(|value| (*value - 0.25).abs() < 1e-6));
}
#[test]
fn decodes_opus_with_ffmpeg_fallback() {
if Command::new("ffmpeg").arg("-version").output().is_err() {
return;
}
let path = std::env::temp_dir().join(format!(
"furumusic-similarity-{}.opus",
uuid::Uuid::new_v4()
));
let generated = Command::new("ffmpeg")
.args([
"-v",
"error",
"-f",
"lavfi",
"-i",
"sine=frequency=440:sample_rate=48000:duration=0.25",
"-c:a",
"libopus",
"-y",
])
.arg(&path)
.status()
.expect("start FFmpeg fixture generation");
assert!(generated.success(), "generate Opus fixture");
let decoded = decode_mono_window(&path, 0.0, None).expect("decode Opus with fallback");
let _ = std::fs::remove_file(&path);
assert!(decoded.len() >= SAMPLE_RATE / 5);
assert!(decoded.iter().all(|sample| sample.is_finite()));
}
}
src/youtube.rs
+1786
View File
File diff suppressed because it is too large Load Diff