Added socks proxy, reworked download manager configuration
Build and Publish / Build and Publish Docker Image (push) Successful in 3m41s

This commit is contained in:
Ultradesu
2026-08-14 10:54:56 +01:00
parent 9d9edcfec8
commit 5402d9595d
12 changed files with 854 additions and 72 deletions
+143 -7
View File
@@ -16,7 +16,7 @@ use sqlx::{PgPool, Postgres, QueryBuilder};
use super::BUILD_INFO;
use crate::agent;
use crate::auth::{self, AuthenticatedUser, Role};
use crate::config::{AppConfig, ConfigEntry, ConfigSources};
use crate::config::{AppConfig, ConfigEntry, ConfigSources, DownloadProxy};
use crate::i18n::{I18n, Translations};
use crate::scheduler::{self, JobRegistry, JobRun, ScheduledJob};
@@ -462,6 +462,50 @@ struct AdminSettingsValues {
similarity_profile: String,
#[serde(default = "default_similarity_workers")]
similarity_workers: String,
#[serde(default = "default_true")]
downloads_enabled: bool,
#[serde(default = "default_true")]
torrent_downloads_enabled: bool,
#[serde(default = "default_true")]
youtube_downloads_enabled: bool,
#[serde(default)]
download_proxies: Vec<AdminDownloadProxy>,
#[serde(default)]
torrent_proxy_id: String,
#[serde(default)]
youtube_proxy_id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
struct AdminDownloadProxy {
id: String,
address: String,
#[serde(default)]
username: String,
#[serde(default)]
password: String,
}
impl From<DownloadProxy> for AdminDownloadProxy {
fn from(proxy: DownloadProxy) -> Self {
Self {
id: proxy.id,
address: proxy.address,
username: proxy.username,
password: proxy.password,
}
}
}
impl From<AdminDownloadProxy> for DownloadProxy {
fn from(proxy: AdminDownloadProxy) -> Self {
Self {
id: proxy.id,
address: proxy.address,
username: proxy.username,
password: proxy.password,
}
}
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
@@ -493,6 +537,12 @@ struct AdminSettingsSources {
similarity_model: &'static str,
similarity_profile: &'static str,
similarity_workers: &'static str,
downloads_enabled: &'static str,
torrent_downloads_enabled: &'static str,
youtube_downloads_enabled: &'static str,
download_proxies: &'static str,
torrent_proxy_id: &'static str,
youtube_proxy_id: &'static str,
}
#[derive(Debug, Deserialize)]
@@ -531,6 +581,18 @@ pub(super) struct UpdateSettingsRequest {
similarity_profile: String,
#[serde(default = "default_similarity_workers")]
similarity_workers: String,
#[serde(default = "default_true")]
downloads_enabled: bool,
#[serde(default = "default_true")]
torrent_downloads_enabled: bool,
#[serde(default = "default_true")]
youtube_downloads_enabled: bool,
#[serde(default)]
download_proxies: Vec<AdminDownloadProxy>,
#[serde(default)]
torrent_proxy_id: String,
#[serde(default)]
youtube_proxy_id: String,
}
fn default_similarity_model() -> String {
@@ -980,15 +1042,15 @@ pub async fn update_settings(
if let Err(response) = require_admin_json(&session, &db).await {
return Ok(response);
}
let similarity_model = body.similarity_model.trim();
if crate::similarity::model_by_id(similarity_model).is_none() {
let similarity_model = body.similarity_model.trim().to_string();
if crate::similarity::model_by_id(&similarity_model).is_none() {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"unknown similarity model",
));
}
let similarity_profile = body.similarity_profile.trim();
if crate::similarity::profile_by_id(similarity_profile).is_none() {
let similarity_profile = body.similarity_profile.trim().to_string();
if crate::similarity::profile_by_id(&similarity_profile).is_none() {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"unknown similarity preprocessing profile",
@@ -1003,6 +1065,47 @@ pub async fn update_settings(
));
}
};
if body.download_proxies.len() > 32 {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"at most 32 download proxies can be saved",
));
}
let mut download_proxies = Vec::with_capacity(body.download_proxies.len());
let mut proxy_ids = HashSet::new();
for (index, proxy) in body.download_proxies.into_iter().enumerate() {
let proxy = match DownloadProxy::from(proxy).normalized() {
Ok(proxy) => proxy,
Err(error) => {
return Ok(json_error(
StatusCode::BAD_REQUEST,
&format!("proxy {}: {error}", index + 1),
));
}
};
if !proxy_ids.insert(proxy.id.clone()) {
return Ok(json_error(
StatusCode::BAD_REQUEST,
"download proxy ids must be unique",
));
}
download_proxies.push(proxy);
}
let torrent_proxy_id = body.torrent_proxy_id.trim().to_string();
let youtube_proxy_id = body.youtube_proxy_id.trim().to_string();
for (method, proxy_id) in [
("torrent", torrent_proxy_id.as_str()),
("YouTube", youtube_proxy_id.as_str()),
] {
if !proxy_id.is_empty() && !proxy_ids.contains(proxy_id) {
return Ok(json_error(
StatusCode::BAD_REQUEST,
&format!("selected {method} proxy is not in the saved proxy list"),
));
}
}
let download_proxies_json = serde_json::to_string(&download_proxies)
.map_err(|error| cot::Error::internal(error.to_string()))?;
let fields = [
(
"auth_password_enabled",
@@ -1058,9 +1161,21 @@ pub async fn update_settings(
body.federation_save_on_listen.to_string(),
),
("similarity_enabled", body.similarity_enabled.to_string()),
("similarity_model", similarity_model.to_string()),
("similarity_profile", similarity_profile.to_string()),
("similarity_model", similarity_model),
("similarity_profile", similarity_profile),
("similarity_workers", similarity_workers.to_string()),
("downloads_enabled", body.downloads_enabled.to_string()),
(
"torrent_downloads_enabled",
body.torrent_downloads_enabled.to_string(),
),
(
"youtube_downloads_enabled",
body.youtube_downloads_enabled.to_string(),
),
("download_proxies", download_proxies_json),
("torrent_proxy_id", torrent_proxy_id),
("youtube_proxy_id", youtube_proxy_id),
];
for (key, value) in fields {
let mut entry = ConfigEntry::new(key.to_string(), value);
@@ -1235,6 +1350,15 @@ pub async fn settings_probe(
}
fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto {
let download_proxies = config
.parsed_download_proxies()
.unwrap_or_else(|error| {
tracing::warn!(%error, "ignoring invalid saved download proxy list");
Vec::new()
})
.into_iter()
.map(AdminDownloadProxy::from)
.collect();
AdminSettingsDto {
lastfm_api_key_configured: !config.lastfm_api_key.trim().is_empty(),
lastfm_shared_secret_configured: !config.lastfm_shared_secret.trim().is_empty(),
@@ -1268,6 +1392,12 @@ fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto {
similarity_model: config.similarity_model,
similarity_profile: config.similarity_profile,
similarity_workers: config.similarity_workers.to_string(),
downloads_enabled: config.downloads_enabled,
torrent_downloads_enabled: config.torrent_downloads_enabled,
youtube_downloads_enabled: config.youtube_downloads_enabled,
download_proxies,
torrent_proxy_id: config.torrent_proxy_id,
youtube_proxy_id: config.youtube_proxy_id,
},
sources: AdminSettingsSources {
auth_password_enabled: sources.auth_password_enabled.code(),
@@ -1297,6 +1427,12 @@ fn settings_dto(config: AppConfig, sources: ConfigSources) -> AdminSettingsDto {
similarity_model: sources.similarity_model.code(),
similarity_profile: sources.similarity_profile.code(),
similarity_workers: sources.similarity_workers.code(),
downloads_enabled: sources.downloads_enabled.code(),
torrent_downloads_enabled: sources.torrent_downloads_enabled.code(),
youtube_downloads_enabled: sources.youtube_downloads_enabled.code(),
download_proxies: sources.download_proxies.code(),
torrent_proxy_id: sources.torrent_proxy_id.code(),
youtube_proxy_id: sources.youtube_proxy_id.code(),
},
}
}
+195
View File
@@ -142,6 +142,12 @@ pub struct ConfigSources {
pub similarity_model: ConfigSource,
pub similarity_profile: ConfigSource,
pub similarity_workers: ConfigSource,
pub downloads_enabled: ConfigSource,
pub torrent_downloads_enabled: ConfigSource,
pub youtube_downloads_enabled: ConfigSource,
pub download_proxies: ConfigSource,
pub torrent_proxy_id: ConfigSource,
pub youtube_proxy_id: ConfigSource,
}
impl Default for ConfigSources {
@@ -176,6 +182,12 @@ impl Default for ConfigSources {
similarity_model: ConfigSource::Default,
similarity_profile: ConfigSource::Default,
similarity_workers: ConfigSource::Default,
downloads_enabled: ConfigSource::Default,
torrent_downloads_enabled: ConfigSource::Default,
youtube_downloads_enabled: ConfigSource::Default,
download_proxies: ConfigSource::Default,
torrent_proxy_id: ConfigSource::Default,
youtube_proxy_id: ConfigSource::Default,
}
}
}
@@ -238,6 +250,84 @@ macro_rules! impl_env_overrides {
// AppConfig
// ---------------------------------------------------------------------------
/// Saved SOCKS5 proxy used by user-facing download methods.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DownloadProxy {
pub id: String,
pub address: String,
#[serde(default)]
pub username: String,
#[serde(default)]
pub password: String,
}
impl DownloadProxy {
/// Validate and normalize a proxy without performing network or DNS I/O.
pub fn normalized(mut self) -> anyhow::Result<Self> {
self.id = self.id.trim().to_string();
self.address = self.address.trim().to_string();
if self.id.is_empty() || self.id.len() > 64 {
anyhow::bail!("proxy id must contain from 1 to 64 characters");
}
if !self
.id
.chars()
.all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_'))
{
anyhow::bail!("proxy id may contain only letters, digits, '-' and '_'");
}
if self.address.is_empty() || self.address.len() > 512 {
anyhow::bail!("proxy address must contain a host and port");
}
if self
.address
.chars()
.any(|character| matches!(character, '/' | '?' | '#' | '@'))
{
anyhow::bail!("proxy address must be in host:port format");
}
if self.username.len() > 256 || self.password.len() > 256 {
anyhow::bail!("proxy credentials are too long");
}
let parsed = reqwest::Url::parse(&format!("socks5://{}", self.address))
.map_err(|_| anyhow::anyhow!("proxy address must be in host:port format"))?;
let host = parsed
.host_str()
.filter(|host| !host.trim().is_empty())
.ok_or_else(|| anyhow::anyhow!("proxy address has no host"))?;
let port = parsed
.port()
.ok_or_else(|| anyhow::anyhow!("proxy address has no port"))?;
if port == 0 {
anyhow::bail!("proxy port must be between 1 and 65535");
}
self.address = if host.starts_with('[') && host.ends_with(']') {
format!("{host}:{port}")
} else if host.contains(':') {
format!("[{host}]:{port}")
} else {
format!("{host}:{port}")
};
Ok(self)
}
/// Build the URL accepted by librqbit and yt-dlp. Credentials are included
/// only when both fields are non-empty.
pub fn socks_url(&self) -> anyhow::Result<String> {
let proxy = self.clone().normalized()?;
let mut url = reqwest::Url::parse(&format!("socks5://{}", proxy.address))?;
if !proxy.username.is_empty() && !proxy.password.is_empty() {
url.set_username(&proxy.username)
.map_err(|_| anyhow::anyhow!("invalid proxy username"))?;
url.set_password(Some(&proxy.password))
.map_err(|_| anyhow::anyhow!("invalid proxy password"))?;
}
Ok(url.to_string())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AppConfig {
/// PostgreSQL connection URL.
@@ -301,6 +391,18 @@ pub struct AppConfig {
pub similarity_profile: String,
/// Maximum number of concurrent CPU embedding workers.
pub similarity_workers: u64,
/// Whether the download manager and local-file uploads are available.
pub downloads_enabled: bool,
/// Whether torrent imports are available when the download manager is enabled.
pub torrent_downloads_enabled: bool,
/// Whether YouTube imports are available when the download manager is enabled.
pub youtube_downloads_enabled: bool,
/// JSON-encoded list of [`DownloadProxy`] entries.
pub download_proxies: String,
/// Saved proxy id used for torrent downloads; empty means a direct connection.
pub torrent_proxy_id: String,
/// Saved proxy id used for YouTube downloads; empty means a direct connection.
pub youtube_proxy_id: String,
}
impl Default for AppConfig {
@@ -337,6 +439,13 @@ impl Default for AppConfig {
similarity_workers: std::thread::available_parallelism()
.map(|count| (count.get() / 2).clamp(1, 4) as u64)
.unwrap_or(1),
// Preserve the behavior from before these controls were added.
downloads_enabled: true,
torrent_downloads_enabled: true,
youtube_downloads_enabled: true,
download_proxies: "[]".into(),
torrent_proxy_id: String::new(),
youtube_proxy_id: String::new(),
}
}
}
@@ -372,6 +481,12 @@ impl_env_overrides!(
similarity_model,
similarity_profile,
similarity_workers,
downloads_enabled,
torrent_downloads_enabled,
youtube_downloads_enabled,
download_proxies,
torrent_proxy_id,
youtube_proxy_id,
);
impl AppConfig {
@@ -506,6 +621,39 @@ impl AppConfig {
apply_db_field!(similarity_model);
apply_db_field!(similarity_profile);
apply_db_field!(similarity_workers);
apply_db_field!(downloads_enabled);
apply_db_field!(torrent_downloads_enabled);
apply_db_field!(youtube_downloads_enabled);
apply_db_field!(download_proxies);
apply_db_field!(torrent_proxy_id);
apply_db_field!(youtube_proxy_id);
}
pub fn parsed_download_proxies(&self) -> anyhow::Result<Vec<DownloadProxy>> {
let proxies: Vec<DownloadProxy> = serde_json::from_str(&self.download_proxies)
.map_err(|_| anyhow::anyhow!("saved download proxy list is invalid"))?;
proxies.into_iter().map(DownloadProxy::normalized).collect()
}
pub fn selected_proxy_url(&self, proxy_id: &str) -> anyhow::Result<Option<String>> {
let proxy_id = proxy_id.trim();
if proxy_id.is_empty() {
return Ok(None);
}
let proxy = self
.parsed_download_proxies()?
.into_iter()
.find(|proxy| proxy.id == proxy_id)
.ok_or_else(|| anyhow::anyhow!("selected download proxy is not configured"))?;
proxy.socks_url().map(Some)
}
pub fn torrent_proxy_url(&self) -> anyhow::Result<Option<String>> {
self.selected_proxy_url(&self.torrent_proxy_id)
}
pub fn youtube_proxy_url(&self) -> anyhow::Result<Option<String>> {
self.selected_proxy_url(&self.youtube_proxy_id)
}
}
@@ -532,6 +680,53 @@ mod tests {
crate::similarity::DEFAULT_PROFILE_ID
);
assert!((1..=4).contains(&cfg.similarity_workers));
assert!(cfg.downloads_enabled);
assert!(cfg.torrent_downloads_enabled);
assert!(cfg.youtube_downloads_enabled);
assert!(cfg.parsed_download_proxies().unwrap().is_empty());
}
#[test]
fn download_proxy_url_encodes_complete_credentials() {
let proxy = DownloadProxy {
id: "proxy-1".into(),
address: "proxy.example:1080".into(),
username: "user name".into(),
password: "p@ss:word".into(),
};
assert_eq!(
proxy.socks_url().unwrap(),
"socks5://user%20name:p%40ss%3Aword@proxy.example:1080"
);
}
#[test]
fn download_proxy_url_omits_partial_credentials() {
let proxy = DownloadProxy {
id: "proxy-1".into(),
address: "127.0.0.1:1080".into(),
username: "user".into(),
password: String::new(),
};
assert_eq!(proxy.socks_url().unwrap(), "socks5://127.0.0.1:1080");
}
#[test]
fn selected_download_proxy_is_resolved_by_id() {
let mut cfg = AppConfig::default();
cfg.download_proxies = serde_json::to_string(&[DownloadProxy {
id: "youtube".into(),
address: "[::1]:9050".into(),
username: String::new(),
password: String::new(),
}])
.unwrap();
cfg.youtube_proxy_id = "youtube".into();
assert_eq!(
cfg.youtube_proxy_url().unwrap().as_deref(),
Some("socks5://[::1]:9050")
);
assert_eq!(cfg.torrent_proxy_url().unwrap(), None);
}
#[test]
+7 -1
View File
@@ -90,7 +90,13 @@ async fn index(
return Ok(auth::redirect("/login"));
}
};
let template = player::PlayerPageTemplate { t: i18n.t };
let (config, _) = AppConfig::load_with_db(&db).await;
let template = player::PlayerPageTemplate {
t: i18n.t,
downloads_enabled: config.downloads_enabled,
torrent_downloads_enabled: config.downloads_enabled && config.torrent_downloads_enabled,
youtube_downloads_enabled: config.downloads_enabled && config.youtube_downloads_enabled,
};
Html::new(template.render()?).into_response()
}
+155 -4
View File
@@ -49,6 +49,50 @@ fn json_error(status: StatusCode, message: &str) -> cot::response::Response {
.expect("valid response")
}
#[derive(Debug, Clone, Copy)]
enum DownloadMethod {
LocalFile,
Torrent,
YouTube,
}
fn require_download_method(
config: &AppConfig,
method: DownloadMethod,
) -> Result<(), cot::response::Response> {
let enabled = config.downloads_enabled
&& match method {
DownloadMethod::LocalFile => true,
DownloadMethod::Torrent => config.torrent_downloads_enabled,
DownloadMethod::YouTube => config.youtube_downloads_enabled,
};
if enabled {
Ok(())
} else {
Err(json_error(
StatusCode::FORBIDDEN,
match method {
DownloadMethod::LocalFile => "downloads are disabled by the administrator",
DownloadMethod::Torrent => "torrent downloads are disabled by the administrator",
DownloadMethod::YouTube => "YouTube downloads are disabled by the administrator",
},
))
}
}
fn download_proxy_for(
config: &AppConfig,
method: DownloadMethod,
) -> Result<Option<String>, cot::response::Response> {
require_download_method(config, method)?;
let result = match method {
DownloadMethod::LocalFile => Ok(None),
DownloadMethod::Torrent => config.torrent_proxy_url(),
DownloadMethod::YouTube => config.youtube_proxy_url(),
};
result.map_err(|error| json_error(StatusCode::BAD_REQUEST, &error.to_string()))
}
#[derive(serde::Serialize)]
struct LocalUploadResponse {
ok: bool,
@@ -1375,6 +1419,40 @@ struct LastfmCallbackQuery {
#[template(path = "player.html")]
pub struct PlayerPageTemplate {
pub t: &'static Translations,
pub downloads_enabled: bool,
pub torrent_downloads_enabled: bool,
pub youtube_downloads_enabled: bool,
}
#[cfg(test)]
mod page_template_tests {
use super::*;
use crate::i18n::Lang;
#[test]
fn download_manager_button_follows_the_global_switch() {
let disabled = PlayerPageTemplate {
t: Translations::for_lang(Lang::En),
downloads_enabled: false,
torrent_downloads_enabled: false,
youtube_downloads_enabled: false,
}
.render()
.unwrap();
assert!(!disabled.contains("<button class=\"torrent-import-btn\""));
assert!(disabled.contains("downloadsEnabled: false"));
let enabled = PlayerPageTemplate {
t: Translations::for_lang(Lang::En),
downloads_enabled: true,
torrent_downloads_enabled: true,
youtube_downloads_enabled: true,
}
.render()
.unwrap();
assert!(enabled.contains("<button class=\"torrent-import-btn\""));
assert!(enabled.contains("downloadsEnabled: true"));
}
}
// ---------------------------------------------------------------------------
@@ -4838,6 +4916,9 @@ async fn local_upload_handler(
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
if let Err(response) = require_download_method(&config, DownloadMethod::LocalFile) {
return Ok(response);
}
let inbox_dir = config.agent_inbox_dir.trim();
if inbox_dir.is_empty() {
@@ -4958,6 +5039,9 @@ async fn local_upload_history_handler(
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
let (config, _) = AppConfig::load_with_db(&db).await;
if let Err(response) = require_download_method(&config, DownloadMethod::LocalFile) {
return Ok(response);
}
match crate::local_uploads::list(pool, user.id, &config.agent_inbox_dir).await {
Ok(items) => Json(items).into_response(),
Err(err) => Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string())),
@@ -4974,6 +5058,10 @@ async fn local_upload_history_remove_handler(
let Some(user) = auth::get_request_user(&auth_ctx, &session, &db).await else {
return Ok(json_error(StatusCode::UNAUTHORIZED, "not authenticated"));
};
let (config, _) = AppConfig::load_with_db(&db).await;
if let Err(response) = require_download_method(&config, DownloadMethod::LocalFile) {
return Ok(response);
}
match crate::local_uploads::remove(pool, user.id, &path.0.id).await {
Ok(()) => Json(serde_json::json!({ "ok": true })).into_response(),
Err(err) => Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string())),
@@ -8446,12 +8534,20 @@ impl App for PlayerApp {
"not authenticated",
));
}
let (live_config, _) = AppConfig::load_with_db(&db).await;
let proxy_url = match download_proxy_for(
&live_config,
DownloadMethod::YouTube,
) {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
let service = youtube_service
.get_or_init(|| async {
Arc::new(YouTubeService::new(Arc::clone(&scheduler_handle)))
})
.await;
match service.preview(json.0).await {
match service.preview(json.0, proxy_url.as_deref()).await {
Ok(preview) => Json(preview).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
@@ -8500,8 +8596,20 @@ impl App for PlayerApp {
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
let proxy_url = match download_proxy_for(
&live_config,
DownloadMethod::YouTube,
) {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
match service
.list(pg_pool, user.id, &live_config.agent_inbox_dir)
.list(
pg_pool,
user.id,
&live_config.agent_inbox_dir,
proxy_url,
)
.await
{
Ok(items) => Json(items).into_response(),
@@ -8555,12 +8663,20 @@ impl App for PlayerApp {
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
let proxy_url = match download_proxy_for(
&live_config,
DownloadMethod::YouTube,
) {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
match service
.start(
pg_pool,
user.id,
json.0,
&live_config.agent_inbox_dir,
proxy_url,
)
.await
{
@@ -8615,12 +8731,20 @@ impl App for PlayerApp {
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
let proxy_url = match download_proxy_for(
&live_config,
DownloadMethod::YouTube,
) {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
match service
.retry(
pg_pool,
user.id,
&path.0.id,
&live_config.agent_inbox_dir,
proxy_url,
)
.await
{
@@ -8778,12 +8902,20 @@ impl App for PlayerApp {
.expect("player pool")
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
let proxy_url = match download_proxy_for(
&live_config,
DownloadMethod::Torrent,
) {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
let service = torrent_service
.get_or_init(|| async {
Arc::new(TorrentService::new(Arc::clone(&scheduler_handle)))
})
.await;
match service.list(pg_pool, user.id).await {
match service.list(pg_pool, user.id, proxy_url).await {
Ok(items) => Json(items).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
@@ -8927,12 +9059,23 @@ impl App for PlayerApp {
.expect("player pool")
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
let proxy_url = match download_proxy_for(
&live_config,
DownloadMethod::Torrent,
) {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
let service = torrent_service
.get_or_init(|| async {
Arc::new(TorrentService::new(Arc::clone(&scheduler_handle)))
})
.await;
match service.preview(pg_pool, user.id, json.0).await {
match service
.preview(pg_pool, user.id, json.0, proxy_url.as_deref())
.await
{
Ok(preview) => Json(preview).into_response(),
Err(err) => {
Ok(json_error(StatusCode::BAD_REQUEST, &err.to_string()))
@@ -9284,6 +9427,13 @@ impl App for PlayerApp {
})
.await;
let (live_config, _) = AppConfig::load_with_db(&db).await;
let proxy_url = match download_proxy_for(
&live_config,
DownloadMethod::Torrent,
) {
Ok(proxy_url) => proxy_url,
Err(response) => return Ok(response),
};
let service = torrent_service
.get_or_init(|| async {
Arc::new(TorrentService::new(Arc::clone(&scheduler_handle)))
@@ -9296,6 +9446,7 @@ impl App for PlayerApp {
json.0.selected_files,
live_config.agent_inbox_dir,
user.id,
proxy_url.as_deref(),
)
.await
{
+70 -39
View File
@@ -373,7 +373,8 @@ impl TorrentJob {
pub struct TorrentService {
temp_root: PathBuf,
session: OnceCell<Arc<Session>>,
sessions: Mutex<HashMap<String, Arc<Session>>>,
job_sessions: Mutex<HashMap<String, Arc<Session>>>,
jobs: Mutex<HashMap<String, TorrentJob>>,
resolving_jobs: Mutex<HashSet<String>>,
scheduler_handle: Arc<OnceCell<Arc<SchedulerHandle>>>,
@@ -383,36 +384,47 @@ impl TorrentService {
pub fn new(scheduler_handle: Arc<OnceCell<Arc<SchedulerHandle>>>) -> Self {
Self {
temp_root: std::env::temp_dir().join("furumusic").join("torrents"),
session: OnceCell::new(),
sessions: Mutex::new(HashMap::new()),
job_sessions: Mutex::new(HashMap::new()),
jobs: Mutex::new(HashMap::new()),
resolving_jobs: Mutex::new(HashSet::new()),
scheduler_handle,
}
}
async fn session(&self) -> anyhow::Result<Arc<Session>> {
let temp_root = self.temp_root.clone();
self.session
.get_or_try_init(|| async move {
tokio::fs::create_dir_all(&temp_root).await?;
Session::new_with_opts(
temp_root,
SessionOptions {
disable_upload: true,
enable_upnp_port_forwarding: false,
..Default::default()
},
)
.await
})
.await
.cloned()
async fn session(&self, proxy_url: Option<&str>) -> anyhow::Result<Arc<Session>> {
let key = proxy_url.unwrap_or_default().to_string();
let mut sessions = self.sessions.lock().await;
if let Some(session) = sessions.get(&key) {
return Ok(Arc::clone(session));
}
tokio::fs::create_dir_all(&self.temp_root).await?;
let session = Session::new_with_opts(
self.temp_root.clone(),
SessionOptions {
// SOCKS is intentionally limited to peer TCP and HTTP(S)
// tracker traffic. DHT and other UDP discovery stay direct.
disable_dht: false,
// Sessions are keyed by proxy and can coexist, so they cannot
// safely share one persisted DHT socket configuration.
disable_dht_persistence: true,
disable_upload: true,
enable_upnp_port_forwarding: false,
socks_proxy_url: proxy_url.map(str::to_owned),
..Default::default()
},
)
.await?;
sessions.insert(key, Arc::clone(&session));
Ok(session)
}
pub async fn list(
self: &Arc<Self>,
pool: &PgPool,
user_id: i64,
proxy_url: Option<String>,
) -> anyhow::Result<Vec<TorrentJobDto>> {
let rows = sqlx::query_as::<_, TorrentSessionRow>(
r#"SELECT id, user_id, name, info_hash, source_kind, source_label, torrent_bytes,
@@ -445,6 +457,7 @@ impl TorrentService {
row.id.clone(),
magnet,
row.created_at.clone(),
proxy_url.clone(),
)
.await;
}
@@ -483,8 +496,9 @@ impl TorrentService {
pool: &PgPool,
user_id: i64,
request: TorrentPreviewRequest,
proxy_url: Option<&str>,
) -> anyhow::Result<TorrentSessionDto> {
let session = self.session().await?;
let session = self.session(proxy_url).await?;
let id = Uuid::new_v4().to_string();
let output_dir = self.temp_root.join(&id).join("download");
tokio::fs::create_dir_all(&output_dir).await?;
@@ -511,8 +525,15 @@ impl TorrentService {
.unwrap_or_else(|| info_hash.clone());
let now = now_string();
insert_pending_magnet(pool, &id, user_id, &name, &info_hash, &magnet, &now).await?;
self.spawn_resolve_pending_magnet(pool.clone(), user_id, id.clone(), magnet, now)
.await;
self.spawn_resolve_pending_magnet(
pool.clone(),
user_id,
id.clone(),
magnet,
now,
proxy_url.map(str::to_owned),
)
.await;
let row = load_row(pool, user_id, &id).await?;
return Ok(TorrentSessionDto {
@@ -611,6 +632,7 @@ impl TorrentService {
id: String,
magnet: String,
created_at: String,
proxy_url: Option<String>,
) {
{
let mut resolving = self.resolving_jobs.lock().await;
@@ -622,7 +644,14 @@ impl TorrentService {
let service = Arc::clone(self);
tokio::spawn(async move {
let result = service
.resolve_pending_magnet(&pool, user_id, &id, &magnet, &created_at)
.resolve_pending_magnet(
&pool,
user_id,
&id,
&magnet,
&created_at,
proxy_url.as_deref(),
)
.await;
if let Err(err) = result {
update_resolving_error(&pool, &id, &err.to_string()).await;
@@ -638,8 +667,9 @@ impl TorrentService {
id: &str,
magnet: &str,
created_at: &str,
proxy_url: Option<&str>,
) -> anyhow::Result<()> {
let session = self.session().await?;
let session = self.session(proxy_url).await?;
let output_dir = self.temp_root.join(id).join("download");
tokio::fs::create_dir_all(&output_dir).await?;
let response = tokio::time::timeout(
@@ -743,7 +773,7 @@ impl TorrentService {
jobs.remove(id).and_then(|job| job.handle)
};
if let Some(handle) = removed {
self.stop_torrent(&handle).await;
self.stop_torrent(id, &handle).await;
}
let result =
@@ -766,6 +796,7 @@ impl TorrentService {
selected_files: Vec<usize>,
inbox_dir: String,
uploader_user_id: i64,
proxy_url: Option<&str>,
) -> anyhow::Result<TorrentJobDto> {
if selected_files.is_empty() {
bail!("select at least one file");
@@ -810,7 +841,7 @@ impl TorrentService {
tokio::fs::create_dir_all(&output_dir).await?;
mark_job_started(pool, id, &selected_files, &self.memory_job_dto(id).await?).await?;
let session = self.session().await?;
let session = self.session(proxy_url).await?;
let response = match session
.add_torrent(
AddTorrent::from_bytes(torrent_bytes),
@@ -838,6 +869,10 @@ impl TorrentService {
return Err(err);
}
};
self.job_sessions
.lock()
.await
.insert(id.to_string(), Arc::clone(&session));
let dto = {
let mut jobs = self.jobs.lock().await;
@@ -856,7 +891,7 @@ impl TorrentService {
if service.is_paused(&id).await {
return;
}
service.stop_torrent(&handle).await;
service.stop_torrent(&id, &handle).await;
service.fail_job(&pool, &id, err.to_string()).await;
crate::metrics::record_torrent_download(
"failed",
@@ -865,7 +900,7 @@ impl TorrentService {
);
return;
}
service.stop_torrent(&handle).await;
service.stop_torrent(&id, &handle).await;
if let Err(err) = service
.finalize_completed(&pool, &id, &inbox_dir, uploader_user_id)
.await
@@ -911,7 +946,7 @@ impl TorrentService {
persist_progress(pool, &dto).await?;
if let Some(handle) = handle {
self.stop_torrent(&handle).await;
self.stop_torrent(id, &handle).await;
}
Ok(dto)
}
@@ -981,16 +1016,12 @@ impl TorrentService {
}
}
async fn stop_torrent(&self, handle: &Arc<ManagedTorrent>) {
match self.session().await {
Ok(session) => {
if let Err(err) = session.delete(handle.id().into(), false).await {
tracing::warn!("failed to stop completed torrent: {err}");
}
}
Err(err) => {
tracing::warn!("failed to access torrent session for shutdown: {err}");
}
async fn stop_torrent(&self, id: &str, handle: &Arc<ManagedTorrent>) {
let session = self.job_sessions.lock().await.remove(id);
if let Some(session) = session
&& let Err(err) = session.delete(handle.id().into(), false).await
{
tracing::warn!("failed to stop completed torrent: {err}");
}
}
+62 -15
View File
@@ -220,9 +220,10 @@ impl YouTubeService {
pub async fn preview(
&self,
request: YouTubePreviewRequest,
proxy_url: Option<&str>,
) -> anyhow::Result<YouTubePreviewDto> {
let url = validate_youtube_url(&request.url)?;
let resolved = resolve_source(&url).await?;
let resolved = resolve_source(&url, proxy_url).await?;
let requested_video_id = requested_video_id(&url);
let select_requested_only = resolved.kind == "playlist" && requested_video_id.is_some();
Ok(YouTubePreviewDto {
@@ -252,6 +253,7 @@ impl YouTubeService {
user_id: i64,
request: YouTubeStartRequest,
inbox_dir: &str,
proxy_url: Option<String>,
) -> anyhow::Result<YouTubeJobDto> {
let url = validate_youtube_url(&request.url)?;
validate_inbox_dir(inbox_dir)?;
@@ -268,7 +270,7 @@ impl YouTubeService {
bail!("YouTube selection contains an invalid video ID");
}
let resolved = resolve_source(&url).await?;
let resolved = resolve_source(&url, proxy_url.as_deref()).await?;
let selected_items: Vec<ResolvedItem> = resolved
.items
.into_iter()
@@ -339,7 +341,7 @@ impl YouTubeService {
}
transaction.commit().await?;
self.spawn_job(pool.clone(), id.clone(), inbox_dir.to_string())
self.spawn_job(pool.clone(), id.clone(), inbox_dir.to_string(), proxy_url)
.await;
load_job_dto(pool, user_id, &id).await
}
@@ -349,6 +351,7 @@ impl YouTubeService {
pool: &PgPool,
user_id: i64,
inbox_dir: &str,
proxy_url: Option<String>,
) -> anyhow::Result<Vec<YouTubeJobDto>> {
validate_inbox_dir(inbox_dir)?;
sync_ai_statuses(pool, user_id).await?;
@@ -364,7 +367,7 @@ impl YouTubeService {
.fetch_all(pool)
.await?;
for (id, _) in resumable {
self.spawn_job(pool.clone(), id, inbox_dir.to_string())
self.spawn_job(pool.clone(), id, inbox_dir.to_string(), proxy_url.clone())
.await;
}
@@ -393,6 +396,7 @@ impl YouTubeService {
user_id: i64,
id: &str,
inbox_dir: &str,
proxy_url: Option<String>,
) -> anyhow::Result<YouTubeJobDto> {
validate_inbox_dir(inbox_dir)?;
let job = load_job_row(pool, user_id, id).await?;
@@ -423,8 +427,13 @@ impl YouTubeService {
.execute(pool)
.await?;
self.spawn_job(pool.clone(), id.to_string(), inbox_dir.to_string())
.await;
self.spawn_job(
pool.clone(),
id.to_string(),
inbox_dir.to_string(),
proxy_url,
)
.await;
load_job_dto(pool, user_id, id).await
}
@@ -507,7 +516,13 @@ impl YouTubeService {
Ok(())
}
async fn spawn_job(self: &Arc<Self>, pool: PgPool, id: String, inbox_dir: String) {
async fn spawn_job(
self: &Arc<Self>,
pool: PgPool,
id: String,
inbox_dir: String,
proxy_url: Option<String>,
) {
{
let mut running = self.running_jobs.lock().await;
if !running.insert(id.clone()) {
@@ -527,7 +542,9 @@ impl YouTubeService {
_ = cancel.cancelled() => None,
};
let result = if let Some(permit) = permit {
let result = service.run_job(&pool, &id, &inbox_dir, &cancel).await;
let result = service
.run_job(&pool, &id, &inbox_dir, proxy_url.as_deref(), &cancel)
.await;
drop(permit);
result
} else {
@@ -554,6 +571,7 @@ impl YouTubeService {
pool: &PgPool,
id: &str,
inbox_dir: &str,
proxy_url: Option<&str>,
cancel: &CancellationToken,
) -> anyhow::Result<()> {
if cancel.is_cancelled() {
@@ -576,7 +594,7 @@ impl YouTubeService {
return Ok(());
}
set_parent_status(pool, id, "resolving", None).await?;
let resolved = resolve_source(&job.source_url).await?;
let resolved = resolve_source(&job.source_url, proxy_url).await?;
if cancel.is_cancelled() {
return Ok(());
}
@@ -643,7 +661,7 @@ impl YouTubeService {
continue;
}
if let Err(err) = self
.process_item(pool, &job, &item, &inbox_root, cancel)
.process_item(pool, &job, &item, &inbox_root, proxy_url, cancel)
.await
{
if cancel.is_cancelled() {
@@ -673,6 +691,7 @@ impl YouTubeService {
job: &YouTubeJobRow,
item: &YouTubeItemRow,
inbox_root: &Path,
proxy_url: Option<&str>,
cancel: &CancellationToken,
) -> anyhow::Result<()> {
if cancel.is_cancelled() {
@@ -683,7 +702,7 @@ impl YouTubeService {
let stage = staging_item_root(inbox_root, &job.id, &item.id);
tokio::fs::create_dir_all(&stage).await?;
run_ytdlp_download(pool, &item.id, &item.source_url, &stage, cancel).await?;
run_ytdlp_download(pool, &item.id, &item.source_url, &stage, proxy_url, cancel).await?;
if cancel.is_cancelled() {
bail!("YouTube import cancelled");
@@ -746,8 +765,8 @@ impl YouTubeService {
}
}
async fn resolve_source(url: &str) -> anyhow::Result<ResolvedSource> {
let mut command = base_ytdlp_command();
async fn resolve_source(url: &str, proxy_url: Option<&str>) -> anyhow::Result<ResolvedSource> {
let mut command = base_ytdlp_command(proxy_url);
command
.arg("--flat-playlist")
.arg("--dump-single-json")
@@ -816,7 +835,7 @@ async fn resolve_source(url: &str) -> anyhow::Result<ResolvedSource> {
})
}
fn base_ytdlp_command() -> Command {
fn base_ytdlp_command(proxy_url: Option<&str>) -> Command {
let mut command = Command::new("yt-dlp");
command
.arg("--no-config")
@@ -825,6 +844,9 @@ fn base_ytdlp_command() -> Command {
.arg("--js-runtimes")
.arg("deno")
.stdin(Stdio::null());
if let Some(proxy_url) = proxy_url {
command.arg("--proxy").arg(proxy_url);
}
command
}
@@ -833,9 +855,10 @@ async fn run_ytdlp_download(
item_id: &str,
url: &str,
stage: &Path,
proxy_url: Option<&str>,
cancel: &CancellationToken,
) -> anyhow::Result<()> {
let mut command = base_ytdlp_command();
let mut command = base_ytdlp_command(proxy_url);
command
.arg("--no-playlist")
.arg("--continue")
@@ -1767,6 +1790,30 @@ mod tests {
assert_eq!(sanitize_component("..."), "YouTube audio");
}
#[test]
fn ytdlp_command_receives_selected_proxy() {
let command = base_ytdlp_command(Some("socks5://user:pass@127.0.0.1:1080/"));
let args: Vec<String> = command
.as_std()
.get_args()
.map(|argument| argument.to_string_lossy().into_owned())
.collect();
assert!(args.windows(2).any(|pair| {
pair == [
"--proxy".to_string(),
"socks5://user:pass@127.0.0.1:1080/".to_string(),
]
}));
let direct = base_ytdlp_command(None);
assert!(
direct
.as_std()
.get_args()
.all(|argument| argument != "--proxy")
);
}
#[test]
fn extracts_explicit_video_from_playlist_links() {
assert_eq!(