Add cover downloader and update dependencies

Introduced a new cover_downloader module in the downloader core. Updated several dependencies in Cargo.toml and Cargo.lock, including major upgrades to image processing, HTTP, and async libraries. Adjusted related code in util, download, plugin, and uploader modules to support new features and dependency changes.
This commit is contained in:
ForgQi committed 2025-12-15 00:41:34 +08:00
1 parent 175be7625e
commit b57dc29e96
16 files changed
+658 -585

No files matched your search

Generated
+332 -456
View File
File diff suppressed because it is too large. Load diff
+3 -1
View File
@@ -41,7 +41,9 @@
## 📜 使用文档
B 站命令行投稿工具,支持**短信登录**、**账号密码登录**、**扫码登录**、**浏览器登录**以及**网页Cookie登录**,并将登录后返回的 cookie 和 token 保存在 `cookie.json` 中,可用于其他项目。
获取命令帮助 `biliup --help` \
- 下载 Release: [biliupR](https://github.com/biliup/biliup/releases/latest)
- 获取命令帮助 `biliup --help`
**文档地址**:<https://biliup.github.io/biliup-rs>
```shell
Upload video to bilibili.
+2 -3
View File
@@ -27,9 +27,9 @@ time = { workspace = true }
glob = "0.3.2"
qrcode = "0.14"
indicatif = "0.17.11"
indicatif = "0.18.0"
image = "0.25"
dialoguer = "0.11.0"
dialoguer = "0.12.0"
chrono = { workspace = true, features = ["serde"] }
indexmap = { version = "2.8.0" }
async-trait = { workspace = true }
@@ -40,7 +40,6 @@ reqwest = { workspace = true }
sqlx = { version = "0.8", features = [ "runtime-tokio-rustls", "sqlite" ] }
regex = { workspace = true }
serde_yaml = { workspace = true }
toml = "0.8"
tempfile = "3.14"
rust-embed = "8.7.2"
mime_guess = "2.0.5"
+1 -1
View File
@@ -1,8 +1,8 @@
pub mod cli;
pub mod downloader;
pub mod server;
pub mod uploader;
pub mod upload_lock;
pub mod uploader;
// use crate::server::api::router::ApplicationController;
use crate::server::app::ApplicationController;
+43 -74
View File
@@ -1,5 +1,7 @@
use crate::server::common::upload::UploaderMessage;
use crate::server::common::util::FileValidator;
use crate::server::config::default_segment_time;
use crate::server::core::downloader::cover_downloader;
use crate::server::core::downloader::{
DanmakuClient, DownloadConfig, DownloadStatus, DownloaderRuntime, DownloaderType, SegmentEvent,
SegmentInfo,
@@ -11,6 +13,7 @@ use crate::server::infrastructure::context::{Context, Stage, WorkerStatus};
use crate::server::infrastructure::models::hook_step::process;
use async_channel::{Receiver, Sender};
use error_stack::{ResultExt, bail};
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -37,68 +40,6 @@ impl RetryPolicy {
}
}
/// 文件验证配置
#[derive(Clone)]
pub struct FileValidator {
min_size: u64,
check_format: bool,
}
impl FileValidator {
pub fn new(min_size: u64, check_format: bool) -> Self {
Self {
min_size,
check_format,
}
}
}
impl Default for FileValidator {
fn default() -> Self {
Self {
min_size: 1024 * 1024 * 100, // 100MB minimum
check_format: true,
}
}
}
impl FileValidator {
/// 验证文件有效性
pub fn validate(&self, path: &Path) -> AppResult<()> {
let metadata = fs::metadata(path).change_context(AppError::Unknown)?;
let size = metadata.len();
if size < self.min_size {
bail!(AppError::Custom(format!(
"File {} too small: {size} bytes, minimum: {} bytes",
path.display(),
self.min_size
)));
}
// 可选:检查文件格式
if self.check_format {
self.validate_format(path)?;
}
Ok(())
}
fn validate_format(&self, path: &Path) -> AppResult<()> {
// 简单的格式验证 - 检查扩展名
if let Some(extension) = path.extension() {
let ext = extension.to_string_lossy().to_lowercase();
match ext.as_str() {
"mp4" | "flv" | "ts" | "m3u8" | "mkv" => Ok(()),
_ => bail!(AppError::Custom(format!("Unsupported format: {}", ext))),
}
} else {
bail!(AppError::Custom("No file extension found".to_string()))
}
}
}
/// 分段事件处理器
pub struct SegmentEventProcessor {
channel: Option<Sender<SegmentInfo>>,
@@ -181,7 +122,7 @@ impl DownloadTask {
pub(self) async fn execute(
&self,
ctx: &Context,
mut processor: SegmentEventProcessor,
sender: Sender<UploaderMessage>,
mut plugin: Box<dyn DownloadBase>,
rooms_handle: Arc<Monitor>,
) -> AppResult<()> {
@@ -201,12 +142,15 @@ impl DownloadTask {
info!("Starting danmaku client for stream: {}", url);
client.download().await?;
}
// 初始化组件
let mut processor = SegmentEventProcessor::new(sender, ctx.clone());
let result = loop {
// 创建守卫确保清理
// 创建事件处理器
// 执行下载
let components = self
.initialize_components(
.download(
&mut processor,
ctx.clone(),
danmaku_client.clone(),
@@ -286,7 +230,7 @@ impl DownloadTask {
Ok(())
}
async fn initialize_components(
async fn download(
&self,
processor: &mut SegmentEventProcessor,
ctx: Context,
@@ -395,22 +339,47 @@ impl DActor {
match msg {
DownloaderMessage::Start(downloader, ctx) => {
// 创建下载任务
let runtime = downloader.downloader(
ctx.config()
.downloader
.unwrap_or(DownloaderType::StreamGears),
);
let task = Arc::new(DownloadTask::new(runtime));
// 初始化组件
let processor = SegmentEventProcessor::new(self.sender.clone(), ctx.clone());
let task = Arc::new(DownloadTask::new(
downloader.downloader(
ctx.config()
.downloader
.unwrap_or(DownloaderType::StreamGears),
),
));
// 更新工作器状态为工作中
ctx.change_status(Stage::Download, WorkerStatus::Working(task.clone()))
.await;
tokio::spawn({
let streamer_info = &ctx.stream_info_ext().streamer_info;
let live_cover_url = streamer_info.live_cover_path.clone();
let format_filename = ctx.recorder(streamer_info.clone()).format_filename();
let client = ctx.stateless_client().client.clone();
let enabled = ctx
.config()
.use_live_cover
.map(|u| u && !live_cover_url.is_empty())
.unwrap_or(false);
async move {
cover_downloader::download_cover_with(
&live_cover_url,
enabled,
&format_filename,
client,
)
.await
}
});
process(&[], &ctx.live_streamer().preprocessor).await;
let _ = task
.execute(&ctx, processor, downloader, self.rooms_handle.clone())
.execute(
&ctx,
self.sender.clone(),
downloader,
self.rooms_handle.clone(),
)
.await;
process(&[], &ctx.live_streamer().downloaded_processor).await;
@@ -1,7 +1,12 @@
use crate::server::errors::{AppError, AppResult};
use crate::server::infrastructure::models::StreamerInfo;
use crate::server::infrastructure::models::hook_step::HookStep;
use chrono::{Duration, Local};
use error_stack::{ResultExt, bail};
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::{Path, PathBuf};
use tracing::{error, info};
use url::Url;
/// 录制器配置结构体
@@ -172,3 +177,75 @@ mod tests {
);
}
}
/// 文件验证配置
#[derive(Clone)]
pub struct FileValidator {
min_size: u64,
check_format: bool,
}
impl FileValidator {
pub fn new(min_size: u64, check_format: bool) -> Self {
Self {
min_size,
check_format,
}
}
}
impl Default for FileValidator {
fn default() -> Self {
Self {
min_size: 1024 * 1024 * 100, // 100MB minimum
check_format: true,
}
}
}
impl FileValidator {
/// 验证文件有效性
pub fn validate(&self, path: &Path) -> AppResult<()> {
let metadata = fs::metadata(path).change_context(AppError::Unknown)?;
let size = metadata.len();
if size < self.min_size {
let display = path.display();
let path = path.to_owned();
tokio::spawn(async move {
let Ok(()) = HookStep::remove_file(&[&path])
.await
.inspect_err(|e| error!(e=?e))
else {
return;
};
info!("过滤删除 - {}", path.display());
});
bail!(AppError::Custom(format!(
"File {display} too small: {size} bytes, minimum: {} bytes",
self.min_size
)));
}
// 可选:检查文件格式
if self.check_format {
self.validate_format(path)?;
}
Ok(())
}
fn validate_format(&self, path: &Path) -> AppResult<()> {
// 简单的格式验证 - 检查扩展名
if let Some(extension) = path.extension() {
let ext = extension.to_string_lossy().to_lowercase();
match ext.as_str() {
"mp4" | "flv" | "ts" | "m3u8" | "mkv" => Ok(()),
_ => bail!(AppError::Custom(format!("Unsupported format: {}", ext))),
}
} else {
bail!(AppError::Custom("No file extension found".to_string()))
}
}
}
@@ -1,3 +1,4 @@
pub mod cover_downloader;
/// FFmpeg下载器实现
pub mod ffmpeg_downloader;
/// Stream-gears下载器实现
@@ -0,0 +1,151 @@
use crate::server::core::downloader::cover_downloader;
use crate::server::errors::{AppError, AppResult};
use axum::http::HeaderValue;
use axum::http::header::USER_AGENT;
use error_stack::ResultExt;
use reqwest::header::HeaderMap;
use std::fs;
use std::path::{Path, PathBuf};
use std::time::Duration;
use tracing::{error, info, instrument, warn};
use url::Url;
/// 支持的图片格式
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ImageFormat {
Jpg,
Png,
Webp,
}
impl ImageFormat {
/// 从 URL 路径中检测图片格式
fn detect(url_path: &str) -> Option<Self> {
// 转小写后匹配,更健壮
let path = url_path.to_ascii_lowercase();
[
(".jpg", Self::Jpg),
(".jpeg", Self::Jpg),
(".png", Self::Png),
(".webp", Self::Webp),
]
.into_iter()
.find(|(ext, _)| path.contains(ext))
.map(|(_, fmt)| fmt)
}
const fn as_ext(self) -> &'static str {
match self {
Self::Jpg => "jpg",
Self::Png => "png",
Self::Webp => "webp",
}
}
const fn needs_conversion(self) -> bool {
matches!(self, Self::Webp)
}
}
/// 封面下载器
/// 下载封面图片
#[instrument(fields(url = %url))]
async fn download(
url: &str,
filename: &str,
base_dir: PathBuf,
client: reqwest::Client,
) -> AppResult<PathBuf> {
// 解析并验证 URL
let parsed =
Url::parse(url).change_context_lazy(|| AppError::Custom(String::from("无效的 URL")))?;
let format = ImageFormat::detect(parsed.path())
.ok_or_else(|| AppError::Custom(format!("不支持的图片格式: {url}")))?;
// 构建保存目录
let save_dir = base_dir;
fs::create_dir_all(&save_dir)
.change_context_lazy(|| AppError::Custom(String::from("创建目录失败")))?;
let file_path = save_dir.join(format!("{filename}.{}", format.as_ext()));
// 下载或使用缓存
if !file_path.exists() {
fetch_and_save(url, &file_path, client).await?;
}
// webp 需要转换为 jpg
if format.needs_conversion() {
return convert_to_jpg(&file_path, &save_dir, filename);
}
Ok(file_path)
}
/// 下载并保存文件
async fn fetch_and_save(url: &str, path: &Path, client: reqwest::Client) -> AppResult<()> {
let bytes = client
.get(url)
.send()
.await
.change_context_lazy(|| AppError::Custom(String::from("网络请求失败")))?
.error_for_status()
.change_context_lazy(|| AppError::Custom(String::from("服务器返回错误状态")))?
.bytes()
.await
.change_context_lazy(|| AppError::Custom(String::from("读取响应数据失败")))?;
fs::write(path, &bytes)
.change_context_lazy(|| AppError::Custom(String::from("写入文件失败")))?;
Ok(())
}
/// 将图片转换为 JPG 格式
fn convert_to_jpg(src: &Path, dir: &Path, filename: &str) -> AppResult<PathBuf> {
let jpg_path = dir.join(format!("{filename}.jpg"));
image::open(src)
.change_context_lazy(|| AppError::Custom(String::from("打开图片失败")))?
.to_rgb8()
.save(&jpg_path)
.change_context_lazy(|| AppError::Custom(String::from("保存 JPG 失败")))?;
// 删除原文件,忽略错误
let _ = fs::remove_file(src);
Ok(jpg_path)
}
pub async fn download_cover_with(
url: &str,
enabled: bool,
fmtname: &str,
client: reqwest::Client,
) -> Option<PathBuf> {
// 使用 guard clause 提前返回
if !enabled {
return None;
}
// 构建请求头
let mut headers = HeaderMap::new();
headers.insert(
USER_AGENT,
HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64)"),
);
// 创建下载器
match cover_downloader::download(url, fmtname, PathBuf::from("data/cover"), client).await {
Ok(path) => {
let display_path = fs::canonicalize(&path).unwrap_or_else(|_| path.clone());
info!("封面下载成功,路径:{}", display_path.display());
Some(path)
}
Err(e) => {
error!("封面下载失败: {e:#}");
None
}
}
}
+8 -1
View File
@@ -1,15 +1,22 @@
mod twitch;
pub mod yy;
use crate::server::core::downloader::{DanmakuClient, DownloaderRuntime, DownloaderType};
use crate::server::core::downloader::{
DanmakuClient, DownloaderRuntime, DownloaderType, cover_downloader,
};
use crate::server::errors::AppError;
use crate::server::infrastructure::context::{Context, PluginContext};
use crate::server::infrastructure::models::StreamerInfo;
use async_trait::async_trait;
use axum::http::header::USER_AGENT;
use axum::http::{HeaderMap, HeaderValue};
use error_stack::Report;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use tracing::{error, info};
/// 流信息结构
/// 包含直播流的详细信息
@@ -41,7 +41,7 @@ impl HookStep {
}
HookStep::Remove(cmd) if cmd == "rm" => {
// 删除文件
self.remove_file(video_paths).await?;
HookStep::remove_file(video_paths).await?;
}
HookStep::Remove(cmd) => {
// 未知命令,返回错误
@@ -254,7 +254,7 @@ impl HookStep {
///
/// # 参数
/// * `video_paths` - 要删除的视频文件路径列表
async fn remove_file(&self, video_paths: &[&Path]) -> AppResult<()> {
pub(crate) async fn remove_file(video_paths: &[&Path]) -> AppResult<()> {
// 逐个删除视频文件
for video_path in video_paths {
info!("删除 - Removing: {}", video_path.display());
+5 -4
View File
@@ -59,7 +59,10 @@ impl UploadLock {
warn!("检测到过期锁文件,自动清理: {:?}", self.lock_path);
let _ = fs::remove_file(&self.lock_path);
} else {
info!("检测到其他进程正在使用该账号上传(锁文件: {:?})", self.lock_path);
info!(
"检测到其他进程正在使用该账号上传(锁文件: {:?})",
self.lock_path
);
return Ok(false);
}
}
@@ -78,9 +81,7 @@ impl UploadLock {
self.acquired = true;
Ok(true)
}
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {
Ok(false)
}
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => Ok(false),
Err(e) => Err(e),
}
}
+26 -19
View File
@@ -20,6 +20,7 @@ use indicatif::{ProgressBar, ProgressStyle};
use qrcode::QrCode;
use qrcode::render::unicode;
use reqwest::Body;
use serde::{Deserialize, Serialize};
use std::ffi::OsStr;
use std::io::Seek;
use std::path::{Path, PathBuf};
@@ -28,7 +29,6 @@ use std::sync::{Arc, Mutex};
use std::task::Poll;
use std::time::Instant;
use tracing::{info, warn};
use serde::{Serialize, Deserialize};
// 断点续传的数据结构
#[derive(Serialize, Deserialize, Debug)]
@@ -67,7 +67,8 @@ impl UploadCheckpoint {
fn add_video(&mut self, file_path: &Path, video: Video) {
self.videos.push(video);
self.uploaded_files.push(file_path.to_string_lossy().to_string());
self.uploaded_files
.push(file_path.to_string_lossy().to_string());
}
}
@@ -292,13 +293,15 @@ async fn login_by_cookies(user_cookie: PathBuf, proxy: Option<&str>) -> AppResul
})?,
_ => {
let bili = result.change_context_lazy(|| AppError::Unknown)?;
let info = bili
.my_info()
.await
.change_context_lazy(|| AppError::Unknown)?;
info!(
"user: {}",
bili.my_info()
.await
.change_context_lazy(|| AppError::Unknown)?["data"]["name"]
info["data"]["name"]
.as_str()
.unwrap_or_default()
.ok_or_else(|| AppError::Custom(format!("{info}no name")))?
);
bili
}
@@ -334,7 +337,8 @@ pub async fn upload(
// 生成断点续传文件路径(基于视频列表的哈希)
let checkpoint_filename = format!(
"biliup_checkpoint_{}.json",
video_path.iter()
video_path
.iter()
.map(|p| p.to_string_lossy().to_string())
.collect::<Vec<_>>()
.join("_")
@@ -357,7 +361,10 @@ pub async fn upload(
});
if !checkpoint.uploaded_files.is_empty() {
info!("Found checkpoint with {} uploaded files, resuming...", checkpoint.uploaded_files.len());
info!(
"Found checkpoint with {} uploaded files, resuming...",
checkpoint.uploaded_files.len()
);
}
let mut videos = checkpoint.videos.clone();
@@ -398,8 +405,10 @@ pub async fn upload(
// 使用通用的 retry 函数处理限流错误(code: 601)
// 配合账号级互斥锁防止多进程同时重试
let credential_id = format!("{}", bili.login_info.token_info.mid);
let upload_lock = Arc::new(Mutex::new(UploadLock::new(&credential_id)
.map_err(|e| AppError::Custom(format!("Failed to create upload lock: {}", e)))?));
let upload_lock = Arc::new(Mutex::new(
UploadLock::new(&credential_id)
.map_err(|e| AppError::Custom(format!("Failed to create upload lock: {}", e)))?,
));
// 在开始上传前检查是否有其他进程正在等待限流恢复
{
@@ -408,7 +417,8 @@ pub async fn upload(
return Err(AppError::Custom(format!(
"另一个使用该账号 ({}) 的上传进程正在等待限流恢复,请稍后重试",
credential_id
)).into());
))
.into());
}
}
@@ -459,13 +469,7 @@ pub async fn upload(
}),
)
.await
.map_err(|e| match e {
Kind::RateLimit { code, message } => AppError::Custom(format!(
"Upload rate limit exceeded after retries (code: {}): {}",
code, message
)),
_ => AppError::Unknown,
})?
.change_context_lazy(|| AppError::Custom("after retries".to_owned()))?
};
// 上传成功后释放锁
@@ -506,7 +510,10 @@ pub async fn upload(
if let Err(e) = checkpoint.save(&checkpoint_path) {
warn!("Failed to save checkpoint: {}", e);
} else {
info!("Checkpoint saved: {} files uploaded", checkpoint.uploaded_files.len());
info!(
"Checkpoint saved: {} files uploaded",
checkpoint.uploaded_files.len()
);
}
videos.push(video);
+2 -2
View File
@@ -12,7 +12,7 @@ serde = { workspace = true }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
serde_urlencoded = { workspace = true }
reqwest_cookie_store = "0.8.0"
reqwest_cookie_store = "0.9.0"
tokio = { workspace = true }
tracing = { workspace = true }
rsa = "0.9.8"
@@ -29,7 +29,7 @@ bytes = { workspace = true }
futures = { workspace = true }
bon = { workspace = true }
reqwest-middleware = { version = "0.4", features = ["json"] }
reqwest-retry = "0.7.0"
reqwest-retry = "0.8.0"
clap = { version = "4", features = ["derive"], optional = true }
time = "0.3"
m3u8-rs = "6"
+1 -6
View File
@@ -57,12 +57,7 @@ impl Parcel {
};
if video.title.is_none() {
if let Some(filename) = self
.video_file
.filepath
.file_stem()
.and_then(OsStr::to_str)
{
if let Some(filename) = self.video_file.filepath.file_stem().and_then(OsStr::to_str) {
// B站限制分P视频标题不能超过80字符,需要截断
video.title = Some(if filename.chars().count() >= 80 {
Video::truncate_title(filename, 80)
+1 -15
View File
@@ -34,22 +34,8 @@ biliup-cli = { path = "../biliup-cli"}
url = { workspace = true }
time = { workspace = true }
chrono = { workspace = true }
fancy-regex = { version = "0.16.1"}
fancy-regex = { version = "0.17.0"}
bon = { workspace = true }
clap = { version = "4", features = ["derive"] }
async-trait = { workspace = true }
error-stack = { workspace = true }
axum-login = "0.18.0"
password-auth = "1.0.0"
tower = "0.5.2"
tower-sessions = { version = "0.14.0", default-features = false, features = [
"signed",
] }
tower-sessions-sqlx-store = { version = "0.15.0", features = ["sqlite"] }
# [target.'cfg(all(target_arch="aarch64", target_os="linux"))'.dependencies]
# downloader = { path = "../downloader", default-features = false , features = ["rustls-tls"] }
# [target.'cfg(not(all(target_arch="aarch64", target_os="linux")))'.dependencies]
# downloader = { path = "../downloader" }
+3 -1
View File
@@ -31,6 +31,7 @@ use pythonize::pythonize;
use std::collections::HashMap;
use std::net::ToSocketAddrs;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::{Arc, LazyLock, RwLock};
use time::macros::format_description;
use tracing::{debug, info, warn};
@@ -68,6 +69,7 @@ pub struct PyDownloader {
url: String,
remark: String,
danmaku: Option<Arc<Py<PyAny>>>,
cfg: OnceConfig,
}
@@ -123,7 +125,7 @@ impl PyDownloader {
self_obj.getattr("raw_stream_url")?.extract()?;
let title: String = self_obj.getattr("room_title")?.extract()?;
let live_cover_path: Option<String> =
self_obj.getattr("live_cover_path")?.extract()?;
self_obj.getattr("live_cover_url")?.extract()?;
let _is_download: bool = self_obj.getattr("is_download")?.extract()?;
let platform: String = self_obj.getattr("platform")?.extract()?;