From b5ddff05643cd99b35a5dbe13486d8206f5b0c5e Mon Sep 17 00:00:00 2001 From: ForgQi <34411314+ForgQi@users.noreply.github.com> Date: Sat, 6 Jun 2026 03:22:54 +0800 Subject: [PATCH] Refactor Python package and simplify entry Remove legacy engine and many plugin modules, refactor upload plugins to standalone classes, and simplify package entry points. __init__.py now provides a safe __version__ fallback; __main__.py directly invokes stream_gears.main_loop. BiliWeb/BiliWebAsync plugins were adapted to use a FileInfo NamedTuple and no longer depend on engine.UploadBase/Plugin decorator. Documentation updated (README expanded with new commands and CLI/SKILL integration) and a new SKILL.md added. Workspace version bumped to 1.2.1 in Cargo.toml (lock updated accordingly). --- Cargo.lock | 8 +- Cargo.toml | 2 +- README.md | 80 +++-- SKILL.md | 189 ++++++++++++ biliup/__init__.py | 59 +--- biliup/__main__.py | 30 +- biliup/engine/__init__.py | 15 - biliup/engine/decorators.py | 40 --- biliup/engine/upload.py | 21 -- biliup/plugins/__init__.py | 157 +--------- biliup/plugins/bili_chromeup.py | 353 ---------------------- biliup/plugins/bili_webup.py | 24 +- biliup/plugins/bili_webup_sync.py | 19 +- biliup/plugins/biliuprs.py | 145 --------- biliup/plugins/noop_uploader.py | 11 - biliup/plugins/stream_gears.py | 135 --------- crates/biliup/src/downloader/live/huya.rs | 15 + crates/biliup/src/downloader/util.rs | 12 +- crates/danmaku/examples/e2e_recorder.rs | 70 +++++ crates/danmaku/examples/test_douyu.rs | 4 +- crates/danmaku/src/client.rs | 12 +- crates/danmaku/src/protocols/douyin.rs | 55 +++- crates/danmaku/src/protocols/mod.rs | 18 +- crates/danmaku/src/protocols/twitch.rs | 20 +- crates/danmaku/src/protocols/youtube.rs | 27 +- pyproject.toml | 23 +- 26 files changed, 480 insertions(+), 1064 deletions(-) create mode 100644 SKILL.md delete mode 100644 biliup/engine/__init__.py delete mode 100644 biliup/engine/decorators.py delete mode 100644 biliup/engine/upload.py delete mode 100644 biliup/plugins/bili_chromeup.py delete mode 100644 biliup/plugins/biliuprs.py delete mode 100644 biliup/plugins/noop_uploader.py delete mode 100644 biliup/plugins/stream_gears.py create mode 100644 crates/danmaku/examples/e2e_recorder.rs diff --git a/Cargo.lock b/Cargo.lock index 0f9fc1b..3cb7c9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,7 +387,7 @@ checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" [[package]] name = "biliup" -version = "1.2.0" +version = "1.2.1" dependencies = [ "async-trait", "base64", @@ -424,7 +424,7 @@ dependencies = [ [[package]] name = "biliup-cli" -version = "1.2.0" +version = "1.2.1" dependencies = [ "async-channel", "async-trait", @@ -978,7 +978,7 @@ dependencies = [ [[package]] name = "danmaku" -version = "1.2.0" +version = "1.2.1" dependencies = [ "async-trait", "base64", @@ -3969,7 +3969,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stream-gears" -version = "1.2.0" +version = "1.2.1" dependencies = [ "async-channel", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 2e89929..2e7c902 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ resolver = "3" [workspace.package] -version = "1.2.0" +version = "1.2.1" [workspace.dependencies] serde = { version = "1.0", features = ["derive"] } diff --git a/README.md b/README.md index edbcf60..e44cec8 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ Commands: upload 上传视频 append 是否要对某稿件追加视频 show 打印视频详情 + comments 查看视频评论 + reply 回复视频评论，默认只打印将要回复的内容 dump-flv 输出flv元数据 download 下载视频 server 启动web服务，默认端口19159 @@ -76,12 +78,31 @@ Options: Usage: biliup server [OPTIONS] Options: - -b, --bind Specify bind address [default: 0.0.0.0] - -p, --port Port to use [default: 19159] - --auth 开启登录密码认证 - -h, --help Print help + -b, --bind Specify bind address [default: 0.0.0.0] + -p, --port Port to use [default: 19159] + --auth 开启登录密码认证 + -c, --config 使用 biliup 1.0.7 风格配置文件启动录制 + -h, --help Print help ``` +### 命令行工具 / Skill + +`biliup` 可以作为自动化流程中的命令行工具封装使用。常用调用方式： + +```shell +# 启动 Web 服务 +biliup server --bind 0.0.0.0 --port 19159 --auth + +# 使用 1.0.7 风格配置文件启动录制 +biliup server --config config.toml + +# 查看命令帮助 +biliup --help +biliup server --help +``` + +通过 `npx skills add biliup/biliup` 可为 Agent 添加 `biliup` 命令行能力。添加后可直接让 Agent 执行 Web 服务启动、配置文件启动、命令帮助查看以及常用上传/下载命令。 + - [使用文档 »](https://docs.biliup.rs) ## 🚀 快速开始 @@ -108,55 +129,56 @@ Options: ### 架构概览 -Rust后端 + Python引擎 + Next.js前端的混合架构。 +Rust后端 + 精简 Python 包 + Next.js前端的混合架构。 ```mermaid graph TB subgraph "🌐 前端层" UI[Next.js Web界面
React + TypeScript
Semi UI组件库] end - + subgraph "⚡ Rust后端服务" - CLI[Web API服务器
biliup-cli
用户认证 & REST API] - CORE[核心上传库
biliup
Bilibili API客户端] - GEARS[Python绑定
stream-gears
性能优化桥接] + CLI[命令行与 Web API
biliup-cli
REST API / WebUI / 配置导入] + CORE[核心库
biliup
直播解析 / 下载 / 上传] + DANMAKU[弹幕库
danmaku
多平台协议 / XML输出] + GEARS[Python绑定
stream-gears
python -m biliup 入口] end - - subgraph "🐍 Python引擎" - ENGINE[下载引擎
biliup
任务调度 & 流处理] - PLUGINS[插件系统
20+平台支持
斗鱼/虎牙/Twitch等] - DANMAKU[弹幕系统
实时弹幕获取
多平台协议支持] + + subgraph "🐍 Python包" + PYENTRY[最小入口
biliup.__main__
调用 stream_gears.main_loop] + PYUPLOAD[投稿库
bili_webup / bili_webup_sync
供外部项目调用] end - + subgraph "🗄️ 数据层" DB[(SQLite数据库
配置存储
任务状态 & 日志)] - FILES[文件系统
临时视频存储
缓存管理] + FILES[文件系统
视频分段 / 弹幕XML
缓存与临时文件] end - + subgraph "🌍 外部服务" BILI[Bilibili API
视频上传服务] - STREAMS[直播平台
斗鱼/虎牙/B站等
实时流媒体] + STREAMS[直播平台
B站/斗鱼/虎牙/抖音/Twitch等] end - + UI --> CLI CLI --> CORE - CLI --> ENGINE + CLI --> DANMAKU CLI --> DB - GEARS --> ENGINE - ENGINE --> PLUGINS - ENGINE --> DANMAKU - ENGINE --> FILES + CLI --> FILES + CORE --> STREAMS CORE --> BILI - PLUGINS --> STREAMS DANMAKU --> STREAMS - + DANMAKU --> FILES + GEARS --> CLI + PYENTRY --> GEARS + PYUPLOAD --> BILI + style UI fill:#e1f5fe style CLI fill:#f3e5f5 style CORE fill:#f3e5f5 + style DANMAKU fill:#f3e5f5 style GEARS fill:#f3e5f5 - style ENGINE fill:#e8f5e8 - style PLUGINS fill:#e8f5e8 - style DANMAKU fill:#e8f5e8 + style PYENTRY fill:#e8f5e8 + style PYUPLOAD fill:#e8f5e8 style DB fill:#fff3e0 style FILES fill:#fff3e0 style BILI fill:#ffebee diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..97ca6ad --- /dev/null +++ b/SKILL.md @@ -0,0 +1,189 @@ +--- +name: biliup +description: Use the biliup command-line tool to start the WebUI server, run recording tasks from config files, inspect help, and operate common upload/download commands. +--- + +# biliup + +Use this skill when the user wants to install or operate the `biliup` command-line tool. + +`biliup` can start the WebUI server, run recording tasks from config files, log in, upload videos, append videos, inspect video information, download videos, and list uploaded videos. + +## Install flow + +When the user needs to install `biliup`, choose one installation path based on the user's operating system and preference. + +### GitHub Releases prebuilt package + +Use this path when the user wants to install a prebuilt binary from GitHub Releases. + +For Linux or macOS, generate and run this script. Set `INSTALL_DIR` to the user's target directory when needed. + +```bash +set -euo pipefail + +REPO="biliup/biliup" +INSTALL_DIR="${INSTALL_DIR:-$HOME/.local/bin}" +OS="$(uname -s)" +ARCH="$(uname -m)" + +case "$OS" in + Linux) os="linux" ;; + Darwin) os="macos" ;; + *) echo "Unsupported OS: $OS" >&2; exit 1 ;; +esac + +case "$ARCH" in + x86_64|amd64) arch="x86_64" ;; + aarch64|arm64) arch="aarch64" ;; + armv7l|armv7*) arch="arm" ;; + *) echo "Unsupported architecture: $ARCH" >&2; exit 1 ;; +esac + +asset="$(curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" \ + | python3 -c 'import json,sys; data=json.load(sys.stdin); os=sys.argv[1]; arch=sys.argv[2]; assets=[a["name"] for a in data["assets"]]; matches=[n for n in assets if n.endswith(".tar.xz") and f"-{arch}-{os}.tar.xz" in n]; print(matches[0] if matches else "")' "$os" "$arch")" + +if [ -z "$asset" ]; then + echo "No matching biliup release asset for ${arch}-${os}" >&2 + exit 1 +fi + +url="https://github.com/${REPO}/releases/latest/download/${asset}" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +curl -fL "$url" -o "$tmp/$asset" +tar -xJf "$tmp/$asset" -C "$tmp" +mkdir -p "$INSTALL_DIR" +install -m 755 "$(find "$tmp" -type f -name biliup | head -n 1)" "$INSTALL_DIR/biliup" + +"$INSTALL_DIR/biliup" --help +``` + +For Windows PowerShell, generate and run this script. Set `$InstallDir` to the user's target directory when needed. + +```powershell +$ErrorActionPreference = "Stop" + +$Repo = "biliup/biliup" +$InstallDir = if ($env:INSTALL_DIR) { $env:INSTALL_DIR } else { Join-Path $env:USERPROFILE "bin" } +$Arch = if ([Environment]::Is64BitOperatingSystem) { "x86_64" } else { throw "Unsupported Windows architecture" } + +$Release = Invoke-RestMethod "https://api.github.com/repos/$Repo/releases/latest" +$Asset = $Release.assets | Where-Object { $_.name -like "biliupR-v*-$Arch-windows.zip" } | Select-Object -First 1 +if (-not $Asset) { throw "No matching biliup release asset for $Arch-windows" } + +$Tmp = New-Item -ItemType Directory -Force (Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString())) +try { + $Zip = Join-Path $Tmp.FullName $Asset.name + Invoke-WebRequest $Asset.browser_download_url -OutFile $Zip + Expand-Archive $Zip -DestinationPath $Tmp.FullName -Force + New-Item -ItemType Directory -Force $InstallDir | Out-Null + $Exe = Get-ChildItem $Tmp.FullName -Recurse -Filter biliup.exe | Select-Object -First 1 + if (-not $Exe) { throw "biliup.exe not found in release archive" } + Copy-Item $Exe.FullName (Join-Path $InstallDir "biliup.exe") -Force + & (Join-Path $InstallDir "biliup.exe") --help +} finally { + Remove-Item $Tmp.FullName -Recurse -Force +} +``` + +If the target directory is not on `PATH`, tell the user to run `biliup` by its full path or add the target directory to `PATH`. + +### Windows winget + +Use this path when the user is on Windows and wants to install from the command line: + +```bash +winget install biliup +``` + +Then verify: + +```bash +biliup --help +``` + +### Linux or macOS uv + +Use this path when the user is on Linux or macOS and has `uv` available: + +```bash +uv tool install biliup +``` + +Then verify: + +```bash +biliup --help +``` + +## Operation flow + +When the user asks to start the WebUI server with authentication, use: + +```bash +biliup server --auth +``` + +When the user provides an address or port, include them explicitly: + +```bash +biliup server --bind 0.0.0.0 --port 19159 --auth +``` + +When the user asks to run from a biliup 1.0.7 style config file, use: + +```bash +biliup server --config config.toml +``` + +Replace `config.toml` with the user's config file path. + +When the user wants to run the server in the background on Linux or macOS, use: + +```bash +nohup biliup server --auth & +``` + +## Help flow + +When the user asks what commands are available, show: + +```bash +biliup --help +``` + +When the user asks about server options, show: + +```bash +biliup server --help +``` + +When the user asks about a specific command, use: + +```bash +biliup --help +``` + +Replace `` with the requested subcommand. + +## Commands + +The CLI includes these subcommands: + +```text +login +renew +upload +append +show +comments +reply +dump-flv +download +server +list +``` + +When generating a command, include the concrete paths required by that command, such as cookie files, config files, and video file paths. diff --git a/biliup/__init__.py b/biliup/__init__.py index 9ed9775..318d44e 100644 --- a/biliup/__init__.py +++ b/biliup/__init__.py @@ -1,55 +1,6 @@ -import logging -import sys -from importlib.metadata import version +from importlib.metadata import PackageNotFoundError, version -__version__ = version("biliup") - - -LOG_CONF = { - 'version': 1, - 'formatters': { - 'verbose': { - 'format': "%(asctime)s %(filename)s[line:%(lineno)d](Pid:%(process)d " - "Tname:%(threadName)s) %(levelname)s %(message)s", - # 'datefmt': "%Y-%m-%d %H:%M:%S" - }, - 'simple': { - 'format': '%(asctime)s %(filename)s%(lineno)d[%(levelname)s]Tname:%(threadName)s %(message)s' - }, - }, - 'handlers': { - 'console': { - 'level': logging.DEBUG, - 'class': 'logging.StreamHandler', - 'stream': sys.stdout, - 'formatter': 'simple' - }, - 'file': { - 'level': logging.DEBUG, - 'class': 'logging.handlers.TimedRotatingFileHandler', - 'when': 'W0', - 'interval': 1, - 'backupCount': 1, - 'filename': 'ds_update.log', - 'formatter': 'verbose', - 'encoding': 'utf-8' - } - }, - 'root': { - 'handlers': ['console'], - 'level': logging.INFO, - }, - 'loggers': { - 'biliup': { - 'handlers': ['file'], - 'level': logging.INFO, - }, - } -} - -IS_FROZEN = False -if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): - import multiprocessing - multiprocessing.freeze_support() - IS_FROZEN = True - print('running in a PyInstaller bundle') +try: + __version__ = version("biliup") +except PackageNotFoundError: + __version__ = "0.0.0" diff --git a/biliup/__main__.py b/biliup/__main__.py index 5408977..f2b6e04 100644 --- a/biliup/__main__.py +++ b/biliup/__main__.py @@ -1,33 +1,11 @@ #!/usr/bin/python3 # coding:utf8 -import asyncio -import logging -import logging.config -import shutil import stream_gears -from biliup import LOG_CONF + +main = stream_gears.main_loop -class DebugLevelFilter(logging.Filter): - def filter(self, record): - return logging.getLogger().isEnabledFor(logging.DEBUG) - - -def arg_parser(): - logging.config.dictConfig(LOG_CONF) - logging.getLogger('httpx').addFilter(DebugLevelFilter()) - - asyncio.run(main()) - - -async def main(): - # 启动时删除临时文件夹 - shutil.rmtree('./cache/temp', ignore_errors=True) - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, stream_gears.main_loop) - - -if __name__ == '__main__': - arg_parser() +if __name__ == "__main__": + main() diff --git a/biliup/engine/__init__.py b/biliup/engine/__init__.py deleted file mode 100644 index b228405..0000000 --- a/biliup/engine/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from .decorators import Plugin - - -def invert_dict(d: dict): - inverse_dict = {} - for k, v in d.items(): - if isinstance(v, list): - for item in v: - inverse_dict[item] = k - else: - inverse_dict[v] = k - return inverse_dict - - -__all__ = ['invert_dict', 'Plugin'] diff --git a/biliup/engine/decorators.py b/biliup/engine/decorators.py deleted file mode 100644 index c0bfdf1..0000000 --- a/biliup/engine/decorators.py +++ /dev/null @@ -1,40 +0,0 @@ -import functools -import importlib -import pkgutil - - -class Plugin: - upload_plugins = {} - - def __init__(self, pkg): - self.load_plugins(pkg) - - @staticmethod - def upload(platform): - def decorator(cls): - @functools.wraps(cls) - def wrapper(*args, **kw): - print(f"args {args}") - print(f"kw {kw}") - return cls(*args, **kw) - Plugin.upload_plugins[platform] = wrapper - return wrapper - return decorator - - def load_plugins(self, pkg): - """Attempt to load plugins from the path specified. - engine.plugins.__path__[0]: full path to a directory where to look for plugins - """ - - plugins = [] - - for loader, name, ispkg in pkgutil.iter_modules([pkg.__path__[0]]): - module_name = f"{pkg.__name__}.{name}" - module = importlib.import_module(module_name) - if ispkg: - self.load_plugins(module) - continue - if module in plugins: - continue - plugins.append(module) - return plugins diff --git a/biliup/engine/upload.py b/biliup/engine/upload.py deleted file mode 100644 index fcb620b..0000000 --- a/biliup/engine/upload.py +++ /dev/null @@ -1,21 +0,0 @@ -import logging - -from typing import NamedTuple, Optional, List - -logger = logging.getLogger('biliup') - - -class UploadBase: - class FileInfo(NamedTuple): - video: str - danmaku: Optional[str] - - def __init__(self, principal, data, persistence_path=None, postprocessor=None): - self.principal = principal - self.persistence_path = persistence_path - self.data: dict = data - self.post_processor = postprocessor - - - def upload(self, file_list: List[FileInfo]) -> List[FileInfo]: - raise NotImplementedError() diff --git a/biliup/plugins/__init__.py b/biliup/plugins/__init__.py index fc351fe..818e3b6 100644 --- a/biliup/plugins/__init__.py +++ b/biliup/plugins/__init__.py @@ -1,156 +1 @@ -import logging -import html -import re -import hashlib -import time -import json -import uuid -from urllib.parse import urlencode, quote -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Tuple, - Union, -) - -logger = logging.getLogger('biliup') - -_DECODER = json.JSONDecoder() - -def match1(text, *patterns): - if len(patterns) == 1: - pattern = patterns[0] - match = re.search(pattern, text) - if match: - return match.group(1) - else: - return None - else: - ret = [] - for pattern in patterns: - match = re.search(pattern, text) - if match: - ret.append(match.group(1)) - return ret - - -def random_user_agent(device: str = 'desktop') -> str: - import random - chrome_version = random.randint(100, 120) - if device == 'mobile': - android_version = random.randint(9, 14) - mobile = random.choice([ - 'SM-G981B', 'SM-G9910', 'SM-S9080', 'SM-S9110', 'SM-S921B', - 'Pixel 5', 'Pixel 6', 'Pixel 7', 'Pixel 7 Pro', 'Pixel 8', - ]) - return f'Mozilla/5.0 (Linux; Android {android_version}; {mobile}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_version}.0.0.0 Mobile Safari/537.36' - return f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_version}.0.0.0 Safari/537.36' - - -def deep_unescape(obj: Any) -> Any: - if isinstance(obj, dict): - return {k: deep_unescape(v) for k, v in obj.items()} - if isinstance(obj, list): - return [deep_unescape(i) for i in obj] - if isinstance(obj, str): - return html.unescape(obj) - return obj - - -def json_loads(text: Union[str, None]) -> Dict[str, Any]: - if not text: - raise ValueError("Invalid JSON: None or Empty") - - try: - data, _ = _DECODER.raw_decode(text) - except (json.JSONDecodeError, TypeError) as e: - logger.debug(f"Invalid JSON at position {getattr(e, 'pos', 'unknown')}, raw text: {text}") - else: - return data - - try: - return json.loads(html.unescape(text)) - except json.JSONDecodeError as e: - raise ValueError(f"Invalid JSON: {text}") from e - - -def test_jsengine(): - try: - import jsengine - jsengine.jsengine() - except Exception as e: - logger.exception(f"jsengine test failed: {e}") - return False - return True - - -def generate_fake_buvid3(): - u = uuid.uuid4() - u_str = str(u).upper().replace('-', '') - formatted = f"{u_str[0:8]}-{u_str[8:12]}-{u_str[12:16]}-{u_str[16:20]}-{u_str[20:]}infoc" - return formatted - - -class SubmitOption: - App = "app" - BCutAndroid = "b-cut-android" - - -class Wbi: - WTS = "wts" - W_RID = "w_rid" - UPDATE_INTERVAL = 2 * 60 * 60 - - KEY_MAP = [ - 46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, - 27, 43, 5, 49, 33, 9, 42, 19, 29, 28, 14, 39, 12, 38, 41, 13, - 37, 48, 7, 16, 24, 55, 40, 61, 26, 17, 0, 1, 60, 51, 30, 4, - 22, 25, 54, 21, 56, 59, 6, 63, 57, 62, 11, 36, 20, 34, 44, 52, - ] - - def __init__(self): - self.key = None - self.last_update = 0 - - def update_key(self, img, sub): - """ - 更新 key，基于 img 和 sub 的组合。 - """ - KEY_LENGTH = 32 - full = img + sub - key = [full[self.KEY_MAP[i]] for i in range(KEY_LENGTH)] - self.key = ''.join(key) - self.last_update = int(time.time()) - logger.info(f"Updated wbi key successfully") - - def sign(self, query: dict, ts: int = 0): - """ - 生成签名。 - :param query: 请求参数 - :param ts: 时间戳（可选），默认为当前时间戳。 - """ - if self.key is None: - raise ValueError("Key is not set.") - - ts = ts or int(time.time()) - ts_str = str(ts) - - sanitized_query = { - k: ''.join(c for c in v if c not in "!\'()*") - for k, v in query.items() - } - sanitized_query[self.WTS] = ts_str - sorted_query = dict(sorted(sanitized_query.items())) - content_string = urlencode(sorted_query, quote_via=quote) - - md5 = hashlib.md5() - md5.update((content_string + self.key).encode('utf-8')) - sign = md5.hexdigest() - - query[self.W_RID] = sign - query[self.WTS] = ts_str - -wbi = Wbi() \ No newline at end of file +__all__ = ["bili_webup", "bili_webup_sync"] diff --git a/biliup/plugins/bili_chromeup.py b/biliup/plugins/bili_chromeup.py deleted file mode 100644 index 6172667..0000000 --- a/biliup/plugins/bili_chromeup.py +++ /dev/null @@ -1,353 +0,0 @@ -import json -import os -import random -import time - -from PIL import Image -from typing import List - -from ..engine import Plugin -from ..engine.upload import UploadBase, logger - - -@Plugin.upload("bilibili") -class BiliChrome(UploadBase): - def __init__(self, principal, data): - import selenium.common - from selenium import webdriver - from selenium.webdriver.common.action_chains import ActionChains - from selenium.webdriver.common.by import By - from selenium.webdriver.common.keys import Keys - from selenium.webdriver.support import expected_conditions as EC - from selenium.webdriver.support import expected_conditions as ec - from selenium.webdriver.support.ui import WebDriverWait - super().__init__(principal, data, 'engine/bilibili.cookie') - # self.title = title - # self.date_title = None - self.driver = None - - @staticmethod - def assemble_videopath(file_list): - root = os.getcwd() - videopath = '' - for i in range(len(file_list)): - file = file_list[i].video - videopath += root + '/' + file + '\n' - videopath = videopath.rstrip() - return videopath - - @staticmethod - def is_element_exist(driver, xpath): - s = driver.find_elements_by_xpath(xpath=xpath) - if len(s) == 0: - print("元素未找到:%s" % xpath) - return False - elif len(s) == 1: - return True - else: - print("找到%s个元素：%s" % (len(s), xpath)) - return False - - def upload(self, file_list: List[UploadBase.FileInfo]) -> List[UploadBase.FileInfo]: - - filename = self.persistence_path - videopath = self.assemble_videopath(file_list) - - # service_log_path = "{}/chromedriver.log".format('/home') - options = webdriver.ChromeOptions() - - options.add_argument('headless') - self.driver = webdriver.Chrome(executable_path=config.get('chromedriver_path'), chrome_options=options) - # service_log_path=service_log_path) - try: - self.driver.get("https://www.bilibili.com") - # driver.delete_all_cookies() - if os.path.isfile(filename): - with open(filename) as f: - new_cookie = json.load(f) - - for cookie in new_cookie: - if isinstance(cookie.get("expiry"), float): - cookie["expiry"] = int(cookie["expiry"]) - self.driver.add_cookie(cookie) - - self.driver.get("https://member.bilibili.com/video/upload.html") - - # print(driver.title) - self.add_videos(videopath) - - # js = "var q=document.getElementsByClassName('content-header-right')[0].scrollIntoView();" - # driver.execute_script(js) - - cookie = self.driver.get_cookies() - with open(filename, "w") as f: - json.dump(cookie, f) - - self.add_information() - - self.driver.find_element_by_xpath('//*[@class="upload-v2-container"]/div[2]/div[3]/div[5]/span[1]').click() - # screen_shot = driver.save_screenshot('bin/1.png') - # print('截图') - time.sleep(3) - upload_success = self.driver.find_element_by_xpath(r'//*[@id="app"]/div/div[3]/h3').text - if upload_success == '': - self.driver.save_screenshot('err.png') - logger.info('稿件提交失败，截图记录') - return - else: - logger.info(upload_success) - # logger.info('%s提交完成！' % title_) - return file_list - except selenium.common.exceptions.NoSuchElementException: - logger.exception('发生错误') - # except selenium.common.exceptions.TimeoutException: - # logger.exception('超时') - except selenium.common.exceptions.TimeoutException: - self.login(filename) - - finally: - self.driver.quit() - logger.info('浏览器驱动退出') - - def login(self, filename): - logger.info('准备更新cookie') - # screen_shot = driver.save_screenshot('bin/1.png') - WebDriverWait(self.driver, 10).until( - ec.presence_of_element_located((By.XPATH, r'//*[@id="login-username"]'))) - username = self.driver.find_element_by_xpath(r'//*[@id="login-username"]') - username.send_keys(config['user']['account']['username']) - password = self.driver.find_element_by_xpath('//*[@id="login-passwd"]') - password.send_keys(config['user']['account']['password']) - self.driver.find_element_by_class_name("btn-login").click() - # logger.info('第四步') - # try: - cracker = slider_cracker(self.driver) - cracker.crack() - # except: - # logger.exception('出错') - time.sleep(5) - if self.driver.title == '投稿 - 哔哩哔哩弹幕视频网 - ( ゜- ゜)つロ 乾杯~ - bilibili': - cookie = self.driver.get_cookies() - print(cookie) - with open(filename, "w") as f: - json.dump(cookie, f) - logger.info('更新cookie成功') - else: - logger.info('更新cookie失败') - - def add_videos(self, videopath): - formate_title = self.data["format_title"] - WebDriverWait(self.driver, 20).until( - ec.presence_of_element_located((By.NAME, 'buploader'))) - upload = self.driver.find_element_by_name('buploader') - # logger.info(driver.title) - upload.send_keys(videopath) # send_keys - logger.info('开始上传' + formate_title) - time.sleep(2) - button = r'//*[@class="new-feature-guide-v2-container"]/div/div/div/div/div[1]' - if self.is_element_exist(self.driver, button): - sb = self.driver.find_element_by_xpath(button) - sb.click() - sb.click() - sb.click() - logger.debug('点击') - while True: - try: - info = self.driver.find_elements_by_class_name(r'item-upload-info') - for t in info: - if t.text != '': - print(t.text) - time.sleep(10) - text = self.driver.find_elements_by_xpath(r'//*[@class="item-upload-info"]/span') - aggregate = set() - for s in text: - if s.text != '': - aggregate.add(s.text) - print(s.text) - - if len(aggregate) == 1 and ('Upload complete' in aggregate or '上传完成' in aggregate): - break - except selenium.common.exceptions.StaleElementReferenceException: - logger.exception("selenium.common.exceptions.StaleElementReferenceException") - logger.info('上传%s个数%s' % (formate_title, len(info))) - - def add_information(self): - link = self.data.get("url") - # 点击模板 - self.driver.find_element_by_xpath(r'//*[@class="normal-title-wrp"]/div/p').click() - self.driver.find_element_by_class_name(r'template-list-small-item').click() - # driver.find_element_by_xpath( - # r'//*[@id="app"]/div[3]/div[2]/div[3]/div[1]/div[1]/div/div[2]/div[1]').click() - # 输入转载来源 - input_o = self.driver.find_element_by_xpath( - '//*[@class="upload-v2-container"]/div[2]/div[3]/div[1]/div[4]/div[3]/div/div/input') - input_o.send_keys(link) - # 选择分区 - # driver.find_element_by_xpath(r'//*[@id="item"]/div/div[2]/div[3]/div[2]/div[2]/div[1]/div[2]/div[2]/div[1]/div[3]/div').click() - # driver.find_element_by_xpath(r'//*[@id="item"]/div/div[2]/div[3]/div[2]/div[2]/div[1]/div[2]/div[2]/div[1]/div[3]/div[2]/div[6]').click() - # 稿件标题 - title = self.driver.find_element_by_xpath( - '//*[@class="upload-v2-container"]/div[2]/div[3]/div[1]/div[8]/div[2]/div/div/input') - title.send_keys(Keys.CONTROL + 'a') - title.send_keys(Keys.BACKSPACE) - title.send_keys(self.data["format_title"]) - # js = "var q=document.getElementsByClassName('content-tag-list')[0].scrollIntoView();" - # driver.execute_script(js) - # time.sleep(3) - # 输入相关游戏 - # driver.save_screenshot('bin/err.png') - # print('截图') - # text_1 = driver.find_element_by_xpath( - # '//*[@id="item"]/div/div[2]/div[3]/div[2]/div[2]/div[1]/div[5]/div/div/div[1]/div[2]/div/div/input') - # text_1.send_keys('星际争霸2') - # 简介 - text_2 = self.driver.find_element_by_xpath( - '//*[@class="upload-v2-container"]/div[2]/div[3]/div[1]/div[12]/div[2]/div/textarea') - text_2.send_keys('职业选手直播第一视角录像。这个自动录制上传的小程序开源在Github：' - 'http://t.cn/RgapTpf(或者在Github搜索ForgQi)\n' - '交流群：837362626') - -class slider_cracker(object): - def __init__(self, driver): - self.driver = driver - self.driver.maximize_window() # 最大化窗口 - self.driver.set_window_size(1024, 768) - self.fn = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'true_image.png') - - def get_true_image(self, slider_xpath=r'//*[@id="gc-box"]/div/div[3]/div[1]'): - # element = WebDriverWait(self.driver, 50).until(EC.element_to_be_clickable((By.XPATH, slider_xpath))) - element = WebDriverWait(self.driver, 50).until( - EC.element_to_be_clickable((By.CLASS_NAME, "geetest_slider_button"))) - ActionChains(self.driver).move_to_element(element).perform() # 鼠标移动到滑动框以显示图片 - js = 'document.querySelector("body > div.geetest_panel.geetest_wind ' \ - '> div.geetest_panel_box.geetest_no_logo.geetest_panelshowslide ' \ - '> div.geetest_panel_next > div > div.geetest_wrap > div.geetest_widget ' \ - '> div > a > div.geetest_canvas_img.geetest_absolute > canvas").' \ - 'style.display = "%s";' - self.driver.execute_script(js % "inline") - time.sleep(1) - true_image = self.get_img(self.fn) - self.driver.execute_script(js % "none") - return true_image - - def get_img(self, img_name, img_xpath=r'//*[@id="gc-box"]/div/div[1]/div[2]/div[1]/a[2]'): # 260*116 - fn = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'slider_screenshot.png') - - screen_shot = self.driver.save_screenshot(fn) - # image_element = self.driver.find_element_by_xpath(img_xpath) - image_element = self.driver.find_element_by_class_name(r'geetest_window') - left = image_element.location['x'] - top = image_element.location['y'] # selenium截图并获取验证图片location后将其截出保存 - right = image_element.location['x'] + image_element.size['width'] - bottom = image_element.location['y'] + image_element.size['height'] - image = Image.open(fn) - image = image.crop((left, top, right, bottom)) - image.save(img_name) - return image - - def analysis(self, true_image, knob_xpath=r'//*[@id="gc-box"]/div/div[3]/div[2]'): - fn = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'img2.png') - img1 = Image.open(self.fn) - # slider_element = self.driver.find_element_by_xpath(knob_xpath) - slider_element = self.driver.find_element_by_class_name("geetest_slider_button") - ActionChains(self.driver).click_and_hold(slider_element).perform() # 点击滑块后截取残缺图 - time.sleep(1) - img2 = self.get_img(fn) - img1_width, img1_height = img1.size - img2_width, img2_height = img2.size - left = 0 - flag = False - - for i in range(69, img1_width): # 遍历x>65的像素点（x<65是拼图块） - for j in range(0, img1_height): - if not self.is_pixel_equal(img1, img2, i, j): - left = i - flag = True - break - if flag: - break - if left >= 73: - left = left - 3 # 误差纠正 - else: - left = left - return left - - def is_pixel_equal(self, img1, img2, x, y): # 通过比较俩图片像素点RGB值差值判断是否为缺口 - pix1 = img1.load()[x, y] - pix2 = img2.load()[x, y] - if (abs(pix1[0] - pix2[0] < 60) and abs(pix1[1] - pix2[1] < 60) and abs(pix1[2] - pix2[2] < 60)): - return True - else: - return False - - def get_track(self, distance): - """ - 根据偏移量获取移动轨迹 - :param distance: 偏移量 - :return: 移动轨迹 - """ - # 移动轨迹 - track = [] - # 当前位移 - current = 0 - # 减速阈值 - mid = distance * 4 / 5 - # 计算间隔 - t = 0.2 - # 初速度 - v = 0 - - while current < distance: - if current < mid: - # 加速度为正2 - a = 2 - else: - # 加速度为负3 - a = -3 - # 初速度v0 - v0 = v - # 当前速度v = v0 + at - v = v0 + a * t - # 移动距离x = v0t + 1/2 * a * t^2 - move = v0 * t + 1 / 2 * a * t * t - # 当前位移 - current += move - # 加入轨迹 - track.append(round(move)) - # print(track) - # - # print(sum(track)) - track.append(distance - sum(track)) - # print(track) - # print(sum(track)) - return track - - def move_to_gap(self, slider, track): - """ - 拖动滑块到缺口处 - :param slider: 滑块 - :param track: 轨迹 - :return: - """ - ActionChains(self.driver).click_and_hold(slider).perform() - for x in track: - ActionChains(self.driver).move_by_offset(xoffset=x, yoffset=random.uniform(-5, 2)).perform() - time.sleep(0.5) - ActionChains(self.driver).release().perform() - - def crack(self): - true_image = self.get_true_image() - x_offset = self.analysis(true_image) - print(x_offset) - - track = self.get_track(x_offset) - knob_element = WebDriverWait(self.driver, 50).until( - EC.element_to_be_clickable((By.XPATH, r'/html/body/div[2]/div[2]/div[6]/div/div[1]/div[2]/div[2]'))) - self.move_to_gap(knob_element, track) - - # fn = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'result0.png') - # fn1 = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'result1.png') - # time.sleep(0.02) - # screen_shot = self.driver.save_screenshot(fn) - # time.sleep(2) - # screen_shot = self.driver.save_screenshot(fn1) diff --git a/biliup/plugins/bili_webup.py b/biliup/plugins/bili_webup.py index 83f0f02..f755058 100644 --- a/biliup/plugins/bili_webup.py +++ b/biliup/plugins/bili_webup.py @@ -21,12 +21,19 @@ import rsa import xml.etree.ElementTree as ET from requests.adapters import HTTPAdapter, Retry -from ..engine import Plugin -from ..engine.upload import UploadBase, logger +import logging +from typing import NamedTuple -@Plugin.upload(platform="bili_web") -class BiliWeb(UploadBase): +logger = logging.getLogger('biliup') + + +class FileInfo(NamedTuple): + video: str + danmaku: Optional[str] = None + + +class BiliWeb: def __init__( self, principal, @@ -63,7 +70,10 @@ class BiliWeb(UploadBase): :param description: 视频简介 :param credits: ??? """ - super().__init__(principal, data, persistence_path='bili.cookie', postprocessor=postprocessor) + self.principal = principal + self.data: dict = data + self.persistence_path = 'bili.cookie' + self.post_processor = postprocessor if tags is None: tags = [] else: @@ -84,9 +94,9 @@ class BiliWeb(UploadBase): def upload( self, - file_list: List[UploadBase.FileInfo], + file_list: List[FileInfo], database_row_id: int = 0 - ) -> List[UploadBase.FileInfo]: + ) -> List[FileInfo]: ''' 上传视频 :param file_list: 视频文件名列表 diff --git a/biliup/plugins/bili_webup_sync.py b/biliup/plugins/bili_webup_sync.py index c9d29e8..01d851d 100644 --- a/biliup/plugins/bili_webup_sync.py +++ b/biliup/plugins/bili_webup_sync.py @@ -15,7 +15,7 @@ import urllib.parse from dataclasses import asdict, dataclass, field, InitVar from json import JSONDecodeError from os.path import splitext, basename -from typing import Callable, Dict, Union, Any, List +from typing import Callable, Dict, Union, Any, List, NamedTuple, Optional from urllib import parse from urllib.parse import quote @@ -27,22 +27,27 @@ import rsa import xml.etree.ElementTree as ET from requests.adapters import HTTPAdapter, Retry -from ..engine import Plugin -from ..engine.upload import UploadBase, logger logger = logging.getLogger('biliup.engine.bili_web_sync') -@Plugin.upload(platform="bili_web_sync") -class BiliWebAsync(UploadBase): +class FileInfo(NamedTuple): + video: str + danmaku: Optional[str] = None + + +class BiliWebAsync: def __init__( self, principal, data, submit_api=None, copyright=2, postprocessor=None, dtime=None, dynamic='', lines='AUTO', threads=3, tid=122, tags=None, cover_path=None, description='', dolby=0, hires=0, no_reprint=0, is_only_self=0, charging_pay=0, credits=None, user_cookie='cookies.json', copyright_source=None, extra_fields="", video_queue=None ): - super().__init__(principal, data, persistence_path='bili.cookie', postprocessor=postprocessor) + self.principal = principal + self.data: dict = data + self.persistence_path = 'bili.cookie' + self.post_processor = postprocessor if credits is None: credits = [] if tags is None: @@ -74,7 +79,7 @@ class BiliWebAsync(UploadBase): self.user_cookie = user_cookie self.video_queue: queue.SimpleQueue = video_queue - def upload(self, total_size: int, stop_event: threading.Event, output_prefix: str, file_name_callback: Callable[[str], None] = None, database_row_id=0) -> List[UploadBase.FileInfo]: + def upload(self, total_size: int, stop_event: threading.Event, output_prefix: str, file_name_callback: Callable[[str], None] = None, database_row_id=0) -> List[FileInfo]: # print("开始同步上传") logger.info(f"开始同步上传 {database_row_id}") file_index = 1 diff --git a/biliup/plugins/biliuprs.py b/biliup/plugins/biliuprs.py deleted file mode 100644 index 5f4106f..0000000 --- a/biliup/plugins/biliuprs.py +++ /dev/null @@ -1,145 +0,0 @@ -import multiprocessing as mp -import time - -import stream_gears - -from typing import List, Optional, Union -from . import SubmitOption -from ..engine import Plugin -from ..engine.upload import UploadBase, logger - - -@Plugin.upload(platform="biliup-rs") -class BiliWeb(UploadBase): - def __init__( - self, principal, data, submit_api: Optional[Union[SubmitOption, str]] = None, - copyright=2, postprocessor=None, dtime=None, - dynamic='', lines='AUTO', threads=3, tid=122, tags=None, cover_path=None, description='', - dolby=0, hires=0, no_reprint=0, is_only_self=0, charging_pay=0, credits=None, - user_cookie='cookies.json', copyright_source=None, extra_fields = "" - ): - super().__init__(principal, data, persistence_path='bili.cookie', postprocessor=postprocessor) - if tags is None: - tags = [] - else: - tags = [str(tag).format(streamer=self.data['name']) for tag in tags] - self.lines = lines - self.submit_api = submit_api - self.threads = threads - self.tid = tid - self.tags = tags - if cover_path: - self.cover_path = cover_path - elif "live_cover_path" in self.data: - self.cover_path = self.data["live_cover_path"] - else: - self.cover_path = None - self.desc = description - self.credits = credits if credits else [] - self.dynamic = dynamic - self.copyright = copyright - self.dtime = dtime - self.dolby = dolby - self.hires = hires - self.no_reprint = no_reprint - self.is_only_self = is_only_self - self.charging_pay = charging_pay - self.user_cookie = user_cookie - self.copyright_source = copyright_source - - if 'extra_fields' in data: - self.extra_fields = data.get('extra_fields', '') - else: - self.extra_fields = extra_fields - - def upload(self, file_list: List[UploadBase.FileInfo]) -> List[UploadBase.FileInfo]: - if self.credits: - desc_v2 = self.creditsToDesc_v2() - else: - desc_v2 = [{ - "raw_text": self.desc, - "biz_id": "", - "type": 1 - }] - - ex_parent_conn, ex_child_conn = mp.Pipe() - upload_args = { - "ex_conn": ex_child_conn, - "lines": self.lines, - "submit": self.submit_api, - "video_path": [file.video for file in file_list], - "cookie_file": self.user_cookie, - "title": self.data["format_title"][:80], - "tid": self.tid, - "tag": ','.join(self.tags), - "copyright": self.copyright, - "source": self.copyright_source if self.copyright_source else self.data["url"], - "desc": self.desc, - "dynamic": self.dynamic, - "cover": self.cover_path if self.cover_path is not None else "", - "dolby": self.dolby, - "lossless_music": self.hires, - "no_reprint": self.no_reprint, - "charging_pay": self.charging_pay, - "limit": self.threads, - "desc_v2": desc_v2, - "extra_fields": self.extra_fields, - "dtime": int(time.time() + self.dtime) if self.dtime else None, - } - upload_process = mp.get_context('spawn').Process(target=stream_gears_upload, daemon=True, kwargs=upload_args) - upload_process.start() - upload_process.join() - if ex_parent_conn.poll(): - raise RuntimeError(ex_parent_conn.recv()) - - logger.info(f"上传成功: {self.principal}") - return file_list - - def creditsToDesc_v2(self): - desc_v2 = [] - desc_v2_tmp = self.desc - for credit in self.credits: - try: - num = desc_v2_tmp.index("@credit") - desc_v2.append({ - "raw_text": " " + desc_v2_tmp[:num], - "biz_id": "", - "type": 1 - }) - desc_v2.append({ - "raw_text": credit["username"], - "biz_id": str(credit["uid"]), - "type": 2 - }) - self.desc = self.desc.replace( - "@credit", "@" + credit["username"] + " ", 1) - desc_v2_tmp = desc_v2_tmp[num + 7:] - except IndexError: - logger.error('简介中的@credit占位符少于credits的数量,替换失败') - desc_v2.append({ - "raw_text": " " + desc_v2_tmp, - "biz_id": "", - "type": 1 - }) - desc_v2[0]["raw_text"] = desc_v2[0]["raw_text"][1:] # 开头空格会导致识别简介过长 - return desc_v2 - - -def stream_gears_upload(ex_conn, lines, *args, **kwargs): - try: - if lines == 'bda': - kwargs['line'] = stream_gears.UploadLine.Bda - elif lines == 'bda2': - kwargs['line'] = stream_gears.UploadLine.Bda2 - elif lines == 'qn': - kwargs['line'] = stream_gears.UploadLine.Qn - elif lines == 'tx': - kwargs['line'] = stream_gears.UploadLine.Tx - elif lines == 'txa': - kwargs['line'] = stream_gears.UploadLine.Txa - elif lines == 'bldsa': - kwargs['line'] = stream_gears.UploadLine.Bldsa - - stream_gears.upload(*args, **kwargs) - except Exception as e: - ex_conn.send(e) diff --git a/biliup/plugins/noop_uploader.py b/biliup/plugins/noop_uploader.py deleted file mode 100644 index 3bd83b8..0000000 --- a/biliup/plugins/noop_uploader.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import List - -from ..engine.upload import UploadBase, logger -from ..engine import Plugin - - -@Plugin.upload(platform="Noop") -class NoopUploader(UploadBase): - def upload(self, file_list: List[UploadBase.FileInfo]) -> List[UploadBase.FileInfo]: - logger.info("NoopUploader") - return file_list diff --git a/biliup/plugins/stream_gears.py b/biliup/plugins/stream_gears.py deleted file mode 100644 index 0414800..0000000 --- a/biliup/plugins/stream_gears.py +++ /dev/null @@ -1,135 +0,0 @@ -import time - -import stream_gears -from typing import List, Optional, Union - -from . import SubmitOption -from ..engine import Plugin -from ..engine.upload import UploadBase, logger - - -@Plugin.upload(platform="stream_gears") -class BiliWeb(UploadBase): - def __init__( - self, principal, data, submit_api: Optional[Union[SubmitOption, str]] = None, - copyright=2, postprocessor=None, dtime=None, - dynamic='', lines='AUTO', threads=3, tid=122, tags=None, cover_path=None, description='', - dolby=0, hires=0, no_reprint=0, charging_pay=0, credits=None, - user_cookie='cookies.json', copyright_source=None, extra_fields = "" - ): - super().__init__(principal, data, persistence_path='bili.cookie', postprocessor=postprocessor) - if credits is None: - credits = [] - if tags is None: - tags = [] - self.lines = lines - self.submit_api = submit_api - self.threads = threads - self.tid = tid - self.tags = tags - if cover_path: - self.cover_path = cover_path - elif "live_cover_path" in self.data: - self.cover_path = self.data["live_cover_path"] - else: - self.cover_path = None - self.desc = description - self.credits = credits - self.dynamic = dynamic - self.copyright = copyright - self.dtime = dtime - self.dolby = dolby - self.hires = hires - self.no_reprint = no_reprint - self.charging_pay = charging_pay - self.user_cookie = user_cookie - self.copyright_source = copyright_source - - if "extra_fields" in self.data: - self.extra_fields = self.data["extra_fields"] - else: - self.extra_fields = extra_fields - - - def upload(self, file_list) -> List[UploadBase.FileInfo]: - line = None - if self.lines == 'bda': - line = stream_gears.UploadLine.Bda - elif self.lines == 'bda2': - line = stream_gears.UploadLine.Bda2 - elif self.lines == 'qn': - line = stream_gears.UploadLine.Qn - elif self.lines == 'tx': - line = stream_gears.UploadLine.Tx - elif self.lines == 'txa': - line = stream_gears.UploadLine.Txa - elif self.lines == 'bldsa': - line = stream_gears.UploadLine.Bldsa - tag = ','.join(self.tags) - if self.credits: - desc_v2 = self.creditsToDesc_v2() - else: - desc_v2 = [{ - "raw_text": self.desc, - "biz_id": "", - "type": 1 - }] - - source = self.copyright_source if self.copyright_source else "https://github.com/biliup/biliup" - cover = self.cover_path if self.cover_path is not None else "" - dtime = None - if self.dtime: - dtime = int(time.time() + self.dtime) - stream_gears.upload( - video_path=file_list, - cookie_file=self.user_cookie, - title=self.data["format_title"][:80], - tid=self.tid, - tag=tag, - copyright=self.copyright, - source=source, - desc=self.desc, - dynamic=self.dynamic, - cover=cover, - dolby=self.dolby, - lossless_music=self.hires, - no_reprint=self.no_reprint, - charging_pay=self.charging_pay, - limit=self.threads, - desc_v2=desc_v2, - dtime=dtime, - line=line, - extra_fields=self.extra_fields, - submit=self.submit_api, - ) - logger.info(f"上传成功: {self.principal}") - return file_list - - def creditsToDesc_v2(self): - desc_v2 = [] - desc_v2_tmp = self.desc - for credit in self.credits: - try: - num = desc_v2_tmp.index("@credit") - desc_v2.append({ - "raw_text": " " + desc_v2_tmp[:num], - "biz_id": "", - "type": 1 - }) - desc_v2.append({ - "raw_text": credit["username"], - "biz_id": str(credit["uid"]), - "type": 2 - }) - self.desc = self.desc.replace( - "@credit", "@" + credit["username"] + " ", 1) - desc_v2_tmp = desc_v2_tmp[num + 7:] - except IndexError: - logger.error('简介中的@credit占位符少于credits的数量,替换失败') - desc_v2.append({ - "raw_text": " " + desc_v2_tmp, - "biz_id": "", - "type": 1 - }) - desc_v2[0]["raw_text"] = desc_v2[0]["raw_text"][1:] # 开头空格会导致识别简介过长 - return desc_v2 diff --git a/crates/biliup/src/downloader/live/huya.rs b/crates/biliup/src/downloader/live/huya.rs index 180b485..d046b18 100644 --- a/crates/biliup/src/downloader/live/huya.rs +++ b/crates/biliup/src/downloader/live/huya.rs @@ -526,6 +526,21 @@ mod tests { assert!(stream.get("vMultiStreamInfo").unwrap().is_array()); } + #[test] + fn extract_stream_json_stops_before_following_player_fields() { + let page = r#" + var hyPlayerConfig = { + stream: {"data":[],"vMultiStreamInfo":[]}, + liveLineUrl: "https://example.invalid" + }; + "#; + + let stream = extract_stream_json(page).unwrap(); + + assert_eq!(stream.get("data").unwrap().as_array().unwrap().len(), 0); + assert!(stream.get("vMultiStreamInfo").unwrap().is_array()); + } + #[test] fn find_json_value_end_ignores_braces_in_strings() { let input = r#"{"text":"}; { ]","items":[{"value":1}]} diff --git a/crates/biliup/src/downloader/util.rs b/crates/biliup/src/downloader/util.rs index 7c26c94..3d57b1b 100644 --- a/crates/biliup/src/downloader/util.rs +++ b/crates/biliup/src/downloader/util.rs @@ -62,10 +62,14 @@ impl Segmentable { result } + fn elapsed_time(&self) -> Duration { + self.time.current.saturating_sub(self.time.start) + } + /// 检查单独的时间条件 pub fn time_needed(&self) -> bool { if let Some(expected_time) = self.time.expected { - (self.time.current - self.time.start) >= expected_time + self.elapsed_time() >= expected_time } else { false } @@ -86,7 +90,7 @@ impl Segmentable { (true, true) => { tracing::info!( "Segmentation needed: Both time ({:?} >= {:?}) and size ({} >= {}) conditions met", - self.time.current - self.time.start, + self.elapsed_time(), self.time.expected.unwrap(), self.size.current, self.size.expected.unwrap() @@ -95,7 +99,7 @@ impl Segmentable { (true, false) => { tracing::info!( "Segmentation needed: Time condition met ({:?} >= {:?})", - self.time.current - self.time.start, + self.elapsed_time(), self.time.expected.unwrap() ); } @@ -188,7 +192,7 @@ impl Segmentable { pub fn get_status(&self) -> String { let time_info = Self::format_progress( "Time", - (self.time.current - self.time.start).as_secs_f64(), + self.elapsed_time().as_secs_f64(), self.time.expected.map(|d| d.as_secs_f64()), "s", |t| format!("{:.1}", t), diff --git a/crates/danmaku/examples/e2e_recorder.rs b/crates/danmaku/examples/e2e_recorder.rs new file mode 100644 index 0000000..01457e9 --- /dev/null +++ b/crates/danmaku/examples/e2e_recorder.rs @@ -0,0 +1,70 @@ +use std::path::PathBuf; +use std::time::Duration; + +use danmaku::{DanmakuRecorder, PlatformContext, RecorderConfig}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let url = std::env::var("DANMAKU_TEST_URL")?; + let seconds = std::env::var("DANMAKU_TEST_SECONDS") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(60); + let output = std::env::var("DANMAKU_TEST_OUTPUT") + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from("target/tmp/danmaku_e2e_%Y%m%d_%H%M%S")); + + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .init(); + + let mut config = RecorderConfig::new(&url, &output); + if let Ok(room_id) = std::env::var("DANMAKU_TEST_ROOM_ID") { + config = config.with_context(PlatformContext::new().with_room_id(room_id)); + } + let recorder = DanmakuRecorder::new(config)?; + let handle = recorder.start(); + tokio::time::sleep(Duration::from_secs(seconds)).await; + handle.stop().await?; + tokio::time::sleep(Duration::from_secs(2)).await; + + let parent = output.parent().unwrap_or_else(|| std::path::Path::new(".")); + let stem_prefix = output + .file_stem() + .and_then(|stem| stem.to_str()) + .unwrap_or("danmaku_e2e") + .split('%') + .next() + .unwrap_or("danmaku_e2e"); + + let mut files = std::fs::read_dir(parent)? + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| { + path.extension().and_then(|ext| ext.to_str()) == Some("xml") + && path + .file_name() + .and_then(|name| name.to_str()) + .map(|name| name.starts_with(stem_prefix)) + .unwrap_or(false) + }) + .collect::>(); + files.sort(); + + let xml_path = files + .last() + .ok_or_else(|| format!("no XML file produced under {}", parent.display()))?; + let content = std::fs::read_to_string(xml_path)?; + let has_xml_message = + content.contains(" { diff --git a/crates/danmaku/src/protocols/douyin.rs b/crates/danmaku/src/protocols/douyin.rs index 6e74915..9af8721 100644 --- a/crates/danmaku/src/protocols/douyin.rs +++ b/crates/danmaku/src/protocols/douyin.rs @@ -22,7 +22,12 @@ use crate::protocols::{ConnectionInfo, DecodeResult, HeartbeatConfig, Platform, /// Heartbeat packet. const HEARTBEAT: &[u8] = b":\x02hb"; const USER_AGENT_STRING: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"; -const DOUYIN_WS_URL: &str = "wss://webcast100-ws-web-lf.douyin.com/webcast/im/push/v2/"; +const DOUYIN_WS_HOSTS: &[&str] = &[ + "wss://webcast100-ws-web-lq.douyin.com/webcast/im/push/v2/", + "wss://webcast100-ws-web-hl.douyin.com/webcast/im/push/v2/", + "wss://webcast100-ws-web-lf.douyin.com/webcast/im/push/v2/", +]; +const DEFAULT_TTWID: &str = "1%7Cu7ogdHsSmHtxbt4hjDCNvcLfVJz78CTM0TTWU8Hio8w%7C1751545220%7C18aac967e501e9d6c13384335ced3523c46a0b1cc4535c7213bc2506a7f462c8"; const XBOGUS_ALPHABET: &[u8; 64] = b"Dkdpgh4ZKsQB80/Mfvw36XI1R25+WUAlEi7NLboqYTOPuzmFjJnryx9HVGcaStCe"; const STANDARD_ALPHABET: &[u8; 64] = @@ -84,7 +89,7 @@ impl Douyin { } /// Build WebSocket URL with parameters. - fn build_ws_url(room_id: &str) -> String { + fn build_ws_url(room_id: &str, host: &str) -> String { let user_unique_id = Self::generate_user_unique_id(); let version_code = "180800"; let webcast_sdk_version = "1.0.15"; @@ -136,11 +141,20 @@ impl Douyin { let query: String = ws_params .iter() - .map(|(k, v)| format!("{}={}", k, urlencoding::encode(v))) + .map(|(k, v)| format!("{}={}", k, v)) .collect::>() .join("&"); - format!("{}?{}", DOUYIN_WS_URL, query) + format!("{}?{}", host, query) + } + + fn build_connection_info(room_id: &str) -> ConnectionInfo { + let urls = DOUYIN_WS_HOSTS + .iter() + .map(|host| Self::build_ws_url(room_id, host)) + .collect::>(); + let mut iter = urls.into_iter(); + ConnectionInfo::new(iter.next().unwrap()).with_fallback_ws_urls(iter.collect()) } fn default_headers(context: &PlatformContext) -> HeaderMap { @@ -173,10 +187,22 @@ impl Douyin { ); } - if let Some(cookie) = context.cookie.as_deref().filter(|value| !value.is_empty()) { - if let Ok(value) = HeaderValue::from_str(cookie) { - headers.insert(COOKIE, value); - } + let cookie = context + .cookie + .as_deref() + .filter(|value| !value.is_empty()) + .map(|value| { + if value.contains("ttwid") { + value.to_string() + } else if value.ends_with(';') { + format!("{}ttwid={};", value, DEFAULT_TTWID) + } else { + format!("{};ttwid={};", value, DEFAULT_TTWID) + } + }) + .unwrap_or_else(|| format!("ttwid={};", DEFAULT_TTWID)); + if let Ok(value) = HeaderValue::from_str(&cookie) { + headers.insert(COOKIE, value); } headers @@ -297,7 +323,8 @@ impl Douyin { fn build_ack(log_id: u64, internal_ext: &str) -> Vec { let mut writer = ProtoWriter::new(); writer.write_varint_field(2, log_id); - writer.write_string(7, internal_ext); + writer.write_string(7, "ack"); + writer.write_bytes(8, internal_ext.as_bytes()); writer.into_buffer() } } @@ -429,10 +456,10 @@ impl Platform for Douyin { ); } - let ws_url = Self::build_ws_url(&room_id); - debug!("Douyin WebSocket URL: {}", ws_url); + let ws_info = Self::build_connection_info(&room_id); + debug!("Douyin WebSocket URL: {}", ws_info.ws_url); - Ok(ConnectionInfo::new(ws_url).with_headers(Self::default_headers(context))) + Ok(ws_info.with_headers(Self::default_headers(context))) } fn heartbeat_config(&self) -> HeartbeatConfig { @@ -514,8 +541,8 @@ mod tests { let fields = reader.parse_all(); assert_eq!(fields[&2][0].as_u64(), Some(12345)); - assert_eq!(fields[&7][0].as_str(), Some("internal_src:dim|seq:1")); - assert!(!fields.contains_key(&8)); + assert_eq!(fields[&7][0].as_str(), Some("ack")); + assert_eq!(fields[&8][0].as_str(), Some("internal_src:dim|seq:1")); } #[test] diff --git a/crates/danmaku/src/protocols/mod.rs b/crates/danmaku/src/protocols/mod.rs index 599972c..86f40a1 100644 --- a/crates/danmaku/src/protocols/mod.rs +++ b/crates/danmaku/src/protocols/mod.rs @@ -141,6 +141,8 @@ pub struct DecodeResult { pub events: Vec, /// Optional acknowledgment packet to send back (e.g., for Douyin). pub ack: Option>, + /// Whether the acknowledgment should be sent as a text WebSocket message. + pub ack_is_text: bool, } impl DecodeResult { @@ -151,12 +153,24 @@ impl DecodeResult { /// Create a result with events. pub fn with_events(events: Vec) -> Self { - Self { events, ack: None } + Self { + events, + ack: None, + ack_is_text: false, + } } - /// Add an acknowledgment packet. + /// Add a binary acknowledgment packet. pub fn with_ack(mut self, ack: Vec) -> Self { self.ack = Some(ack); + self.ack_is_text = false; + self + } + + /// Add a text acknowledgment packet. + pub fn with_text_ack(mut self, ack: impl Into) -> Self { + self.ack = Some(ack.into().into_bytes()); + self.ack_is_text = true; self } } diff --git a/crates/danmaku/src/protocols/twitch.rs b/crates/danmaku/src/protocols/twitch.rs index 69ca3d6..b2693df 100644 --- a/crates/danmaku/src/protocols/twitch.rs +++ b/crates/danmaku/src/protocols/twitch.rs @@ -8,8 +8,6 @@ //! - Send "PING" heartbeat every 40 seconds //! - Parse PRIVMSG for chat messages -use std::time::Duration; - use async_trait::async_trait; use regex::Regex; @@ -86,20 +84,17 @@ impl Platform for Twitch { // IRC registration commands let reg_data = vec![ - RegistrationData::Text( - "CAP REQ :twitch.tv/tags twitch.tv/commands twitch.tv/membership".to_string(), - ), - RegistrationData::Text("PASS SCHMOOPIIE".to_string()), + RegistrationData::Text("CAP REQ :twitch.tv/tags twitch.tv/commands".to_string()), + RegistrationData::Text("PASS oauth:".to_string()), RegistrationData::Text(format!("NICK {}", nick)), - RegistrationData::Text(format!("USER {} 8 * :{}", nick, nick)), RegistrationData::Text(format!("JOIN #{}", room_id)), ]; - Ok(ConnectionInfo::new("wss://irc-ws.chat.twitch.tv").with_registration(reg_data)) + Ok(ConnectionInfo::new("wss://irc-ws.chat.twitch.tv:443").with_registration(reg_data)) } fn heartbeat_config(&self) -> HeartbeatConfig { - HeartbeatConfig::text("PING", Duration::from_secs(40)) + HeartbeatConfig::none() } fn decode_message(&self, msg: &[u8]) -> Result { @@ -110,11 +105,16 @@ impl Platform for Twitch { let mut events = Vec::new(); for line in text.lines() { - // Skip empty lines + let line = line.trim(); if line.is_empty() { continue; } + if line.starts_with("PING") { + let pong_target = line.strip_prefix("PING ").unwrap_or(":tmi.twitch.tv"); + return Ok(DecodeResult::empty().with_text_ack(format!("PONG {}", pong_target))); + } + // Try to parse PRIVMSG if let Some(content_cap) = self.privmsg_regex.captures(line) { if let Some(content) = content_cap.get(1) { diff --git a/crates/danmaku/src/protocols/youtube.rs b/crates/danmaku/src/protocols/youtube.rs index 4f1d4af..60c311f 100644 --- a/crates/danmaku/src/protocols/youtube.rs +++ b/crates/danmaku/src/protocols/youtube.rs @@ -36,10 +36,24 @@ impl YouTube { } fn extract_channel_id(text: &str) -> Option { - let re = Regex::new(r#"\\?"channelId\\?":\\?"([^"\\]{24})\\?""#).ok()?; - re.captures(text) - .and_then(|c| c.get(1)) - .map(|m| m.as_str().to_string()) + [ + r#""channelId":"(UC[^"\\]{22})""#, + r#"\\"channelId\\":\\"(UC[^"\\]{22})\\""#, + r#""externalChannelId":"(UC[^"\\]{22})""#, + r#"\\"externalChannelId\\":\\"(UC[^"\\]{22})\\""#, + r#""browseId":"(UC[^"\\]{22})""#, + r#"\\"browseId\\":\\"(UC[^"\\]{22})\\""#, + r#"/channel/(UC[A-Za-z0-9_-]{22})"#, + r#"(UC[A-Za-z0-9_-]{22})"#, + ] + .into_iter() + .find_map(|pattern| { + Regex::new(pattern) + .ok()? + .captures(text) + .and_then(|c| c.get(1)) + .map(|m| m.as_str().to_string()) + }) } fn extract_live_video_id(text: &str) -> Option { @@ -99,7 +113,10 @@ impl YouTube { async fn initial_continuation(&self, url: &str) -> Result { let channel_id = self.resolve_channel_id(url).await?; - let video_id = self.resolve_live_video_id(&channel_id).await?; + let video_id = match extract_video_id(url) { + Some(video_id) => video_id, + None => self.resolve_live_video_id(&channel_id).await?, + }; Ok(liveparam(&video_id, &channel_id, 1, false)) } diff --git a/pyproject.toml b/pyproject.toml index 9883e3f..81775d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,36 +20,17 @@ keywords = ["bilibili douyu huya"] urls = { Homepage = "https://github.com/ForgQi/bilibiliupload" } requires-python = ">= 3.9" dependencies = [ - "tomli-w>=1.0.0", - "psutil >= 5.4.6", - "aiofiles >= 22.1.0", "yt-dlp >= 2025.7.21", + "aiohttp[speedups] >= 3.9.5", "Pillow >= 5.2.0", - "aiohttp[speedups] >= 3.9.5", # aio-libs/aiohttp#10519 - "aiohttp_cors >= 0.7.0", "requests >= 2.32.3", - "httpx[http2] >= 0.28.1", - "httpcore >= 1.0.7", - "PyYAML >= 4.2b1", "streamlink >= 7.5.0", - "ykdl >= 1.8.0", "rsa >= 4.6", - "tomli >= 1.1.0; python_version < '3.11'", - "protobuf >= 4.23.0", - "SQLAlchemy >= 2.0.0", - "alembic >= 1.12.1", - "m3u8 >= 6.0.0", - "truststore >= 0.10.1; python_version >= '3.10'", - "async-lru >= 2.0.5", - "gmssl >= 3.2.2" ] dynamic = ["version"] -[project.optional-dependencies] -selenium = ["selenium >= 3.8.1"] - [project.scripts] -biliup = "biliup.__main__:arg_parser" +biliup = "biliup.__main__:main" #[tool.setuptools] #license-files = ["LICENSE"]