From 4a67f773ac366dda14ef2b2a2d797ce7a05a18fe Mon Sep 17 00:00:00 2001 From: kelriclink Date: Fri, 6 Mar 2026 23:00:54 +0800 Subject: [PATCH] Ship admin, YouTube, and deployment updates --- .dockerignore | 15 + .env.example | 14 +- .gitignore | 6 +- Dockerfile | 2 +- README.md | 380 ++++++++- compose.yml | 95 +++ deploy_prod/docker-compose.yml | 8 +- docker-compose.yml | 106 +-- docker/entrypoint.sh | 9 +- docs/PROJECT_SPEC.md | 21 +- scripts/dev_down.sh | 3 +- scripts/dev_health.sh | 10 +- scripts/dev_logs.sh | 4 +- scripts/dev_up.sh | 12 +- scripts/dev_web.sh | 3 +- scripts/smoke_local.sh | 2 +- src/videoroll/apps/bilibili_publisher/main.py | 2 + .../apps/bilibili_publisher/worker.py | 2 + .../apps/orchestrator_api/admin_auth_store.py | 190 +++++ src/videoroll/apps/orchestrator_api/main.py | 634 +++++++++++++- .../apps/orchestrator_api/schemas.py | 30 +- .../orchestrator_api/youtube_downloader.py | 62 +- src/videoroll/apps/subtitle_service/main.py | 319 +++++-- .../apps/subtitle_service/processing.py | 93 ++- .../subtitle_service/render_queue_store.py | 51 +- .../apps/subtitle_service/schemas.py | 30 +- .../apps/subtitle_service/task_title_store.py | 47 +- src/videoroll/apps/subtitle_service/worker.py | 787 +++++++++++++++--- src/videoroll/apps/youtube_ingest/main.py | 40 +- src/videoroll/apps/youtube_meta_store.py | 49 ++ src/videoroll/apps/youtube_settings_store.py | 263 +++++- src/videoroll/config.py | 18 +- src/videoroll/db/auto_migrate.py | 76 ++ src/videoroll/db/models.py | 23 + src/videoroll/storage/s3.py | 7 +- src/videoroll/utils/internal_api_token.py | 16 + {web => src/web}/.dockerignore | 0 {web => src/web}/Dockerfile | 0 {web => src/web}/index.html | 0 {web => src/web}/nginx.conf | 16 +- {web => src/web}/package-lock.json | 0 {web => src/web}/package.json | 0 {web => src/web}/postcss.config.js | 0 src/web/src/App.tsx | 130 +++ src/web/src/components/AuthGate.tsx | 189 +++++ .../web}/src/components/StatusBadge.tsx | 0 {web => src/web}/src/lib/http.ts | 3 +- {web => src/web}/src/lib/types.ts | 1 + {web => src/web}/src/lib/urls.ts | 12 +- {web => src/web}/src/main.tsx | 0 {web => src/web}/src/pages/DashboardPage.tsx | 0 .../web}/src/pages/RenderQueuePage.tsx | 75 +- .../web}/src/pages/SettingsASRPage.tsx | 0 .../web}/src/pages/SettingsAutoPage.tsx | 0 .../web}/src/pages/SettingsBilibiliPage.tsx | 0 .../web}/src/pages/SettingsStoragePage.tsx | 0 .../web}/src/pages/SettingsTranslatePage.tsx | 0 src/web/src/pages/SettingsYouTubePage.tsx | 303 +++++++ {web => src/web}/src/pages/TaskDetailPage.tsx | 125 ++- {web => src/web}/src/pages/TaskNewPage.tsx | 0 {web => src/web}/src/pages/TasksPage.tsx | 36 +- {web => src/web}/src/pages/VideosPage.tsx | 0 .../web}/src/pages/YouTubeSourcesPage.tsx | 0 {web => src/web}/src/styles.css | 0 {web => src/web}/tailwind.config.js | 0 {web => src/web}/tsconfig.json | 0 src/web/vite.config.ts | 19 + web/src/App.tsx | 96 --- web/src/pages/SettingsYouTubePage.tsx | 156 ---- web/vite.config.ts | 11 - 70 files changed, 3851 insertions(+), 750 deletions(-) create mode 100644 .dockerignore create mode 100644 compose.yml create mode 100644 src/videoroll/apps/orchestrator_api/admin_auth_store.py create mode 100644 src/videoroll/apps/youtube_meta_store.py create mode 100644 src/videoroll/db/auto_migrate.py create mode 100644 src/videoroll/utils/internal_api_token.py rename {web => src/web}/.dockerignore (100%) rename {web => src/web}/Dockerfile (100%) rename {web => src/web}/index.html (100%) rename {web => src/web}/nginx.conf (74%) rename {web => src/web}/package-lock.json (100%) rename {web => src/web}/package.json (100%) rename {web => src/web}/postcss.config.js (100%) create mode 100644 src/web/src/App.tsx create mode 100644 src/web/src/components/AuthGate.tsx rename {web => src/web}/src/components/StatusBadge.tsx (100%) rename {web => src/web}/src/lib/http.ts (79%) rename {web => src/web}/src/lib/types.ts (98%) rename {web => src/web}/src/lib/urls.ts (67%) rename {web => src/web}/src/main.tsx (100%) rename {web => src/web}/src/pages/DashboardPage.tsx (100%) rename {web => src/web}/src/pages/RenderQueuePage.tsx (69%) rename {web => src/web}/src/pages/SettingsASRPage.tsx (100%) rename {web => src/web}/src/pages/SettingsAutoPage.tsx (100%) rename {web => src/web}/src/pages/SettingsBilibiliPage.tsx (100%) rename {web => src/web}/src/pages/SettingsStoragePage.tsx (100%) rename {web => src/web}/src/pages/SettingsTranslatePage.tsx (100%) create mode 100644 src/web/src/pages/SettingsYouTubePage.tsx rename {web => src/web}/src/pages/TaskDetailPage.tsx (93%) rename {web => src/web}/src/pages/TaskNewPage.tsx (100%) rename {web => src/web}/src/pages/TasksPage.tsx (76%) rename {web => src/web}/src/pages/VideosPage.tsx (100%) rename {web => src/web}/src/pages/YouTubeSourcesPage.tsx (100%) rename {web => src/web}/src/styles.css (100%) rename {web => src/web}/tailwind.config.js (100%) rename {web => src/web}/tsconfig.json (100%) create mode 100644 src/web/vite.config.ts delete mode 100644 web/src/App.tsx delete mode 100644 web/src/pages/SettingsYouTubePage.tsx delete mode 100644 web/vite.config.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..dc1b1a6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +.git +.venv +__pycache__ +*.py[cod] + +# Local data / scratch +data +tmp + +# Frontend lives under src/ but isn't needed for backend image builds +src/web + +# Large vendored folders not used in the backend image +biliup-master +bilibili-API-collect-main diff --git a/.env.example b/.env.example index 82eaa74..48812f0 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,6 @@ -DATABASE_URL=postgresql+psycopg://videoroll:videoroll@postgres:5432/videoroll +# Current compose files do not start PostgreSQL for you. +# If PostgreSQL runs on the host, host.docker.internal works from the app container. +DATABASE_URL=postgresql+psycopg://videoroll:videoroll@host.docker.internal:5432/videoroll REDIS_URL=redis://redis:6379/0 S3_ENDPOINT_URL=http://minio:9000 @@ -20,7 +22,7 @@ SUBTITLE_WHISPER_MODEL_DIR=/models/whisper FFMPEG_PATH=ffmpeg WORK_DIR=/tmp/videoroll -YOUTUBE_USER_AGENT=videoroll/0.1 +YOUTUBE_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" # Optional: YouTube downloader (yt-dlp) # - Put a Netscape cookies.txt at ./data/secrets/youtube_cookies.txt (mounted to /secrets) @@ -28,8 +30,7 @@ YOUTUBE_USER_AGENT=videoroll/0.1 # YOUTUBE_COOKIE_FILE=/secrets/youtube_cookies.txt # Optional proxy for yt-dlp (http/socks5): # YOUTUBE_PROXY=http://host.docker.internal:7890 -# Advanced: yt-dlp format string / extractor args -# YOUTUBE_YTDLP_FORMAT=bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best +# Advanced: yt-dlp extractor args # YOUTUBE_EXTRACTOR_ARGS_JSON={"youtube":{"player_client":["tv","android_sdkless","web"]}} BILIBILI_PUBLISH_MODE=mock # Bilibili uploads are rate-limited; keep publish worker single-threaded by default. @@ -40,3 +41,8 @@ INSTALL_ASR=1 # Optional: allow browser origins (comma-separated) CORS_ALLOW_ORIGINS=http://localhost:3000,http://127.0.0.1:3000 + +# Optional publish bindings +# PUBLISH_ADDR=127.0.0.1 +# WEB_PORT=3000 +# MINIO_CONSOLE_PORT=9001 diff --git a/.gitignore b/.gitignore index 671d0b3..2e3a692 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,6 @@ build/ data/ -web/node_modules/ -web/dist/ -web/*.tsbuildinfo +src/web/node_modules/ +src/web/dist/ +src/web/*.tsbuildinfo diff --git a/Dockerfile b/Dockerfile index 0603a06..591ab7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN INSTALL_ASR="$INSTALL_ASR" python -c "import os, tomllib; from pathlib impor && pip install --no-cache-dir -U pip \ && pip install --no-cache-dir -r /tmp/requirements.txt -COPY src ./src +COPY src/videoroll ./src/videoroll COPY docs ./docs COPY docker ./docker diff --git a/README.md b/README.md index 8df27a5..cca6dd7 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,369 @@ # videoroll -模块化视频处理流水线（合规版）：内容接入 → 字幕/翻译 →（可选）压制/封装 →（可选）投稿到哔哩哔哩。 +模块化视频处理流水线：内容接入 -> 字幕/翻译 -> 压制/封装 -> 可选投稿到哔哩哔哩。 -详细规格：`docs/PROJECT_SPEC.md` +详细规格见 [docs/PROJECT_SPEC.md](docs/PROJECT_SPEC.md)。 -## Quickstart（后端骨架） +## 当前状态 -1) 准备环境变量： -- `cp .env.example .env` +- Web UI 已接入管理密码登录，首次打开需要设置 admin password。 +- 后端容器会同时启动 `uvicorn`、字幕 Celery worker、投稿 Celery worker。 +- YouTube 下载统一使用 `yt-dlp` 默认格式选择，不再额外强制 `mp4+m4a`。 +- YouTube 支持在 Web UI 中配置代理、测试出口连通性、保存 `cookies.txt`，并显示 cookies 摘要。 +- 若未配置 `YOUTUBE_COOKIE_FILE`，已保存到数据库的 YouTube cookies 会在下载/元信息提取时写入临时文件供 `yt-dlp` 使用。 +- YouTube 触发风控时，接口会返回更明确的 cookies / proxy 提示。 +- Bilibili 投稿支持在 Web UI 中保存 Cookie、测试登录、维护默认投稿模板。 -2) 启动： -- `./scripts/dev_up.sh`（包含后端 + Web） +## 架构概览 -3) 核心服务端口： -- API（monolith）：`http://localhost:8000/docs`（`src/videoroll/apps/monolith/main.py`） -- Subtitle Service：`http://localhost:8000/subtitle-service/docs` -- YouTube Ingest：`http://localhost:8000/youtube-ingest/docs` -- Bilibili Publisher：`http://localhost:8000/bilibili-publisher/docs` -- Web UI：`http://localhost:3000` +当前默认部署使用单容器应用服务： -## 最小流程（本地视频 → 字幕/硬字幕） +- `app` + 运行 `videoroll.apps.monolith.main:app` + 挂载子服务： + - `/api` + - `/api/subtitle-service` + - `/api/youtube-ingest` + - `/api/bilibili-publisher` + 同时启动两个 Celery worker： + - `subtitle` + - `publish` +- `web` + 提供前端 UI，并将同源 `/api` 反代到后端 +- `redis` + Celery broker / backend +- `minio` + 资产存储 +- `postgresql` + 需要你自行提供；当前仓库的 compose 文件不再内置 PostgreSQL -1) 创建任务（示例：本地自制/已授权内容）： -- `POST http://localhost:8000/tasks` +## 环境要求 -2) 上传视频： -- `POST http://localhost:8000/tasks/{task_id}/upload/video`（multipart file） +- Docker Engine + Docker Compose Plugin +- 一个可用的 PostgreSQL 16+ 实例 +- 访问 YouTube 所需的网络环境 +- 如果要使用真实 ASR： + `INSTALL_ASR=1` -3) 触发字幕任务： -- `POST http://localhost:8000/tasks/{task_id}/actions/subtitle` +可选但常见： -字幕任务产物会写入 MinIO（默认 bucket：`videoroll`），并在 `GET /tasks/{task_id}/assets` 可看到对应 `storage_key`。 +- Python 3.12+，用于本地运行辅助脚本 +- Node.js 20+，用于单独启动前端开发服务器 `./scripts/dev_web.sh` -## ASR 说明 +## 安装与启动 -- 默认启用 `SUBTITLE_ASR_ENGINE=faster-whisper`（默认模型：`SUBTITLE_WHISPER_MODEL=tiny`）。 -- 模型管理：Web `Settings → ASR/Whisper`（`http://localhost:3000/settings/asr`），模型会落到本地目录 `data/models/whisper/`（容器内路径：`/models/whisper`）。 -- 如需回退到示例字幕：将 `SUBTITLE_ASR_ENGINE=mock`。 +### 1. 克隆仓库 -## 翻译（可选：OpenAI） +```bash +git clone git@github.com:kelriclink/videoroll.git +cd videoroll +``` -- 任务详情页可开启翻译并选择 `translate_provider=openai`。 -- 配置页：`http://localhost:3000/settings/translate`（可查看当前配置并测试）。 -- OpenAI 配置在 Web 设置页保存（不再通过 `.env`）。密钥会加密存储在 DB 中，解密密钥保存在本地目录 `data/secrets/fernet.key`（容器内：`/secrets/fernet.key`）。 +### 2. 准备 PostgreSQL + +当前 compose 不会帮你起数据库，你需要自己提供一个 PostgreSQL，并把它暴露给 `app` 容器。 + +一种简单方式是在宿主机单独跑一个 PostgreSQL： + +```bash +docker run -d \ + --name videoroll-postgres \ + -e POSTGRES_USER=videoroll \ + -e POSTGRES_PASSWORD=videoroll \ + -e POSTGRES_DB=videoroll \ + -p 5432:5432 \ + postgres:16 +``` + +### 3. 创建 `.env` + +```bash +cp .env.example .env +``` + +至少确认这些变量： + +- `DATABASE_URL` +- `REDIS_URL` +- `S3_*` +- `INSTALL_ASR` + +如果数据库跑在宿主机，推荐这样配置： + +```env +DATABASE_URL=postgresql+psycopg://videoroll:videoroll@host.docker.internal:5432/videoroll +REDIS_URL=redis://redis:6379/0 +S3_ENDPOINT_URL=http://minio:9000 +S3_ACCESS_KEY_ID=videoroll +S3_SECRET_ACCESS_KEY=videorollsecret +S3_BUCKET=videoroll +S3_REGION_NAME=us-east-1 +S3_USE_SSL=false +``` + +说明： + +- `app` 容器已注入 `host.docker.internal` 映射，因此 Linux/macOS/Windows 都可以按这个地址连接宿主机数据库。 +- 如果你的 PostgreSQL 在别的主机或别的 Docker 网络里，把 `host.docker.internal` 改成实际地址即可。 + +### 4. 启动服务 + +推荐直接使用仓库脚本： + +```bash +./scripts/dev_up.sh +``` + +如果你想手动启动，请显式指定 compose 文件，避免仓库里同时存在 `compose.yml` 和 `docker-compose.yml` 时的歧义： + +```bash +docker compose -f docker-compose.yml --env-file .env up --build -d +``` + +### 5. 首次登录 + +打开 Web UI： + +- `http://localhost:3000` + +首次访问会进入 admin password 初始化页面： + +- 第一次使用：设置管理密码 +- 后续新设备或浏览器数据被清除后：输入管理密码登录 + +这是设备记住模式，密码不会每次都要求输入。 + +## 主要访问地址 + +- Web UI: `http://localhost:3000` +- Orchestrator API: `http://localhost:3000/api/docs` +- Subtitle Service: `http://localhost:3000/api/subtitle-service/docs` +- YouTube Ingest: `http://localhost:3000/api/youtube-ingest/docs` +- Bilibili Publisher: `http://localhost:3000/api/bilibili-publisher/docs` +- MinIO Console: `http://localhost:9001` + +健康检查： + +```bash +./scripts/dev_health.sh +``` + +查看日志： + +```bash +./scripts/dev_logs.sh +``` + +停止服务： + +```bash +./scripts/dev_down.sh +``` + +## 目录与持久化数据 + +- `data/minio` + MinIO 数据目录 +- `data/models` + Whisper 模型目录 +- `data/secrets` + 本地密钥和机密目录 +- `data/secrets/fernet.key` + 首次保存加密配置时自动生成，用于加密存储 OpenAI key、YouTube cookies、Bilibili cookies 等敏感数据 + +不要随意删除 `data/secrets/fernet.key`，否则之前存储在数据库里的加密配置将无法解密。 + +## 配置说明 + +### 核心环境变量 + +- `DATABASE_URL` + PostgreSQL 连接串 +- `REDIS_URL` + Redis 连接串 +- `S3_ENDPOINT_URL` + MinIO / S3 地址 +- `S3_ACCESS_KEY_ID` +- `S3_SECRET_ACCESS_KEY` +- `S3_BUCKET` +- `S3_REGION_NAME` +- `S3_USE_SSL` +- `FFMPEG_PATH` + 默认 `ffmpeg` +- `WORK_DIR` + 默认 `/tmp/videoroll` +- `INSTALL_ASR` + `1` 时构建真实 ASR 依赖；`0` 时减小镜像体积 +- `PUBLISH_ADDR` + Web / MinIO console 的监听地址，默认 `127.0.0.1` +- `WEB_PORT` + Web 端口，默认 `3000` +- `MINIO_CONSOLE_PORT` + MinIO console 端口，默认 `9001` + +### YouTube 配置 + +推荐通过 Web UI 管理： + +- `Settings -> YouTube` + +支持的内容： + +- 保存 / 清空代理 +- 测试指定 URL 是否能通过当前代理访问 +- 粘贴并保存 Netscape 格式的 `cookies.txt` +- 启用或禁用已保存 cookies +- 查看 cookies 是否包含登录态 / 是否可能包含 bot-check 豁免 + +当前默认行为： + +- 下载直接使用 `yt-dlp` 默认格式选择 +- 不再提供 `YOUTUBE_YTDLP_FORMAT` +- 默认 `User-Agent` 为浏览器 UA，而不是自定义短 UA + +可选环境变量： + +- `YOUTUBE_USER_AGENT` +- `YOUTUBE_COOKIE_FILE` +- `YOUTUBE_PROXY` +- `YOUTUBE_EXTRACTOR_ARGS_JSON` + +说明： + +- `YOUTUBE_COOKIE_FILE` 是可选的；如果不配，系统会优先使用你在 UI 中保存的 cookies。 +- `YOUTUBE_EXTRACTOR_ARGS_JSON` 用于高级场景，例如： + +```env +YOUTUBE_EXTRACTOR_ARGS_JSON={"youtube":{"player_client":["tv","android_sdkless","web"]}} +``` + +YouTube 风控注意事项： + +- 浏览器导出 cookies 时，请使用和服务端相同的代理 / 出口 IP。 +- 如果浏览器中出现过“确认你不是机器人”，通常需要在同一出口 IP 下先完成验证，再重新导出 cookies。 +- 只有 `VISITOR_INFO1_LIVE` 一类 cookie 通常不够，建议确保 cookies 中包含登录态。 +- YouTube cookies 轮换很快，过期后需要重新导出。 + +### ASR / 翻译 / 自动模式 + +相关页面： + +- `Settings -> ASR` +- `Settings -> Translate` +- `Settings -> Auto` + +说明： + +- 默认 ASR 为 `faster-whisper` +- `Settings -> Auto` 用于配置 YouTube 自动模式的默认参数： + - 字幕格式 + - 是否 burn-in / soft-sub + - 编码参数 + - 翻译目标语言与 provider + - 是否自动投稿 Bilibili + +如果只是想做轻量演示，可把： + +```env +INSTALL_ASR=0 +SUBTITLE_ASR_ENGINE=mock +``` + +### Bilibili 配置 + +相关页面： + +- `Settings -> Bilibili` + +支持： + +- 保存 Bilibili Cookie +- 检查是否成功解析出 `SESSDATA` 和 `bili_jct` +- 测试当前 Cookie 是否可登录 +- 保存默认投稿模板 `default_meta` + +真实投稿前，建议至少确认： + +- Cookie 有效 +- 含 `bili_jct` +- 默认模板里的分区、标签、简介符合你的账号需求 + +## 使用流程 + +### 本地视频 + +1. `New Task -> 本地上传` +2. 上传视频 +3. 在任务详情页执行字幕、翻译、压制、投稿 + +### YouTube 手动模式 + +1. `New Task -> YouTube 链接` +2. 填写授权类型和可选证明链接 +3. 创建任务 +4. 在任务详情页手动执行： + - 获取元信息 + - 下载 YouTube 视频 + - 生成字幕 / 翻译 + - 压制或投稿 + +### YouTube 自动模式 + +1. 先在 `Settings -> Auto` 配好默认参数 +2. `New Task -> YouTube 自动模式` +3. 系统会按配置执行： + - 下载 + - 字幕 / 翻译 + - burn-in 或 soft-sub + - 可选自动投稿 Bilibili + +## 常见问题 + +### 1. 启动后直接报数据库连接错误 + +优先检查： + +- PostgreSQL 是否真的已经启动 +- `.env` 里的 `DATABASE_URL` 是否指向了可达地址 +- 如果数据库在宿主机，是否使用了 `host.docker.internal` + +### 2. 打开页面后要求设置或输入密码 + +这是预期行为。所有非 `/auth/*` 和 `/health` 接口都受 admin 设备登录保护。 + +### 3. YouTube 下载报 `Sign in to confirm you're not a bot` + +先检查： + +- 当前网络 / 代理是否可稳定访问 YouTube +- 是否已经在 `Settings -> YouTube` 保存有效的 `cookies.txt` +- cookies 是否包含登录态 +- 导出 cookies 时是否使用了同一代理 / 同一出口 IP + +### 4. YouTube 下载报 `Requested format is not available` + +当前代码已经统一改成 `yt-dlp` 默认格式选择；如果你还看到这个错误，通常不是项目手写格式规则的问题，而是： + +- YouTube 当前返回的可用流本身异常 +- cookies / 风控导致某些格式不可见 +- `yt-dlp` / YouTube 抽取链路临时波动 + +优先更新 cookies、检查代理，再看是否需要调整 `YOUTUBE_EXTRACTOR_ARGS_JSON`。 + +### 5. OpenAI 配置存在哪里 + +OpenAI 配置通过 Web UI 保存到数据库，密钥内容用 `data/secrets/fernet.key` 加密。 + +## 前端开发模式 + +如果你只想本地调前端： + +```bash +./scripts/dev_web.sh +``` + +这会在 `src/web` 下启动 Vite dev server。默认情况下，生产部署仍建议使用 `web` 容器。 ## 合规边界 -本项目仅用于处理你拥有版权/已获授权/明确允许再分发的视频内容；不要用于批量搬运或抓取他人热门内容。 +本项目仅用于处理你拥有版权、已获授权、或明确允许再分发的视频内容。不要将其用于批量搬运、绕过平台限制、或处理无授权内容。 diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..14c2edb --- /dev/null +++ b/compose.yml @@ -0,0 +1,95 @@ +services: + redis: + image: redis:7 + restart: unless-stopped + + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-videoroll} + MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-videorollsecret} + ports: + - "${PUBLISH_ADDR:-127.0.0.1}:${MINIO_CONSOLE_PORT:-9001}:9001" + volumes: + - ./data/minio:/data + restart: unless-stopped + + minio-init: + image: minio/mc:latest + depends_on: + - minio + entrypoint: /bin/sh + command: > + -c " + until mc alias set local http://minio:9000 ${S3_ACCESS_KEY_ID:-videoroll} ${S3_SECRET_ACCESS_KEY:-videorollsecret}; do + echo 'waiting for minio...'; + sleep 1; + done && + mc mb -p local/${S3_BUCKET:-videoroll} || true + " + restart: "no" + + app: + build: + context: . + args: + INSTALL_ASR: ${INSTALL_ASR:-1} + image: videoroll:prod + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + S3_ENDPOINT_URL: ${S3_ENDPOINT_URL} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + S3_BUCKET: ${S3_BUCKET} + S3_REGION_NAME: ${S3_REGION_NAME:-us-east-1} + S3_USE_SSL: ${S3_USE_SSL:-false} + SUBTITLE_ASR_ENGINE: ${SUBTITLE_ASR_ENGINE:-faster-whisper} + SUBTITLE_WHISPER_MODEL: ${SUBTITLE_WHISPER_MODEL:-tiny} + SUBTITLE_WHISPER_DEVICE: ${SUBTITLE_WHISPER_DEVICE:-cpu} + SUBTITLE_WHISPER_COMPUTE_TYPE: ${SUBTITLE_WHISPER_COMPUTE_TYPE:-int8} + SUBTITLE_WHISPER_MODEL_DIR: ${SUBTITLE_WHISPER_MODEL_DIR:-/models/whisper} + FFMPEG_PATH: ${FFMPEG_PATH:-ffmpeg} + WORK_DIR: ${WORK_DIR:-/tmp/videoroll} + UVICORN_ROOT_PATH: ${UVICORN_ROOT_PATH:-/api} + YOUTUBE_USER_AGENT: "${YOUTUBE_USER_AGENT:-Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36}" + YOUTUBE_COOKIE_FILE: ${YOUTUBE_COOKIE_FILE:-} + YOUTUBE_PROXY: ${YOUTUBE_PROXY:-} + YOUTUBE_EXTRACTOR_ARGS_JSON: ${YOUTUBE_EXTRACTOR_ARGS_JSON:-} + BILIBILI_PUBLISH_MODE: ${BILIBILI_PUBLISH_MODE:-web} + CELERY_PUB_CONCURRENCY: ${CELERY_PUB_CONCURRENCY:-1} + CORS_ALLOW_ORIGINS: ${CORS_ALLOW_ORIGINS:-http://localhost:3000,http://127.0.0.1:3000} + SUBTITLE_SERVICE_URL: ${SUBTITLE_SERVICE_URL:-http://localhost:8000/subtitle-service} + YOUTUBE_INGEST_URL: ${YOUTUBE_INGEST_URL:-http://localhost:8000/youtube-ingest} + BILIBILI_PUBLISHER_URL: ${BILIBILI_PUBLISHER_URL:-http://localhost:8000/bilibili-publisher} + STORAGE_CLEANUP_INTERVAL_SECONDS: ${STORAGE_CLEANUP_INTERVAL_SECONDS:-3600} + volumes: + - ./data/models:/models + - ./data/secrets:/secrets + extra_hosts: + - host.docker.internal:host-gateway + depends_on: + redis: + condition: service_started + minio: + condition: service_started + minio-init: + condition: service_completed_successfully + restart: unless-stopped + + web: + build: + context: ./src/web + args: + VITE_ORCHESTRATOR_URL: ${VITE_ORCHESTRATOR_URL:-} + VITE_SUBTITLE_SERVICE_URL: ${VITE_SUBTITLE_SERVICE_URL:-} + VITE_YOUTUBE_INGEST_URL: ${VITE_YOUTUBE_INGEST_URL:-} + VITE_BILIBILI_PUBLISHER_URL: ${VITE_BILIBILI_PUBLISHER_URL:-} + image: videoroll-web:prod + ports: + - "${PUBLISH_ADDR:-127.0.0.1}:${WEB_PORT:-3000}:80" + depends_on: + app: + condition: service_started + restart: unless-stopped diff --git a/deploy_prod/docker-compose.yml b/deploy_prod/docker-compose.yml index 843004d..14c2edb 100644 --- a/deploy_prod/docker-compose.yml +++ b/deploy_prod/docker-compose.yml @@ -52,10 +52,10 @@ services: SUBTITLE_WHISPER_MODEL_DIR: ${SUBTITLE_WHISPER_MODEL_DIR:-/models/whisper} FFMPEG_PATH: ${FFMPEG_PATH:-ffmpeg} WORK_DIR: ${WORK_DIR:-/tmp/videoroll} - YOUTUBE_USER_AGENT: ${YOUTUBE_USER_AGENT:-videoroll/0.1} + UVICORN_ROOT_PATH: ${UVICORN_ROOT_PATH:-/api} + YOUTUBE_USER_AGENT: "${YOUTUBE_USER_AGENT:-Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36}" YOUTUBE_COOKIE_FILE: ${YOUTUBE_COOKIE_FILE:-} YOUTUBE_PROXY: ${YOUTUBE_PROXY:-} - YOUTUBE_YTDLP_FORMAT: ${YOUTUBE_YTDLP_FORMAT:-bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best} YOUTUBE_EXTRACTOR_ARGS_JSON: ${YOUTUBE_EXTRACTOR_ARGS_JSON:-} BILIBILI_PUBLISH_MODE: ${BILIBILI_PUBLISH_MODE:-web} CELERY_PUB_CONCURRENCY: ${CELERY_PUB_CONCURRENCY:-1} @@ -64,8 +64,6 @@ services: YOUTUBE_INGEST_URL: ${YOUTUBE_INGEST_URL:-http://localhost:8000/youtube-ingest} BILIBILI_PUBLISHER_URL: ${BILIBILI_PUBLISHER_URL:-http://localhost:8000/bilibili-publisher} STORAGE_CLEANUP_INTERVAL_SECONDS: ${STORAGE_CLEANUP_INTERVAL_SECONDS:-3600} - ports: - - "${PUBLISH_ADDR:-127.0.0.1}:${ORCHESTRATOR_PORT:-8000}:8000" volumes: - ./data/models:/models - ./data/secrets:/secrets @@ -82,7 +80,7 @@ services: web: build: - context: ./web + context: ./src/web args: VITE_ORCHESTRATOR_URL: ${VITE_ORCHESTRATOR_URL:-} VITE_SUBTITLE_SERVICE_URL: ${VITE_SUBTITLE_SERVICE_URL:-} diff --git a/docker-compose.yml b/docker-compose.yml index de25549..14c2edb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,52 +1,7 @@ services: - postgres: - image: postgres:16 - environment: - POSTGRES_USER: videoroll - POSTGRES_PASSWORD: videoroll - POSTGRES_DB: videoroll - ports: - - "5432:5432" - volumes: - - postgres-data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U videoroll -d videoroll"] - interval: 5s - timeout: 5s - retries: 20 - redis: image: redis:7 - ports: - - "6379:6379" - - db-init: - build: - context: . - args: - INSTALL_ASR: ${INSTALL_ASR:-1} - image: videoroll:dev - environment: - DATABASE_URL: ${DATABASE_URL} - REDIS_URL: ${REDIS_URL} - S3_ENDPOINT_URL: ${S3_ENDPOINT_URL} - S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} - S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} - S3_BUCKET: ${S3_BUCKET} - S3_REGION_NAME: ${S3_REGION_NAME:-us-east-1} - S3_USE_SSL: ${S3_USE_SSL:-false} - command: > - python -c "from videoroll.config import get_orchestrator_settings; - from videoroll.db.base import Base; - from videoroll.db.session import get_engine; - import videoroll.db.models # noqa: F401 - ; - s=get_orchestrator_settings(); - Base.metadata.create_all(get_engine(s.database_url)); - print('db init ok')" - depends_on: - postgres: - condition: service_healthy + restart: unless-stopped minio: image: minio/minio:latest @@ -55,10 +10,10 @@ services: MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-videoroll} MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-videorollsecret} ports: - - "9000:9000" - - "9001:9001" + - "${PUBLISH_ADDR:-127.0.0.1}:${MINIO_CONSOLE_PORT:-9001}:9001" volumes: - - minio-data:/data + - ./data/minio:/data + restart: unless-stopped minio-init: image: minio/mc:latest @@ -67,26 +22,20 @@ services: entrypoint: /bin/sh command: > -c " - mc alias set local http://minio:9000 ${S3_ACCESS_KEY_ID:-videoroll} ${S3_SECRET_ACCESS_KEY:-videorollsecret} && + until mc alias set local http://minio:9000 ${S3_ACCESS_KEY_ID:-videoroll} ${S3_SECRET_ACCESS_KEY:-videorollsecret}; do + echo 'waiting for minio...'; + sleep 1; + done && mc mb -p local/${S3_BUCKET:-videoroll} || true " - - web: - build: - context: ./web - image: videoroll-web:dev - ports: - - "3000:80" - depends_on: - app: - condition: service_started + restart: "no" app: build: context: . args: INSTALL_ASR: ${INSTALL_ASR:-1} - image: videoroll:dev + image: videoroll:prod environment: DATABASE_URL: ${DATABASE_URL} REDIS_URL: ${REDIS_URL} @@ -103,33 +52,44 @@ services: SUBTITLE_WHISPER_MODEL_DIR: ${SUBTITLE_WHISPER_MODEL_DIR:-/models/whisper} FFMPEG_PATH: ${FFMPEG_PATH:-ffmpeg} WORK_DIR: ${WORK_DIR:-/tmp/videoroll} - YOUTUBE_USER_AGENT: ${YOUTUBE_USER_AGENT:-videoroll/0.1} + UVICORN_ROOT_PATH: ${UVICORN_ROOT_PATH:-/api} + YOUTUBE_USER_AGENT: "${YOUTUBE_USER_AGENT:-Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36}" YOUTUBE_COOKIE_FILE: ${YOUTUBE_COOKIE_FILE:-} YOUTUBE_PROXY: ${YOUTUBE_PROXY:-} - YOUTUBE_YTDLP_FORMAT: ${YOUTUBE_YTDLP_FORMAT:-bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best} YOUTUBE_EXTRACTOR_ARGS_JSON: ${YOUTUBE_EXTRACTOR_ARGS_JSON:-} - BILIBILI_PUBLISH_MODE: ${BILIBILI_PUBLISH_MODE:-mock} + BILIBILI_PUBLISH_MODE: ${BILIBILI_PUBLISH_MODE:-web} + CELERY_PUB_CONCURRENCY: ${CELERY_PUB_CONCURRENCY:-1} CORS_ALLOW_ORIGINS: ${CORS_ALLOW_ORIGINS:-http://localhost:3000,http://127.0.0.1:3000} SUBTITLE_SERVICE_URL: ${SUBTITLE_SERVICE_URL:-http://localhost:8000/subtitle-service} YOUTUBE_INGEST_URL: ${YOUTUBE_INGEST_URL:-http://localhost:8000/youtube-ingest} BILIBILI_PUBLISHER_URL: ${BILIBILI_PUBLISHER_URL:-http://localhost:8000/bilibili-publisher} - ports: - - "8000:8000" + STORAGE_CLEANUP_INTERVAL_SECONDS: ${STORAGE_CLEANUP_INTERVAL_SECONDS:-3600} volumes: - ./data/models:/models - ./data/secrets:/secrets + extra_hosts: + - host.docker.internal:host-gateway depends_on: - postgres: - condition: service_healthy - db-init: - condition: service_completed_successfully redis: condition: service_started minio: condition: service_started minio-init: condition: service_completed_successfully + restart: unless-stopped -volumes: - postgres-data: - minio-data: + web: + build: + context: ./src/web + args: + VITE_ORCHESTRATOR_URL: ${VITE_ORCHESTRATOR_URL:-} + VITE_SUBTITLE_SERVICE_URL: ${VITE_SUBTITLE_SERVICE_URL:-} + VITE_YOUTUBE_INGEST_URL: ${VITE_YOUTUBE_INGEST_URL:-} + VITE_BILIBILI_PUBLISHER_URL: ${VITE_BILIBILI_PUBLISHER_URL:-} + image: videoroll-web:prod + ports: + - "${PUBLISH_ADDR:-127.0.0.1}:${WEB_PORT:-3000}:80" + depends_on: + app: + condition: service_started + restart: unless-stopped diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 79324cb..cd8e37c 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -4,6 +4,7 @@ set -euo pipefail UVICORN_APP="${UVICORN_APP:-videoroll.apps.monolith.main:app}" UVICORN_HOST="${UVICORN_HOST:-0.0.0.0}" UVICORN_PORT="${UVICORN_PORT:-8000}" +UVICORN_ROOT_PATH="${UVICORN_ROOT_PATH:-}" CELERY_SUB_APP="${CELERY_SUB_APP:-${CELERY_APP:-videoroll.apps.subtitle_service.worker:celery_app}}" CELERY_SUB_QUEUE="${CELERY_SUB_QUEUE:-${CELERY_QUEUE:-subtitle}}" @@ -19,8 +20,12 @@ echo "Starting celery worker (publish): $CELERY_PUB_APP queue=$CELERY_PUB_QUEUE celery -A "$CELERY_PUB_APP" worker -Q "$CELERY_PUB_QUEUE" -l INFO --concurrency "$CELERY_PUB_CONCURRENCY" & CELERY_PUB_PID=$! -echo "Starting uvicorn: $UVICORN_APP on $UVICORN_HOST:$UVICORN_PORT" -uvicorn "$UVICORN_APP" --host "$UVICORN_HOST" --port "$UVICORN_PORT" & +echo "Starting uvicorn: $UVICORN_APP on $UVICORN_HOST:$UVICORN_PORT (root_path=${UVICORN_ROOT_PATH:-/})" +uvicorn_cmd=(uvicorn "$UVICORN_APP" --host "$UVICORN_HOST" --port "$UVICORN_PORT") +if [[ -n "$UVICORN_ROOT_PATH" ]]; then + uvicorn_cmd+=(--root-path "$UVICORN_ROOT_PATH") +fi +"${uvicorn_cmd[@]}" & UVICORN_PID=$! term_handler() { diff --git a/docs/PROJECT_SPEC.md b/docs/PROJECT_SPEC.md index 596e409..bc984b2 100644 --- a/docs/PROJECT_SPEC.md +++ b/docs/PROJECT_SPEC.md @@ -48,16 +48,16 @@ 建议按 `tenant/account/task_id` 分层，便于多账号与隔离： -- `raw/{task_id}/video.mp4`（原视频） -- `raw/{task_id}/metadata.json` -- `work/{task_id}/audio.wav` -- `sub/{task_id}/segments.json`（ASR 分段结构化结果） -- `sub/{task_id}/subtitle_src.srt` -- `sub/{task_id}/subtitle_zh.srt` -- `sub/{task_id}/subtitle_zh.ass` -- `final/{task_id}/video_burnin.mp4` -- `final/{task_id}/video_softsub.mkv` -- `final/{task_id}/cover.jpg` +- `raw/{task_id}/video_{sha256[:16]}.mp4`（原视频，建议内容寻址避免覆盖） +- `raw/{task_id}/metadata_{sha256[:16]}.json` +- `work/{task_id}/audio_{sha256[:16]}.wav` +- `sub/{task_id}/segments_{sha256[:16]}.json`（ASR 分段结构化结果） +- `sub/{task_id}/subtitle_src_{sha256[:16]}.srt` +- `sub/{task_id}/subtitle_zh_{sha256[:16]}.srt` +- `sub/{task_id}/subtitle_zh_{sha256[:16]}.ass` +- `final/{task_id}/video_burnin_{sha256[:16]}.mp4` +- `final/{task_id}/video_softsub_{sha256[:16]}.mkv` +- `final/{task_id}/cover_{sha256[:16]}.jpg` - `meta/{task_id}/publish_meta.json` - `meta/{task_id}/publish_result.json` - `logs/{task_id}/stage_{name}.log` @@ -353,4 +353,3 @@ 3) 翻译：是否需要、是否需要术语表、是否需要双语 4) 字幕输出：只字幕 / burn-in / soft-sub / 全要 5) 发布策略：全自动还是必须人工审核后才能投稿 - diff --git a/scripts/dev_down.sh b/scripts/dev_down.sh index acdbdba..1fe3aaf 100644 --- a/scripts/dev_down.sh +++ b/scripts/dev_down.sh @@ -5,5 +5,4 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" ENV_FILE="${ENV_FILE:-.env}" -docker compose --env-file "$ENV_FILE" down - +docker compose -f docker-compose.yml --env-file "$ENV_FILE" down diff --git a/scripts/dev_health.sh b/scripts/dev_health.sh index 50966e7..43e2a2c 100644 --- a/scripts/dev_health.sh +++ b/scripts/dev_health.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash set -euo pipefail -curl -fsS http://localhost:8000/health >/dev/null && echo "orchestrator-api ok" -curl -fsS http://localhost:8000/subtitle-service/health >/dev/null && echo "subtitle-service ok" -curl -fsS http://localhost:8000/youtube-ingest/health >/dev/null && echo "youtube-ingest ok" -curl -fsS http://localhost:8000/bilibili-publisher/health >/dev/null && echo "bilibili-publisher ok" +BASE="${BASE:-http://localhost:3000/api}" + +curl -fsS "$BASE/health" >/dev/null && echo "orchestrator-api ok" +curl -fsS "$BASE/subtitle-service/health" >/dev/null && echo "subtitle-service ok" +curl -fsS "$BASE/youtube-ingest/health" >/dev/null && echo "youtube-ingest ok" +curl -fsS "$BASE/bilibili-publisher/health" >/dev/null && echo "bilibili-publisher ok" if curl -fsS http://localhost:3000 >/dev/null; then echo "web ok" diff --git a/scripts/dev_logs.sh b/scripts/dev_logs.sh index c93638e..8c2ed94 100644 --- a/scripts/dev_logs.sh +++ b/scripts/dev_logs.sh @@ -5,5 +5,5 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" ENV_FILE="${ENV_FILE:-.env}" -docker compose --env-file "$ENV_FILE" logs -f --tail=200 \ - app web postgres redis minio +docker compose -f docker-compose.yml --env-file "$ENV_FILE" logs -f --tail=200 \ + app web redis minio diff --git a/scripts/dev_up.sh b/scripts/dev_up.sh index b5c3b88..6b2594e 100644 --- a/scripts/dev_up.sh +++ b/scripts/dev_up.sh @@ -10,14 +10,14 @@ if [[ ! -f "$ENV_FILE" ]]; then echo "Created $ENV_FILE from .env.example" fi -docker compose --env-file "$ENV_FILE" up --build -d +docker compose -f docker-compose.yml --env-file "$ENV_FILE" up --build -d echo "" echo "Web UI: http://localhost:3000" -echo "API (monolith): http://localhost:8000/docs" -echo "Subtitle Service: http://localhost:8000/subtitle-service/docs" -echo "YouTube Ingest: http://localhost:8000/youtube-ingest/docs" -echo "Bilibili Publisher: http://localhost:8000/bilibili-publisher/docs" +echo "API (monolith): http://localhost:3000/api/docs" +echo "Subtitle Service: http://localhost:3000/api/subtitle-service/docs" +echo "YouTube Ingest: http://localhost:3000/api/youtube-ingest/docs" +echo "Bilibili Publisher: http://localhost:3000/api/bilibili-publisher/docs" echo "MinIO Console: http://localhost:9001 (user/pass from .env)" echo "" -docker compose --env-file "$ENV_FILE" ps +docker compose -f docker-compose.yml --env-file "$ENV_FILE" ps diff --git a/scripts/dev_web.sh b/scripts/dev_web.sh index 8a81875..b42b570 100644 --- a/scripts/dev_web.sh +++ b/scripts/dev_web.sh @@ -2,11 +2,10 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT_DIR/web" +cd "$ROOT_DIR/src/web" if [[ ! -d node_modules ]]; then npm install --no-fund --no-audit fi npm run dev -- --host 0.0.0.0 --port 3000 - diff --git a/scripts/smoke_local.sh b/scripts/smoke_local.sh index 548d83a..d76647c 100644 --- a/scripts/smoke_local.sh +++ b/scripts/smoke_local.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -ORCH="${ORCH:-http://localhost:8000}" +ORCH="${ORCH:-http://localhost:3000/api}" SUB="${SUB:-$ORCH/subtitle-service}" VIDEO_PATH="${1:-}" diff --git a/src/videoroll/apps/bilibili_publisher/main.py b/src/videoroll/apps/bilibili_publisher/main.py index 720fc56..5c42875 100644 --- a/src/videoroll/apps/bilibili_publisher/main.py +++ b/src/videoroll/apps/bilibili_publisher/main.py @@ -13,6 +13,7 @@ from sqlalchemy.orm import Session from videoroll.config import BilibiliPublisherSettings, get_bilibili_publisher_settings, get_subtitle_settings from videoroll.db.base import Base +from videoroll.db.auto_migrate import auto_migrate from videoroll.db.models import Asset, AssetKind, PublishJob, PublishState, Task, TaskStatus from videoroll.db.session import db_session, get_engine from videoroll.storage.s3 import S3Store @@ -66,6 +67,7 @@ def _startup() -> None: settings = get_bilibili_publisher_settings() engine = get_engine(settings.database_url) Base.metadata.create_all(engine) + auto_migrate(settings.database_url) S3Store(settings).ensure_bucket() diff --git a/src/videoroll/apps/bilibili_publisher/worker.py b/src/videoroll/apps/bilibili_publisher/worker.py index 6ff3184..6651b97 100644 --- a/src/videoroll/apps/bilibili_publisher/worker.py +++ b/src/videoroll/apps/bilibili_publisher/worker.py @@ -21,6 +21,7 @@ from videoroll.apps.bilibili_publisher.schemas import BilibiliPublishMeta from videoroll.apps.bilibili_publisher.typeid_recommender import flatten_typelist, recommend_typeid_openai from videoroll.config import get_bilibili_publisher_settings, get_subtitle_settings from videoroll.db.base import Base +from videoroll.db.auto_migrate import auto_migrate from videoroll.db.models import Asset, AssetKind, PublishJob, PublishState, Task, TaskStatus from videoroll.db.session import get_engine, get_sessionmaker from videoroll.storage.s3 import S3Store @@ -48,6 +49,7 @@ def _db() -> Session: def _ensure_db() -> None: engine = get_engine(settings.database_url) Base.metadata.create_all(engine) + auto_migrate(settings.database_url) def _utcnow() -> datetime: diff --git a/src/videoroll/apps/orchestrator_api/admin_auth_store.py b/src/videoroll/apps/orchestrator_api/admin_auth_store.py new file mode 100644 index 0000000..a8a7bb4 --- /dev/null +++ b/src/videoroll/apps/orchestrator_api/admin_auth_store.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import base64 +import hashlib +import hmac +import secrets +import time +from dataclasses import dataclass +from typing import Any + +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from videoroll.db.models import AppSetting + + +ADMIN_AUTH_SETTINGS_KEY = "admin.auth" + +DEVICE_COOKIE_NAME = "videoroll_admin_device" +INTERNAL_TOKEN_HEADER = "X-Videoroll-Internal-Token" + +_PBKDF2_ITERATIONS = 200_000 +_PBKDF2_SALT_BYTES = 16 + +# Device cookies are "trusted device" sessions; user can always re-login with the shared admin password. +_DEVICE_COOKIE_MAX_AGE_SECONDS = 3600 * 24 * 180 # 180 days + + +@dataclass(frozen=True) +class AdminAuthStatus: + password_set: bool + trusted: bool + + +def _as_dict(v: Any) -> dict[str, Any]: + return v if isinstance(v, dict) else {} + + +def _get_row(db: Session) -> AppSetting: + row = db.get(AppSetting, ADMIN_AUTH_SETTINGS_KEY) + if row: + return row + row = AppSetting(key=ADMIN_AUTH_SETTINGS_KEY, value_json={}) + db.add(row) + try: + db.commit() + db.refresh(row) + except IntegrityError: + # Concurrency: another request created the row first. + db.rollback() + row2 = db.get(AppSetting, ADMIN_AUTH_SETTINGS_KEY) + if row2: + return row2 + raise + return row + + +def _b64url_encode(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("=") + + +def _b64url_decode(text: str) -> bytes: + t = str(text or "").strip() + if not t: + return b"" + pad = "=" * ((4 - (len(t) % 4)) % 4) + return base64.urlsafe_b64decode((t + pad).encode("utf-8")) + + +def _hash_password_pbkdf2(password: str, *, salt: bytes, iterations: int) -> bytes: + return hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) + + +def encode_password_hash(password: str) -> str: + """ + Returns: pbkdf2_sha256$$$ + """ + pw = str(password or "") + if not pw: + raise ValueError("password is required") + salt = secrets.token_bytes(_PBKDF2_SALT_BYTES) + digest = _hash_password_pbkdf2(pw, salt=salt, iterations=_PBKDF2_ITERATIONS) + return f"pbkdf2_sha256${_PBKDF2_ITERATIONS}${_b64url_encode(salt)}${_b64url_encode(digest)}" + + +def verify_password_hash(password: str, encoded: str) -> bool: + pw = str(password or "") + encoded = str(encoded or "").strip() + if not pw or not encoded: + return False + parts = encoded.split("$") + if len(parts) != 4: + return False + algo, iter_s, salt_s, digest_s = parts + if algo != "pbkdf2_sha256": + return False + try: + iterations = int(iter_s) + except Exception: + return False + salt = _b64url_decode(salt_s) + digest = _b64url_decode(digest_s) + if not salt or not digest or iterations <= 0: + return False + actual = _hash_password_pbkdf2(pw, salt=salt, iterations=iterations) + return hmac.compare_digest(actual, digest) + + +def get_password_hash(db: Session) -> str: + row = db.get(AppSetting, ADMIN_AUTH_SETTINGS_KEY) + stored = dict(_as_dict(row.value_json)) if row else {} + val = str(stored.get("password_hash") or "").strip() + return val + + +def set_password_hash(db: Session, password_hash: str) -> None: + row = _get_row(db) + stored = dict(_as_dict(row.value_json)) + stored["password_hash"] = str(password_hash or "").strip() + row.value_json = stored + db.add(row) + db.commit() + + +def is_password_set(db: Session) -> bool: + return bool(get_password_hash(db)) + + +def device_cookie_max_age_seconds() -> int: + return int(_DEVICE_COOKIE_MAX_AGE_SECONDS) + + +def _cookie_signing_key(*, internal_secret: str, password_hash: str) -> bytes: + """ + Derive a stable signing key. Including password_hash allows invalidating + existing device cookies after an admin password change. + """ + internal_secret = str(internal_secret or "").strip() + password_hash = str(password_hash or "").strip() + h = hashlib.sha256() + h.update(("videoroll-admin-cookie-key:v1:" + internal_secret + ":" + password_hash).encode("utf-8")) + return h.digest() + + +def mint_device_cookie_value(*, internal_secret: str, password_hash: str, now: float | None = None) -> str: + """ + Cookie format: .. + """ + now_ts = float(time.time() if now is None else now) + exp = int(now_ts) + device_cookie_max_age_seconds() + token = _b64url_encode(secrets.token_bytes(32)) + msg = f"{token}.{exp}" + key = _cookie_signing_key(internal_secret=internal_secret, password_hash=password_hash) + sig = hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() + return f"{msg}.{_b64url_encode(sig)}" + + +def verify_device_cookie_value(value: str, *, internal_secret: str, password_hash: str, now: float | None = None) -> bool: + raw = str(value or "").strip() + if not raw: + return False + parts = raw.split(".") + if len(parts) != 3: + return False + token, exp_s, sig_s = parts + if not token or not exp_s or not sig_s: + return False + try: + exp = int(exp_s) + except Exception: + return False + now_ts = float(time.time() if now is None else now) + if int(now_ts) >= exp: + return False + msg = f"{token}.{exp}" + key = _cookie_signing_key(internal_secret=internal_secret, password_hash=password_hash) + expected_sig = hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() + got_sig = _b64url_decode(sig_s) + if not got_sig: + return False + return hmac.compare_digest(expected_sig, got_sig) + + +def validate_new_password(password: str) -> str: + pw = str(password or "") + if len(pw) < 8: + raise ValueError("password too short (min 8 chars)") + if len(pw) > 128: + raise ValueError("password too long (max 128 chars)") + return pw diff --git a/src/videoroll/apps/orchestrator_api/main.py b/src/videoroll/apps/orchestrator_api/main.py index 6cc4151..f4b62b1 100644 --- a/src/videoroll/apps/orchestrator_api/main.py +++ b/src/videoroll/apps/orchestrator_api/main.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import hmac import json import os import tempfile @@ -13,19 +14,27 @@ from typing import Any, Generator, Optional from urllib.parse import quote, urlparse import httpx +from botocore.exceptions import ClientError from fastapi import Depends, FastAPI, File, HTTPException, Query, Request, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, Response, StreamingResponse +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.types import ASGIApp from sqlalchemy import func from sqlalchemy.orm import Session from videoroll.config import OrchestratorSettings, get_orchestrator_settings from videoroll.db.base import Base -from videoroll.db.models import Asset, AssetKind, PublishJob, Subtitle, SubtitleJob, SubtitleJobStatus, Task, TaskStatus +from videoroll.db.auto_migrate import auto_migrate +from videoroll.db.models import AppSetting, Asset, AssetKind, PublishJob, Subtitle, SubtitleJob, SubtitleJobStatus, Task, TaskStatus from videoroll.db.session import db_session, get_engine, get_sessionmaker from videoroll.storage.s3 import S3Store +from videoroll.utils.internal_api_token import internal_api_token from videoroll.utils.hashing import sha256_file from videoroll.apps.orchestrator_api.schemas import ( + AdminAuthLoginRequest, + AdminAuthSetupRequest, + AdminAuthStatusRead, AssetRead, AutoYouTubeRequest, AutoYouTubeResponse, @@ -48,6 +57,18 @@ from videoroll.apps.orchestrator_api.schemas import ( YouTubeMetaActionResponse, YouTubeMetaRead, ) +from videoroll.apps.orchestrator_api.admin_auth_store import ( + DEVICE_COOKIE_NAME, + INTERNAL_TOKEN_HEADER, + device_cookie_max_age_seconds, + encode_password_hash, + get_password_hash, + set_password_hash, + mint_device_cookie_value, + validate_new_password, + verify_device_cookie_value, + verify_password_hash, +) from videoroll.apps.orchestrator_api.youtube_downloader import ( download_youtube_video, download_thumbnail_jpg, @@ -59,8 +80,14 @@ from videoroll.apps.orchestrator_api.storage_retention_store import ( update_storage_retention_settings, ) from videoroll.apps.subtitle_service.bilibili_tags_store import get_task_bilibili_tags -from videoroll.apps.subtitle_service.task_title_store import get_task_display_title -from videoroll.apps.youtube_settings_store import get_youtube_settings, update_youtube_settings +from videoroll.apps.subtitle_service.task_title_store import get_task_display_title_with_s3 +from videoroll.apps.youtube_settings_store import ( + get_youtube_cookies_txt, + get_youtube_settings, + normalize_and_validate_netscape_cookies_txt, + summarize_netscape_cookies_txt, + update_youtube_settings, +) def get_settings() -> OrchestratorSettings: @@ -152,10 +179,10 @@ def _content_disposition(filename: str, *, inline: bool) -> str: return f"{disp}; filename=\"{fallback}\"; filename*=UTF-8''{encoded}" -def _suggest_asset_filename(db: Session, task_id: uuid.UUID, asset: Asset) -> str: +def _suggest_asset_filename(db: Session, task_id: uuid.UUID, asset: Asset, *, s3: S3Store | None) -> str: base = Path(asset.storage_key).name or "download.bin" if asset.kind == AssetKind.video_final: - title = get_task_display_title(db, str(task_id)).strip() + title = get_task_display_title_with_s3(db, str(task_id), s3=s3).strip() if title: ext = Path(base).suffix if ext and len(ext) <= 8: @@ -241,12 +268,102 @@ def _read_s3_bytes(s3: S3Store, key: str) -> bytes: pass +def _publish_meta_s3_key(task_id: uuid.UUID) -> str: + return f"meta/{task_id}/publish_meta.json" + + +def _read_s3_json_object(s3: S3Store, key: str) -> dict[str, Any] | None: + try: + raw = _read_s3_bytes(s3, key) + except ClientError as e: + code = str((_as_dict(e.response.get("Error")).get("Code") or "")).strip() + if code in {"NoSuchKey", "404", "NotFound"}: + return None + raise + if not raw: + return None + try: + parsed = json.loads(raw.decode("utf-8")) + except Exception: + return None + return parsed if isinstance(parsed, dict) else None + + +def _is_s3_object_missing(e: ClientError) -> bool: + code = str((_as_dict(e.response.get("Error")).get("Code") or "")).strip() + return code in {"NoSuchKey", "404", "NotFound"} + + +def _write_s3_json(s3: S3Store, key: str, obj: dict[str, Any]) -> bytes: + payload = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") + s3.put_bytes(payload, key, content_type="application/json") + return payload + + +def _internal_header_token(settings: OrchestratorSettings) -> str: + return internal_api_token(settings.s3_secret_access_key) + + +def _admin_cookie_secret(settings: OrchestratorSettings) -> str: + # Separate derivation so leaking the internal header token does not automatically + # reveal the cookie signing secret. + return internal_api_token(settings.s3_secret_access_key + ":admin-cookie") + + +def _internal_http_headers(settings: OrchestratorSettings) -> dict[str, str]: + tok = _internal_header_token(settings) + return {INTERNAL_TOKEN_HEADER: tok} if tok else {} + + +class _AdminAuthMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + if request.method == "OPTIONS": + return await call_next(request) + + # Normalize path for allowlist checks. + # + # When running behind a reverse proxy or with `uvicorn --root-path`, + # `request.url.path` may include `scope["root_path"]` (e.g. "/api/auth/status"), + # while route matching is done on the "app path" (e.g. "/auth/status"). + scope_path = str(request.scope.get("path") or getattr(request.url, "path", "") or "/") + root_path = str(request.scope.get("root_path") or "").strip() + path = scope_path + if root_path and path.startswith(root_path): + path = path[len(root_path) :] or "/" + if path == "/health" or path.endswith("/health"): + return await call_next(request) + if path.startswith("/auth"): + return await call_next(request) + + pw_hash = _get_admin_password_hash(request) + if not pw_hash: + return JSONResponse(status_code=403, content={"detail": "admin password not set"}) + + internal_header_token = str(getattr(request.app.state, "internal_header_token", "") or "").strip() + header_tok = str(request.headers.get(INTERNAL_TOKEN_HEADER) or "").strip() + if internal_header_token and header_tok and hmac.compare_digest(header_tok, internal_header_token): + return await call_next(request) + + cookie_val = str(request.cookies.get(DEVICE_COOKIE_NAME) or "").strip() + cookie_secret = str(getattr(request.app.state, "admin_cookie_secret", "") or "").strip() + if cookie_val and cookie_secret and verify_device_cookie_value( + cookie_val, + internal_secret=cookie_secret, + password_hash=pw_hash, + ): + return await call_next(request) + + return JSONResponse(status_code=401, content={"detail": "unauthorized"}) + + app = FastAPI(title="videoroll-orchestrator", version="0.1.0") _cleanup_stop = threading.Event() _cleanup_thread: Optional[threading.Thread] = None _cleanup_interval_seconds = int(os.getenv("STORAGE_CLEANUP_INTERVAL_SECONDS", "3600") or "3600") +app.add_middleware(_AdminAuthMiddleware) + _cors_origins = [ o.strip() @@ -267,7 +384,17 @@ def _startup() -> None: settings = get_orchestrator_settings() engine = get_engine(settings.database_url) Base.metadata.create_all(engine) + auto_migrate(settings.database_url) S3Store(settings).ensure_bucket() + app.state.database_url = settings.database_url + app.state.internal_header_token = _internal_header_token(settings) + app.state.admin_cookie_secret = _admin_cookie_secret(settings) + SessionLocal = get_sessionmaker(settings.database_url) + db = SessionLocal() + try: + app.state.admin_password_hash = get_password_hash(db) + finally: + db.close() _start_cleanup_thread() @@ -368,10 +495,94 @@ def _start_cleanup_thread() -> None: _cleanup_thread = t -def _effective_youtube_settings(settings: OrchestratorSettings, db: Session) -> OrchestratorSettings: +def _effective_youtube_settings(settings: OrchestratorSettings, db: Session, *, cookie_dir: Path | None = None) -> OrchestratorSettings: cfg = get_youtube_settings(db, default_proxy=settings.youtube_proxy) proxy = str(cfg.get("proxy") or "").strip() - return settings.model_copy(update={"youtube_proxy": proxy or None}) + cookies_enabled = bool(cfg.get("cookies_enabled")) + + cookie_file = str(settings.youtube_cookie_file or "").strip() or None + if cookie_file: + try: + if not Path(cookie_file).is_file() and cookie_dir is not None and cookies_enabled: + # Fall back to the DB-stored cookies when the env-configured cookie file is missing. + cookie_file = None + except Exception: + pass + if not cookie_file and cookie_dir is not None and cookies_enabled: + cookies_txt = get_youtube_cookies_txt(db) + if cookies_txt: + cookies_txt = normalize_and_validate_netscape_cookies_txt(cookies_txt) + try: + cookie_dir.mkdir(parents=True, exist_ok=True) + cookie_path = cookie_dir / "youtube_cookies.txt" + cookie_path.write_text(cookies_txt, encoding="utf-8") + try: + os.chmod(cookie_path, 0o600) + except Exception: + pass + cookie_file = str(cookie_path) + except Exception: + cookie_file = None + + return settings.model_copy(update={"youtube_proxy": proxy or None, "youtube_cookie_file": cookie_file}) + + +def _looks_like_youtube_bot_check_error(message: str) -> bool: + m = str(message or "").lower() + return ( + "not a bot" in m + or "unusual traffic" in m + or "/sorry/" in m + or "confirm you" in m and "not a bot" in m + ) + + +def _youtube_bot_check_hint(message: str, *, yt_settings: OrchestratorSettings, db: Session) -> str | None: + if not _looks_like_youtube_bot_check_error(message): + return None + + proxy = str(yt_settings.youtube_proxy or "").strip() + cookie_file = str(yt_settings.youtube_cookie_file or "").strip() + + summary: dict[str, Any] | None = None + if cookie_file: + try: + p = Path(cookie_file) + if p.is_file(): + summary = summarize_netscape_cookies_txt(p.read_text(encoding="utf-8", errors="ignore")) + except Exception: + summary = None + + if summary is None: + try: + summary = summarize_netscape_cookies_txt(get_youtube_cookies_txt(db)) + except Exception: + summary = summarize_netscape_cookies_txt("") + + lines: list[str] = [] + lines.append("提示：YouTube 触发了“确认你不是机器人/异常流量”风控页面（与出口 IP/代理或 cookies 有关）。") + if proxy: + lines.append(f"当前代理：{proxy}") + lines.append("请用同一个代理/IP 在浏览器打开 YouTube，按提示完成登录/验证码后再导出 cookies.txt 更新到系统。") + else: + lines.append("如果当前出口 IP 被风控，建议更换网络或配置可用代理后再导出 cookies.txt。") + + if not bool(summary.get("cookies_has_auth")): + lines.append("你保存的 cookies 看起来不包含登录态（缺少 SID/SAPISID 等）；仅 VISITOR_INFO1_LIVE 这类 cookie 通常不够。") + if not bool(summary.get("cookies_has_bot_check_bypass")): + lines.append("若浏览器出现过“确认你不是机器人”页面，需要在同一出口 IP 下通过验证码后再导出（通常会生成 GOOGLE_ABUSE_EXEMPTION）。") + lines.append("建议用无痕/新会话导出并尽快更新，YouTube 会频繁轮换账号 cookies，旧 cookies 可能很快失效。") + return "\n".join(lines) + + +def _youtube_cookie_file_status(settings: OrchestratorSettings) -> tuple[bool, bool]: + cookie_file = str(settings.youtube_cookie_file or "").strip() + if not cookie_file: + return False, False + try: + return True, Path(cookie_file).is_file() + except Exception: + return True, False @app.get("/health") @@ -379,6 +590,110 @@ def health() -> dict[str, str]: return {"status": "ok"} +def _secure_cookie(request: Request) -> bool: + proto = str(request.headers.get("x-forwarded-proto") or request.url.scheme or "").lower() + return proto == "https" + + +def _set_device_cookie(resp: Response, value: str, *, secure: bool) -> None: + resp.set_cookie( + key=DEVICE_COOKIE_NAME, + value=value, + max_age=device_cookie_max_age_seconds(), + httponly=True, + samesite="lax", + secure=bool(secure), + path="/", + ) + + +def _get_admin_password_hash(request: Request, db: Session | None = None) -> str: + cached = str(getattr(request.app.state, "admin_password_hash", "") or "").strip() + if cached: + return cached + + pw_hash = "" + if db is not None: + try: + pw_hash = str(get_password_hash(db) or "").strip() + except Exception: + pw_hash = "" + else: + database_url = str(getattr(request.app.state, "database_url", "") or "").strip() + if database_url: + SessionLocal = get_sessionmaker(database_url) + db2 = SessionLocal() + try: + pw_hash = str(get_password_hash(db2) or "").strip() + finally: + db2.close() + + if pw_hash: + request.app.state.admin_password_hash = pw_hash + return pw_hash + + +@app.get("/auth/status", response_model=AdminAuthStatusRead) +def auth_status(request: Request, db: Session = Depends(get_db)) -> AdminAuthStatusRead: + pw_hash = _get_admin_password_hash(request, db) + password_set = bool(pw_hash) + + trusted = False + if password_set: + cookie_secret = str(getattr(request.app.state, "admin_cookie_secret", "") or "").strip() + cookie_val = str(request.cookies.get(DEVICE_COOKIE_NAME) or "").strip() + if cookie_secret and cookie_val: + trusted = verify_device_cookie_value(cookie_val, internal_secret=cookie_secret, password_hash=pw_hash) + + return AdminAuthStatusRead(password_set=password_set, trusted=trusted) + + +@app.post("/auth/setup", response_model=AdminAuthStatusRead) +def auth_setup(payload: AdminAuthSetupRequest, request: Request, db: Session = Depends(get_db)) -> JSONResponse: + existing = _get_admin_password_hash(request, db) + if existing: + raise HTTPException(status_code=400, detail="admin password already set") + + pw = validate_new_password(payload.password) + encoded = encode_password_hash(pw) + set_password_hash(db, encoded) + request.app.state.admin_password_hash = encoded + + cookie_secret = str(getattr(request.app.state, "admin_cookie_secret", "") or "").strip() + cookie_val = mint_device_cookie_value(internal_secret=cookie_secret, password_hash=encoded) + body = AdminAuthStatusRead(password_set=True, trusted=True).model_dump(mode="json") + resp = JSONResponse(status_code=200, content=body) + _set_device_cookie(resp, cookie_val, secure=_secure_cookie(request)) + return resp + + +@app.post("/auth/login", response_model=AdminAuthStatusRead) +def auth_login(payload: AdminAuthLoginRequest, request: Request, db: Session = Depends(get_db)) -> JSONResponse: + pw_hash = _get_admin_password_hash(request, db) + if not pw_hash: + raise HTTPException(status_code=400, detail="admin password is not set") + + if not verify_password_hash(str(payload.password or ""), pw_hash): + raise HTTPException(status_code=401, detail="invalid password") + + cookie_secret = str(getattr(request.app.state, "admin_cookie_secret", "") or "").strip() + cookie_val = mint_device_cookie_value(internal_secret=cookie_secret, password_hash=pw_hash) + body = AdminAuthStatusRead(password_set=True, trusted=True).model_dump(mode="json") + resp = JSONResponse(status_code=200, content=body) + _set_device_cookie(resp, cookie_val, secure=_secure_cookie(request)) + return resp + + +@app.post("/auth/logout", response_model=AdminAuthStatusRead) +def auth_logout(request: Request, db: Session = Depends(get_db)) -> JSONResponse: + pw_hash = _get_admin_password_hash(request, db) + password_set = bool(pw_hash) + body = AdminAuthStatusRead(password_set=password_set, trusted=False).model_dump(mode="json") + resp = JSONResponse(status_code=200, content=body) + resp.delete_cookie(key=DEVICE_COOKIE_NAME, path="/") + return resp + + @app.post("/auto/youtube", response_model=AutoYouTubeResponse) def auto_youtube( payload: AutoYouTubeRequest, @@ -391,7 +706,7 @@ def auto_youtube( raise HTTPException(status_code=400, detail="url is not a valid youtube url") try: - with httpx.Client(timeout=30.0) as client: + with httpx.Client(timeout=30.0, headers=_internal_http_headers(settings)) as client: resp = client.post( f"{settings.youtube_ingest_url}/youtube/ingest", json={"url": url, "license": payload.license.value, "proof_url": payload.proof_url}, @@ -440,6 +755,7 @@ def create_task(payload: TaskCreate, db: Session = Depends(get_db)) -> Task: def list_converted_videos( limit: int = Query(default=50, ge=1, le=200), db: Session = Depends(get_db), + s3: S3Store = Depends(get_s3), ) -> list[dict[str, Any]]: # Fetch recent video_final assets and dedupe by task_id (keep latest per task). fetch_n = min(1000, max(limit, 1) * 5) @@ -466,7 +782,7 @@ def list_converted_videos( .order_by(Asset.created_at.desc()) .first() ) - display_title = get_task_display_title(db, str(a.task_id)) + display_title = get_task_display_title_with_s3(db, str(a.task_id), s3=s3) items.append({"task": task, "final_asset": a, "cover_asset": cover_asset, "display_title": display_title or None}) if len(items) >= limit: break @@ -478,11 +794,65 @@ def list_tasks( status: Optional[TaskStatus] = Query(default=None), limit: int = Query(default=50, ge=1, le=500), db: Session = Depends(get_db), -) -> list[Task]: + s3: S3Store = Depends(get_s3), +) -> list[dict[str, Any]]: + def _title_key(tid: uuid.UUID) -> str: + return f"task.title.{tid}" + q = db.query(Task).order_by(Task.created_at.desc()) if status is not None: q = q.filter(Task.status == status) - return q.limit(limit).all() + tasks = q.limit(limit).all() + + ids = [t.id for t in tasks] + + # Batch load translated/source titles from AppSetting first, then fall back to YouTube meta title. + title_map: dict[uuid.UUID, str] = {} + if ids: + keys = [_title_key(tid) for tid in ids] + rows = db.query(AppSetting).filter(AppSetting.key.in_(keys)).all() + by_key: dict[str, dict[str, Any]] = {} + for r in rows: + by_key[str(r.key)] = _as_dict(getattr(r, "value_json", None)) + + for tid in ids: + data = by_key.get(_title_key(tid)) or {} + for k in ("translated_title", "source_title"): + v = data.get(k) + if isinstance(v, str) and v.strip(): + title_map[tid] = v.strip() + break + + missing = [tid for tid in ids if tid not in title_map] + if missing: + assets = ( + db.query(Asset) + .filter(Asset.task_id.in_(missing), Asset.kind == AssetKind.metadata_json) + .order_by(Asset.created_at.desc()) + .all() + ) + picked: dict[uuid.UUID, Asset] = {} + for a in assets: + if a.task_id not in picked: + picked[a.task_id] = a + for tid, asset in picked.items(): + try: + raw = _read_s3_bytes(s3, asset.storage_key) + parsed = json.loads(raw.decode("utf-8")) if raw else {} + info = parsed if isinstance(parsed, dict) else {} + except Exception: + continue + title = str(info.get("title") or info.get("fulltitle") or info.get("alt_title") or "").strip() + if title: + title_map[tid] = title + + out: list[dict[str, Any]] = [] + for t in tasks: + item = TaskRead.model_validate(t).model_dump() + display_title = str(title_map.get(t.id) or "").strip() + item["display_title"] = display_title or None + out.append(item) + return out @app.get("/settings/storage", response_model=StorageRetentionSettingsRead) @@ -503,7 +873,8 @@ def put_storage_settings(payload: StorageRetentionSettingsUpdate, db: Session = @app.get("/settings/youtube", response_model=YouTubeSettingsRead) def get_youtube_settings_view(settings: OrchestratorSettings = Depends(get_settings), db: Session = Depends(get_db)) -> YouTubeSettingsRead: cfg = get_youtube_settings(db, default_proxy=settings.youtube_proxy) - return YouTubeSettingsRead(**cfg) + cookie_file_configured, cookie_file_exists = _youtube_cookie_file_status(settings) + return YouTubeSettingsRead(**cfg, cookie_file_configured=cookie_file_configured, cookie_file_exists=cookie_file_exists) @app.put("/settings/youtube", response_model=YouTubeSettingsRead) @@ -516,7 +887,8 @@ def put_youtube_settings_view( cfg = update_youtube_settings(db, payload.model_dump(exclude_unset=True), default_proxy=settings.youtube_proxy) except Exception as e: raise HTTPException(status_code=400, detail=str(e)) from e - return YouTubeSettingsRead(**cfg) + cookie_file_configured, cookie_file_exists = _youtube_cookie_file_status(settings) + return YouTubeSettingsRead(**cfg, cookie_file_configured=cookie_file_configured, cookie_file_exists=cookie_file_exists) @app.post("/settings/youtube/test", response_model=YouTubeProxyTestResponse) @@ -581,11 +953,14 @@ def test_youtube_proxy( @app.get("/tasks/{task_id}", response_model=TaskRead) -def get_task(task_id: uuid.UUID, db: Session = Depends(get_db)) -> Task: +def get_task(task_id: uuid.UUID, db: Session = Depends(get_db), s3: S3Store = Depends(get_s3)) -> dict[str, Any]: task = db.get(Task, task_id) if not task: raise HTTPException(status_code=404, detail="task not found") - return task + item = TaskRead.model_validate(task).model_dump() + title = get_task_display_title_with_s3(db, str(task_id), s3=s3).strip() + item["display_title"] = title or None + return item @app.post("/tasks/{task_id}/upload/video", response_model=AssetRead) @@ -600,7 +975,6 @@ async def upload_task_video( raise HTTPException(status_code=404, detail="task not found") suffix = Path(file.filename or "").suffix or ".mp4" - key = f"raw/{task_id}/video{suffix}" with tempfile.NamedTemporaryFile(prefix="videoroll_", suffix=suffix, delete=False) as tmp: tmp_path = Path(tmp.name) @@ -612,6 +986,7 @@ async def upload_task_video( sha256 = sha256_file(tmp_path) size_bytes = tmp_path.stat().st_size + key = f"raw/{task_id}/video_{sha256[:16]}{suffix}" s3.upload_file(tmp_path, key) try: tmp_path.unlink(missing_ok=True) @@ -648,7 +1023,6 @@ async def upload_task_cover( raise HTTPException(status_code=400, detail="cover must be an image") suffix = Path(file.filename or "").suffix or ".jpg" - key = f"final/{task_id}/cover{suffix}" with tempfile.NamedTemporaryFile(prefix="videoroll_cover_", suffix=suffix, delete=False) as tmp: tmp_path = Path(tmp.name) @@ -660,6 +1034,7 @@ async def upload_task_cover( sha256 = sha256_file(tmp_path) size_bytes = tmp_path.stat().st_size + key = f"final/{task_id}/cover_{sha256[:16]}{suffix}" s3.upload_file(tmp_path, key) try: tmp_path.unlink(missing_ok=True) @@ -697,10 +1072,15 @@ def download_task_asset( if not asset or asset.task_id != task_id: raise HTTPException(status_code=404, detail="asset not found") - resp = s3.get_object(asset.storage_key) + try: + resp = s3.get_object(asset.storage_key) + except ClientError as e: + if _is_s3_object_missing(e): + raise HTTPException(status_code=404, detail="asset object not found") from e + raise body = resp["Body"] media_type = resp.get("ContentType") or "application/octet-stream" - filename = _suggest_asset_filename(db, task_id, asset) + filename = _suggest_asset_filename(db, task_id, asset, s3=s3) headers = {"Content-Disposition": _content_disposition(filename, inline=False)} length = resp.get("ContentLength") or asset.size_bytes @@ -722,7 +1102,7 @@ def stream_task_asset( if not asset or asset.task_id != task_id: raise HTTPException(status_code=404, detail="asset not found") - filename = _suggest_asset_filename(db, task_id, asset) + filename = _suggest_asset_filename(db, task_id, asset, s3=s3) total_size: int | None = None media_type = "application/octet-stream" @@ -747,7 +1127,12 @@ def stream_task_asset( return Response(status_code=416, headers={"Content-Range": f"bytes */{total_size}"}) start, end = parsed - resp = s3.get_object(asset.storage_key, range_bytes=f"bytes={start}-{end}") + try: + resp = s3.get_object(asset.storage_key, range_bytes=f"bytes={start}-{end}") + except ClientError as e: + if _is_s3_object_missing(e): + raise HTTPException(status_code=404, detail="asset object not found") from e + raise body = resp["Body"] headers = { **base_headers, @@ -761,7 +1146,12 @@ def stream_task_asset( headers=headers, ) - resp = s3.get_object(asset.storage_key) + try: + resp = s3.get_object(asset.storage_key) + except ClientError as e: + if _is_s3_object_missing(e): + raise HTTPException(status_code=404, detail="asset object not found") from e + raise body = resp["Body"] media_type = resp.get("ContentType") or media_type headers = dict(base_headers) @@ -789,6 +1179,11 @@ def delete_task_asset( db.query(Subtitle).filter(Subtitle.task_id == task_id, Subtitle.storage_key == asset.storage_key).delete( synchronize_session=False ) + remaining_final = ( + db.query(Asset) + .filter(Asset.task_id == task_id, Asset.kind == AssetKind.video_final, Asset.id != asset_id) + .first() + ) db.delete(asset) db.commit() return {"deleted": True} @@ -803,6 +1198,68 @@ def _is_youtube_url(url: str) -> bool: return host.endswith("youtube.com") or host == "youtu.be" or host.endswith(".youtube.com") +@app.get("/tasks/{task_id}/youtube_meta", response_model=YouTubeMetaRead) +def get_cached_youtube_meta(task_id: uuid.UUID, db: Session = Depends(get_db), s3: S3Store = Depends(get_s3)) -> YouTubeMetaRead: + task = db.get(Task, task_id) + if not task: + raise HTTPException(status_code=404, detail="task not found") + if task.source_type.value != "youtube": + raise HTTPException(status_code=400, detail="task is not a youtube source") + + latest = ( + db.query(Asset) + .filter(Asset.task_id == task_id, Asset.kind == AssetKind.metadata_json) + .order_by(Asset.created_at.desc()) + .first() + ) + if not latest: + raise HTTPException(status_code=404, detail="youtube meta not found") + + try: + raw = _read_s3_bytes(s3, latest.storage_key) + parsed = json.loads(raw.decode("utf-8")) if raw else {} + info = parsed if isinstance(parsed, dict) else {} + meta = summarize_info(info, fallback_url=str(task.source_url or "").strip()) + except Exception as e: + raise HTTPException(status_code=502, detail=f"failed to read youtube meta: {e}") from e + + webpage_url = str(meta.webpage_url or "").strip() + if not webpage_url: + raise HTTPException(status_code=400, detail="task.source_url is empty") + + return _youtube_meta_to_read(meta) + + +@app.get("/tasks/{task_id}/publish_meta") +def get_task_publish_meta(task_id: uuid.UUID, db: Session = Depends(get_db), s3: S3Store = Depends(get_s3)) -> dict[str, Any]: + if not db.get(Task, task_id): + raise HTTPException(status_code=404, detail="task not found") + key = _publish_meta_s3_key(task_id) + obj = _read_s3_json_object(s3, key) + if obj is None: + raise HTTPException(status_code=404, detail="publish_meta not found") + return obj + + +@app.put("/tasks/{task_id}/publish_meta") +def put_task_publish_meta( + task_id: uuid.UUID, + meta: dict[str, Any], + db: Session = Depends(get_db), + s3: S3Store = Depends(get_s3), +) -> dict[str, Any]: + if not db.get(Task, task_id): + raise HTTPException(status_code=404, detail="task not found") + if not isinstance(meta, dict): + raise HTTPException(status_code=400, detail="meta must be an object") + key = _publish_meta_s3_key(task_id) + try: + _write_s3_json(s3, key, meta) + except Exception as e: + raise HTTPException(status_code=502, detail=f"failed to write publish_meta: {e}") from e + return {"stored": True, "key": key} + + @app.post("/tasks/{task_id}/actions/youtube_meta", response_model=YouTubeMetaActionResponse) def fetch_youtube_meta( task_id: uuid.UUID, @@ -822,6 +1279,8 @@ def fetch_youtube_meta( if not _is_youtube_url(url): raise HTTPException(status_code=400, detail="task.source_url is not a valid youtube url") + yt_settings: OrchestratorSettings = settings + latest = ( db.query(Asset) .filter(Asset.task_id == task_id, Asset.kind == AssetKind.metadata_json) @@ -839,19 +1298,33 @@ def fetch_youtube_meta( pass try: - yt_settings = _effective_youtube_settings(settings, db) - info, meta = extract_youtube_metadata(url, yt_settings) + work_root = Path(settings.work_dir) / "youtube" / str(task_id) + work_root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="ytmeta_", dir=str(work_root)) as tmp: + try: + yt_settings = _effective_youtube_settings(settings, db, cookie_dir=Path(tmp)) + except ValueError as e: + raise HTTPException(status_code=400, detail=f"invalid youtube cookies: {e}") from e + info, meta = extract_youtube_metadata(url, yt_settings) + except HTTPException: + raise except Exception as e: - raise HTTPException(status_code=502, detail=f"youtube metadata failed: {e}") from e - key = f"raw/{task_id}/metadata.json" + hint = _youtube_bot_check_hint(str(e), yt_settings=yt_settings, db=db) + detail = f"youtube metadata failed: {e}" + if hint: + detail = f"{detail}\n\n{hint}" + raise HTTPException(status_code=502, detail=detail) from e + payload = json.dumps(info, ensure_ascii=False, indent=2).encode("utf-8") + sha = _sha256_bytes(payload) + key = f"raw/{task_id}/metadata_{sha[:16]}.json" s3.put_bytes(payload, key, content_type="application/json") asset = Asset( task_id=task_id, kind=AssetKind.metadata_json, storage_key=key, - sha256=_sha256_bytes(payload), + sha256=sha, size_bytes=len(payload), ) db.add(asset) @@ -886,47 +1359,76 @@ def download_youtube( .first() ) - yt_settings = _effective_youtube_settings(settings, db) - - work_root = Path(yt_settings.work_dir) / "youtube" / str(task_id) + work_root = Path(settings.work_dir) / "youtube" / str(task_id) work_root.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix="ytdlp_", dir=str(work_root)) as tmp: tmp_dir = Path(tmp) + try: + yt_settings = _effective_youtube_settings(settings, db, cookie_dir=tmp_dir) + except ValueError as e: + raise HTTPException(status_code=400, detail=f"invalid youtube cookies: {e}") from e video_asset = existing_video - info: dict[str, Any] - meta: Any + info: dict[str, Any] = {} + meta: Any = None if video_asset is None: try: video_path, info, meta = download_youtube_video(url, yt_settings, work_dir=tmp_dir) except Exception as e: - raise HTTPException(status_code=502, detail=f"youtube download failed: {e}") from e + hint = _youtube_bot_check_hint(str(e), yt_settings=yt_settings, db=db) + detail = f"youtube download failed: {e}" + if hint: + detail = f"{detail}\n\n{hint}" + raise HTTPException(status_code=502, detail=detail) from e suffix = video_path.suffix.lower() or ".mp4" - video_key = f"raw/{task_id}/video{suffix}" + video_sha = sha256_file(video_path) + video_key = f"raw/{task_id}/video_{video_sha[:16]}{suffix}" s3.upload_file(video_path, video_key) video_asset = Asset( task_id=task_id, kind=AssetKind.video_raw, storage_key=video_key, - sha256=sha256_file(video_path), + sha256=video_sha, size_bytes=video_path.stat().st_size, ) db.add(video_asset) else: - try: - info, meta = extract_youtube_metadata(url, yt_settings) - except Exception as e: - raise HTTPException(status_code=502, detail=f"youtube metadata failed: {e}") from e + # Prefer cached metadata to avoid extra YouTube requests. + latest_meta_asset = ( + db.query(Asset) + .filter(Asset.task_id == task_id, Asset.kind == AssetKind.metadata_json) + .order_by(Asset.created_at.desc()) + .first() + ) + + info_cached: dict[str, Any] | None = None + if latest_meta_asset: + try: + raw = _read_s3_bytes(s3, latest_meta_asset.storage_key) + parsed = json.loads(raw.decode("utf-8")) if raw else {} + info_cached = parsed if isinstance(parsed, dict) else None + except Exception: + info_cached = None + + if info_cached is not None: + info = _as_dict(info_cached) + meta = summarize_info(_as_dict(info_cached), fallback_url=url) + else: + try: + info, meta = extract_youtube_metadata(url, yt_settings) + except Exception as e: + raise HTTPException(status_code=502, detail=f"youtube metadata failed: {e}") from e - meta_key = f"raw/{task_id}/metadata.json" meta_payload = json.dumps(info, ensure_ascii=False, indent=2).encode("utf-8") + meta_sha = _sha256_bytes(meta_payload) + meta_key = f"raw/{task_id}/metadata_{meta_sha[:16]}.json" s3.put_bytes(meta_payload, meta_key, content_type="application/json") meta_asset = Asset( task_id=task_id, kind=AssetKind.metadata_json, storage_key=meta_key, - sha256=_sha256_bytes(meta_payload), + sha256=meta_sha, size_bytes=len(meta_payload), ) db.add(meta_asset) @@ -958,6 +1460,14 @@ def download_youtube( except Exception: cover_asset = None + if cover_asset is None: + cover_asset = ( + db.query(Asset) + .filter(Asset.task_id == task_id, Asset.kind == AssetKind.cover_image) + .order_by(Asset.created_at.desc()) + .first() + ) + if task.status in {TaskStatus.created, TaskStatus.ingested}: task.status = TaskStatus.downloaded db.add(task) @@ -1045,6 +1555,27 @@ def enqueue_subtitle_job( if not task: raise HTTPException(status_code=404, detail="task not found") + # Avoid creating duplicate jobs for the same task when a job is already queued/running. + # This commonly happens when the user clicks multiple times and results in a "queued" job + # that won't start until the current one finishes. + in_flight = ( + db.query(SubtitleJob) + .filter( + SubtitleJob.task_id == task_id, + SubtitleJob.status.in_([SubtitleJobStatus.queued, SubtitleJobStatus.running]), + ) + .order_by(SubtitleJob.created_at.desc()) + .first() + ) + if in_flight: + # Best-effort scheduler kick: helps when a job is queued but a tick was dropped. + try: + with httpx.Client(timeout=5.0, headers=_internal_http_headers(settings)) as client: + client.post(f"{settings.subtitle_service_url}/subtitle/task_queue/tick") + except httpx.HTTPError: + pass + return RemoteJobResponse(job_id=in_flight.id, status=in_flight.status.value) + raw_asset = ( db.query(Asset) .filter(Asset.task_id == task_id, Asset.kind == AssetKind.video_raw) @@ -1091,7 +1622,7 @@ def enqueue_subtitle_job( } try: - with httpx.Client(timeout=30.0) as client: + with httpx.Client(timeout=30.0, headers=_internal_http_headers(settings)) as client: resp = client.post(f"{settings.subtitle_service_url}/subtitle/jobs", json=req) resp.raise_for_status() data = resp.json() @@ -1142,7 +1673,7 @@ def resume_subtitle_job( req_out["output_prefix"] = f"sub/{task_id}/" try: - with httpx.Client(timeout=30.0) as client: + with httpx.Client(timeout=30.0, headers=_internal_http_headers(settings)) as client: resp = client.post(f"{settings.subtitle_service_url}/subtitle/jobs", json=req_out) resp.raise_for_status() data = resp.json() @@ -1158,6 +1689,7 @@ def enqueue_publish_job( payload: PublishActionRequest, settings: OrchestratorSettings = Depends(get_settings), db: Session = Depends(get_db), + s3: S3Store = Depends(get_s3), ) -> RemotePublishResponse: task = db.get(Task, task_id) if not task: @@ -1177,7 +1709,13 @@ def enqueue_publish_job( raise HTTPException(status_code=400, detail="no final video asset found; render first") video_key = final_asset.storage_key - meta = dict(payload.meta or {}) + if payload.meta is None: + stored = _read_s3_json_object(s3, _publish_meta_s3_key(task_id)) + if stored is None: + raise HTTPException(status_code=400, detail="meta is missing and publish_meta is not found") + meta = dict(stored) + else: + meta = dict(payload.meta or {}) try: copyright_val = int(meta.get("copyright") or 1) except Exception: @@ -1204,6 +1742,12 @@ def enqueue_publish_job( if merged_tags: meta["tags"] = merged_tags + # Persist the final publish meta so publish is reproducible and editable before/after. + try: + _write_s3_json(s3, _publish_meta_s3_key(task_id), meta) + except Exception as e: + raise HTTPException(status_code=502, detail=f"failed to persist publish_meta: {e}") from e + req = { "task_id": str(task_id), "account_id": payload.account_id, @@ -1216,7 +1760,7 @@ def enqueue_publish_job( req["typeid_mode"] = typeid_mode try: - with httpx.Client(timeout=30.0) as client: + with httpx.Client(timeout=30.0, headers=_internal_http_headers(settings)) as client: resp = client.post(f"{settings.bilibili_publisher_url}/bilibili/publish", json=req) resp.raise_for_status() data = resp.json() diff --git a/src/videoroll/apps/orchestrator_api/schemas.py b/src/videoroll/apps/orchestrator_api/schemas.py index 78f9b5d..370f030 100644 --- a/src/videoroll/apps/orchestrator_api/schemas.py +++ b/src/videoroll/apps/orchestrator_api/schemas.py @@ -27,6 +27,7 @@ class TaskRead(BaseModel): status: TaskStatus priority: int created_by: Optional[str] + display_title: Optional[str] = None error_code: Optional[str] error_message: Optional[str] retry_count: int @@ -50,6 +51,19 @@ class AssetRead(BaseModel): from_attributes = True +class AdminAuthStatusRead(BaseModel): + password_set: bool + trusted: bool + + +class AdminAuthSetupRequest(BaseModel): + password: str + + +class AdminAuthLoginRequest(BaseModel): + password: str + + class SubtitleActionRequest(BaseModel): formats: list[str] = Field(default_factory=lambda: ["srt"]) resume: bool = False @@ -78,7 +92,7 @@ class PublishActionRequest(BaseModel): video_key: Optional[str] = None cover_key: Optional[str] = None typeid_mode: Optional[str] = None - meta: dict[str, Any] + meta: Optional[dict[str, Any]] = None class RemoteJobResponse(BaseModel): @@ -136,7 +150,7 @@ class YouTubeMetaRead(BaseModel): class YouTubeMetaActionResponse(BaseModel): metadata: YouTubeMetaRead - metadata_asset: AssetRead + metadata_asset: Optional[AssetRead] = None class YouTubeDownloadActionResponse(BaseModel): @@ -176,10 +190,22 @@ class StorageRetentionSettingsUpdate(BaseModel): class YouTubeSettingsRead(BaseModel): proxy: str = "" + cookies_set: bool = False + cookies_enabled: bool = False + cookies_updated_at: Optional[str] = None + cookies_count: int = 0 + cookies_domains_count: int = 0 + cookies_has_auth: bool = False + cookies_has_bot_check_bypass: bool = False + cookies_has_visitor_info: bool = False + cookie_file_configured: bool = False + cookie_file_exists: bool = False class YouTubeSettingsUpdate(BaseModel): proxy: Optional[str] = None + cookies_txt: Optional[str] = None + cookies_enabled: Optional[bool] = None class YouTubeProxyTestRequest(BaseModel): diff --git a/src/videoroll/apps/orchestrator_api/youtube_downloader.py b/src/videoroll/apps/orchestrator_api/youtube_downloader.py index cd343c0..4d96309 100644 --- a/src/videoroll/apps/orchestrator_api/youtube_downloader.py +++ b/src/videoroll/apps/orchestrator_api/youtube_downloader.py @@ -104,9 +104,6 @@ def build_ydl_opts(settings: OrchestratorSettings, *, outtmpl: str | None = None if outtmpl: opts["outtmpl"] = outtmpl - if for_download: - opts["format"] = (settings.youtube_ytdlp_format or "").strip() or "best" - return opts @@ -251,28 +248,7 @@ def extract_youtube_metadata(url: str, settings: OrchestratorSettings) -> tuple[ return _as_dict(sanitized), meta -def download_youtube_video(url: str, settings: OrchestratorSettings, *, work_dir: Path) -> tuple[Path, dict[str, Any], YouTubeMeta]: - """ - Downloads the best quality video to work_dir and returns: - - output video file path - - sanitized info dict - - meta summary - """ - work_dir.mkdir(parents=True, exist_ok=True) - outtmpl = str(work_dir / "%(id)s.%(ext)s") - try: - with yt_dlp.YoutubeDL(build_ydl_opts(settings, outtmpl=outtmpl, for_download=True)) as ydl: - info_raw = ydl.extract_info(url, download=True) - info = _pick_video_info(info_raw) - if not info: - raise ValueError("yt-dlp returned empty info after download") - sanitized = ydl.sanitize_info(info) - except DownloadError as e: - raise RuntimeError(str(e)) from e - - meta = summarize_info(_as_dict(sanitized), fallback_url=url) - - # Determine output file. +def _resolve_downloaded_file(info: dict[str, Any], *, work_dir: Path) -> Path: info_d = _as_dict(info) candidates: list[Path] = [] for key in ("filepath", "_filename"): @@ -291,7 +267,7 @@ def download_youtube_video(url: str, settings: OrchestratorSettings, *, work_dir for p in candidates: if p.is_file(): - return p, _as_dict(sanitized), meta + return p # Fallback: pick the largest plausible media file in work_dir. media_exts = {".mp4", ".mkv", ".webm", ".mov", ".flv", ".avi", ".m4v"} @@ -299,10 +275,40 @@ def download_youtube_video(url: str, settings: OrchestratorSettings, *, work_dir media_files = [p for p in files if p.suffix.lower() in media_exts and not p.name.endswith(".part")] if media_files: media_files.sort(key=lambda p: p.stat().st_size, reverse=True) - return media_files[0], _as_dict(sanitized), meta + return media_files[0] if files: files.sort(key=lambda p: p.stat().st_size, reverse=True) - return files[0], _as_dict(sanitized), meta + return files[0] raise RuntimeError("yt-dlp download succeeded but no output file was found") + + +def _download_once(url: str, settings: OrchestratorSettings, *, work_dir: Path) -> tuple[Path, dict[str, Any], YouTubeMeta]: + work_dir.mkdir(parents=True, exist_ok=True) + outtmpl = str(work_dir / "%(id)s.%(ext)s") + with yt_dlp.YoutubeDL(build_ydl_opts(settings, outtmpl=outtmpl, for_download=True)) as ydl: + info_raw = ydl.extract_info(url, download=True) + info = _pick_video_info(info_raw) + if not info: + raise ValueError("yt-dlp returned empty info after download") + sanitized = ydl.sanitize_info(info) + + meta = summarize_info(_as_dict(sanitized), fallback_url=url) + video_path = _resolve_downloaded_file(_as_dict(info), work_dir=work_dir) + return video_path, _as_dict(sanitized), meta + + +def download_youtube_video(url: str, settings: OrchestratorSettings, *, work_dir: Path) -> tuple[Path, dict[str, Any], YouTubeMeta]: + """ + Downloads the best quality video to work_dir and returns: + - output video file path + - sanitized info dict + - meta summary + """ + try: + return _download_once(url, settings, work_dir=work_dir) + except DownloadError as e: + raise RuntimeError(str(e)) from e + except Exception as e: + raise RuntimeError(str(e)) from e diff --git a/src/videoroll/apps/subtitle_service/main.py b/src/videoroll/apps/subtitle_service/main.py index 1260ae9..66699d9 100644 --- a/src/videoroll/apps/subtitle_service/main.py +++ b/src/videoroll/apps/subtitle_service/main.py @@ -7,6 +7,7 @@ import tempfile import time import uuid import zipfile +from datetime import datetime, timezone from pathlib import Path from typing import Any, Generator @@ -17,7 +18,8 @@ from sqlalchemy.orm import Session from videoroll.config import SubtitleServiceSettings, get_subtitle_settings from videoroll.db.base import Base -from videoroll.db.models import Asset, AssetKind, RenderJob, RenderJobStatus, SubtitleJob +from videoroll.db.auto_migrate import auto_migrate +from videoroll.db.models import Asset, AssetKind, RenderJob, RenderJobStatus, SubtitleJob, SubtitleJobStatus, Task from videoroll.db.session import db_session, get_engine from videoroll.storage.s3 import S3Store from videoroll.apps.subtitle_service.schemas import ( @@ -36,16 +38,16 @@ from videoroll.apps.subtitle_service.schemas import ( WhisperSettingsRead, ModelDownloadProxyTestRequest, ModelDownloadProxyTestResponse, - RenderJobRead, - RenderQueueRead, - RenderQueueSettingsRead, - RenderQueueSettingsUpdate, + TaskQueueItemRead, + TaskQueueRead, + TaskQueueSettingsRead, + TaskQueueSettingsUpdate, ) from videoroll.apps.subtitle_service.asr_settings_store import get_asr_settings, update_asr_settings from videoroll.apps.subtitle_service.auto_profile_store import get_auto_profile, update_auto_profile -from videoroll.apps.subtitle_service.render_queue_store import get_render_queue_settings, update_render_queue_settings +from videoroll.apps.subtitle_service.render_queue_store import get_task_queue_settings, update_task_queue_settings from videoroll.apps.subtitle_service.translate_settings_store import get_translate_settings, update_translate_settings -from videoroll.apps.subtitle_service.worker import celery_app +from videoroll.apps.subtitle_service.worker import TASK_QUEUE_LOCK_OWNER, celery_app from videoroll.utils.cpu import process_cpu_count from videoroll.utils.hf_hub import configure_hf_hub_proxy @@ -116,6 +118,7 @@ def _startup() -> None: settings = get_subtitle_settings() engine = get_engine(settings.database_url) Base.metadata.create_all(engine) + auto_migrate(settings.database_url) S3Store(settings).ensure_bucket() _models_dir(settings).mkdir(parents=True, exist_ok=True) @@ -453,7 +456,7 @@ def create_job(payload: SubtitleJobCreate, db: Session = Depends(get_db)) -> dic db.add(job) db.commit() db.refresh(job) - celery_app.send_task("subtitle_service.process_job", args=[str(job.id)], queue="subtitle") + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") return {"job_id": str(job.id), "status": job.status.value} @@ -495,62 +498,262 @@ def get_job(job_id: uuid.UUID, db: Session = Depends(get_db)) -> SubtitleJobRead ) -@app.get("/subtitle/render_queue", response_model=RenderQueueRead) -def get_render_queue( - limit: int = 200, - db: Session = Depends(get_db), -) -> RenderQueueRead: - limit = int(limit or 200) - if limit < 1: - limit = 1 - if limit > 2000: - limit = 2000 +def _clamp_queue_limit(v: int) -> int: + v = int(v or 200) + if v < 1: + return 1 + if v > 2000: + return 2000 + return v - cfg = get_render_queue_settings(db) - running = ( - db.query(RenderJob) - .filter(RenderJob.status == RenderJobStatus.running) - .order_by(RenderJob.started_at.asc(), RenderJob.created_at.asc()) - .limit(limit) - .all() - ) - queued = ( - db.query(RenderJob) - .filter(RenderJob.status == RenderJobStatus.queued) - .order_by(RenderJob.created_at.asc()) - .limit(max(0, limit - len(running))) - .all() - ) - jobs = [*running, *queued] - def _to_read(j: RenderJob) -> RenderJobRead: - return RenderJobRead( - id=j.id, - task_id=j.task_id, - subtitle_job_id=j.subtitle_job_id, - status=j.status.value, - progress=int(j.progress or 0), - retry_count=int(j.retry_count or 0), - error_message=j.error_message, - created_at=j.created_at, - updated_at=j.updated_at, - started_at=j.started_at, - finished_at=j.finished_at, +def _read_task_queue(db: Session, *, limit: int) -> TaskQueueRead: + limit = _clamp_queue_limit(limit) + cfg = get_task_queue_settings(db) + now = datetime.now(tz=timezone.utc) + + locked_q = db.query(Task).filter(Task.lock_owner == TASK_QUEUE_LOCK_OWNER, Task.lock_until.is_not(None), Task.lock_until > now) + running_count = int(locked_q.count() or 0) + locked = locked_q.order_by(Task.lock_until.asc()).limit(limit).all() + locked_ids = [t.id for t in locked] + + render_running_by_task: dict[uuid.UUID, RenderJob] = {} + subtitle_running_by_task: dict[uuid.UUID, SubtitleJob] = {} + render_queued_by_task: dict[uuid.UUID, RenderJob] = {} + subtitle_queued_by_task: dict[uuid.UUID, SubtitleJob] = {} + + if locked_ids: + for rj in ( + db.query(RenderJob) + .filter(RenderJob.task_id.in_(locked_ids), RenderJob.status == RenderJobStatus.running) + .order_by(RenderJob.started_at.desc().nullslast(), RenderJob.updated_at.desc(), RenderJob.created_at.desc()) + .all() + ): + render_running_by_task.setdefault(rj.task_id, rj) + for sj in ( + db.query(SubtitleJob) + .filter(SubtitleJob.task_id.in_(locked_ids), SubtitleJob.status == SubtitleJobStatus.running) + .order_by(SubtitleJob.updated_at.desc(), SubtitleJob.created_at.desc()) + .all() + ): + subtitle_running_by_task.setdefault(sj.task_id, sj) + for rj in ( + db.query(RenderJob) + .filter(RenderJob.task_id.in_(locked_ids), RenderJob.status == RenderJobStatus.queued) + .order_by(RenderJob.created_at.asc()) + .all() + ): + render_queued_by_task.setdefault(rj.task_id, rj) + for sj in ( + db.query(SubtitleJob) + .filter(SubtitleJob.task_id.in_(locked_ids), SubtitleJob.status == SubtitleJobStatus.queued) + .order_by(SubtitleJob.created_at.asc()) + .all() + ): + subtitle_queued_by_task.setdefault(sj.task_id, sj) + + running_items: list[TaskQueueItemRead] = [] + for t in locked: + tid = t.id + r_run = render_running_by_task.get(tid) + s_run = subtitle_running_by_task.get(tid) + r_q = render_queued_by_task.get(tid) + s_q = subtitle_queued_by_task.get(tid) + + if r_run: + running_items.append( + TaskQueueItemRead( + task_id=tid, + state="running", + stage="render", + render_job_id=r_run.id, + subtitle_job_id=r_run.subtitle_job_id, + progress=int(r_run.progress or 0), + error_message=r_run.error_message, + created_at=r_run.created_at, + updated_at=r_run.updated_at, + ) + ) + continue + if s_run: + running_items.append( + TaskQueueItemRead( + task_id=tid, + state="running", + stage="subtitle", + subtitle_job_id=s_run.id, + progress=int(s_run.progress or 0), + error_message=s_run.error_message, + created_at=s_run.created_at, + updated_at=s_run.updated_at, + ) + ) + continue + if r_q: + running_items.append( + TaskQueueItemRead( + task_id=tid, + state="running", + stage="waiting_render", + render_job_id=r_q.id, + subtitle_job_id=r_q.subtitle_job_id, + progress=int(r_q.progress or 0), + error_message=r_q.error_message, + created_at=r_q.created_at, + updated_at=r_q.updated_at, + ) + ) + continue + if s_q: + running_items.append( + TaskQueueItemRead( + task_id=tid, + state="running", + stage="waiting_subtitle", + subtitle_job_id=s_q.id, + progress=int(s_q.progress or 0), + error_message=s_q.error_message, + created_at=s_q.created_at, + updated_at=s_q.updated_at, + ) + ) + continue + + running_items.append( + TaskQueueItemRead( + task_id=tid, + state="running", + stage="idle", + progress=0, + error_message=None, + created_at=t.created_at, + updated_at=t.updated_at, + ) ) - running_count = db.query(RenderJob).filter(RenderJob.status == RenderJobStatus.running).count() - queued_count = db.query(RenderJob).filter(RenderJob.status == RenderJobStatus.queued).count() + unlocked = (Task.lock_owner != TASK_QUEUE_LOCK_OWNER) | (Task.lock_until.is_(None)) | (Task.lock_until <= now) + # queued_count: distinct tasks with queued subtitle/render jobs (best-effort, capped). + queued_task_ids: set[uuid.UUID] = set() + for tid, in ( + db.query(SubtitleJob.task_id) + .join(Task, Task.id == SubtitleJob.task_id) + .filter(SubtitleJob.status == SubtitleJobStatus.queued, unlocked) + .distinct() + .limit(5000) + .all() + ): + queued_task_ids.add(tid) + for tid, in ( + db.query(RenderJob.task_id) + .join(Task, Task.id == RenderJob.task_id) + .filter(RenderJob.status == RenderJobStatus.queued, unlocked) + .distinct() + .limit(5000) + .all() + ): + queued_task_ids.add(tid) - return RenderQueueRead( - settings=RenderQueueSettingsRead(**cfg), - running_count=int(running_count), + queued_count = len(queued_task_ids) + + remaining = max(0, limit - len(running_items)) + queued_items: list[TaskQueueItemRead] = [] + seen: set[uuid.UUID] = set() + + fetch_n = min(5000, max(50, remaining * 20)) + for sj in ( + db.query(SubtitleJob) + .join(Task, Task.id == SubtitleJob.task_id) + .filter(SubtitleJob.status == SubtitleJobStatus.queued, unlocked) + .order_by(SubtitleJob.created_at.asc()) + .limit(fetch_n) + .all() + ): + if sj.task_id in seen: + continue + seen.add(sj.task_id) + queued_items.append( + TaskQueueItemRead( + task_id=sj.task_id, + state="queued", + stage="subtitle", + subtitle_job_id=sj.id, + progress=int(sj.progress or 0), + error_message=sj.error_message, + created_at=sj.created_at, + updated_at=sj.updated_at, + ) + ) + if len(queued_items) >= remaining: + break + + if len(queued_items) < remaining: + for rj in ( + db.query(RenderJob) + .join(Task, Task.id == RenderJob.task_id) + .filter(RenderJob.status == RenderJobStatus.queued, unlocked) + .order_by(RenderJob.created_at.asc()) + .limit(fetch_n) + .all() + ): + if rj.task_id in seen: + continue + seen.add(rj.task_id) + queued_items.append( + TaskQueueItemRead( + task_id=rj.task_id, + state="queued", + stage="render", + render_job_id=rj.id, + subtitle_job_id=rj.subtitle_job_id, + progress=int(rj.progress or 0), + error_message=rj.error_message, + created_at=rj.created_at, + updated_at=rj.updated_at, + ) + ) + if len(queued_items) >= remaining: + break + + return TaskQueueRead( + settings=TaskQueueSettingsRead(**cfg), + running_count=running_count, queued_count=int(queued_count), - jobs=[_to_read(j) for j in jobs], + tasks=[*running_items, *queued_items], ) -@app.put("/subtitle/render_queue/settings", response_model=RenderQueueSettingsRead) -def put_render_queue_settings_view(payload: RenderQueueSettingsUpdate, db: Session = Depends(get_db)) -> RenderQueueSettingsRead: - cfg = update_render_queue_settings(db, payload.model_dump(exclude_unset=True)) - celery_app.send_task("subtitle_service.render_queue_tick", args=[], queue="subtitle") - return RenderQueueSettingsRead(**cfg) +@app.get("/subtitle/task_queue", response_model=TaskQueueRead) +def get_task_queue(limit: int = 200, db: Session = Depends(get_db)) -> TaskQueueRead: + return _read_task_queue(db, limit=limit) + + +@app.get("/subtitle/render_queue", response_model=TaskQueueRead) +def get_render_queue_legacy(limit: int = 200, db: Session = Depends(get_db)) -> TaskQueueRead: + return _read_task_queue(db, limit=limit) + + +@app.put("/subtitle/task_queue/settings", response_model=TaskQueueSettingsRead) +def put_task_queue_settings_view(payload: TaskQueueSettingsUpdate, db: Session = Depends(get_db)) -> TaskQueueSettingsRead: + cfg = update_task_queue_settings(db, payload.model_dump(exclude_unset=True)) + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") + return TaskQueueSettingsRead(**cfg) + + +@app.put("/subtitle/render_queue/settings", response_model=TaskQueueSettingsRead) +def put_render_queue_settings_legacy(payload: TaskQueueSettingsUpdate, db: Session = Depends(get_db)) -> TaskQueueSettingsRead: + return put_task_queue_settings_view(payload, db=db) + + +@app.post("/subtitle/task_queue/tick") +def post_task_queue_tick() -> dict[str, str]: + """ + Best-effort scheduler kick. + Useful when a job is queued but no tick was delivered/consumed. + """ + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") + return {"status": "queued"} + + +@app.post("/subtitle/render_queue/tick") +def post_render_queue_tick_legacy() -> dict[str, str]: + return post_task_queue_tick() diff --git a/src/videoroll/apps/subtitle_service/processing.py b/src/videoroll/apps/subtitle_service/processing.py index 255299b..f5aede0 100644 --- a/src/videoroll/apps/subtitle_service/processing.py +++ b/src/videoroll/apps/subtitle_service/processing.py @@ -7,7 +7,7 @@ import subprocess import time from dataclasses import dataclass from pathlib import Path -from typing import Any, Iterable +from typing import Any, Callable, Iterable import httpx @@ -26,26 +26,80 @@ def _run(cmd: list[str]) -> None: subprocess.run(cmd, check=True) -def _run_logged(cmd: list[str], *, log_path: Path | None) -> None: +def _run_logged( + cmd: list[str], + *, + log_path: Path | None, + live_upload_cb: Callable[[], None] | None = None, + live_upload_interval_seconds: float = 3.0, +) -> None: if log_path is None: _run(cmd) return log_path.parent.mkdir(parents=True, exist_ok=True) try: - with log_path.open("ab") as f: - try: - f.write(("\n$ " + " ".join(cmd) + "\n").encode("utf-8", errors="replace")) - f.flush() - except Exception: - pass - subprocess.run(cmd, check=True, stdout=f, stderr=f) + f = log_path.open("ab") except Exception: - # If logging fails, still run the command so the pipeline doesn't break. + # If logging can't even open the file, still run the command so the pipeline doesn't break. _run(cmd) + return + + with f: + try: + f.write(("\n$ " + " ".join(cmd) + "\n").encode("utf-8", errors="replace")) + f.flush() + except Exception: + pass + + if live_upload_cb is None: + subprocess.run(cmd, check=True, stdout=f, stderr=f) + return + + try: + proc = subprocess.Popen(cmd, stdout=f, stderr=f) + except Exception: + # If we can't start the process with live upload, fall back to the simple runner. + subprocess.run(cmd, check=True, stdout=f, stderr=f) + return + + interval = float(live_upload_interval_seconds or 0) + if interval <= 0: + interval = 3.0 + # Keep it responsive without busy-looping. + tick_sleep = min(0.25, interval) + + next_upload_at = time.monotonic() + interval + while True: + rc = proc.poll() + now = time.monotonic() + if now >= next_upload_at: + try: + live_upload_cb() + except Exception: + pass + next_upload_at = now + interval + if rc is not None: + break + time.sleep(tick_sleep) + + try: + live_upload_cb() + except Exception: + pass + + if rc != 0: + raise subprocess.CalledProcessError(int(rc), cmd) -def extract_audio(ffmpeg_path: str, video_path: Path, audio_path: Path, *, log_path: Path | None = None) -> None: +def extract_audio( + ffmpeg_path: str, + video_path: Path, + audio_path: Path, + *, + log_path: Path | None = None, + live_upload_cb: Callable[[], None] | None = None, +) -> None: audio_path.parent.mkdir(parents=True, exist_ok=True) cmd = [ ffmpeg_path, @@ -59,7 +113,7 @@ def extract_audio(ffmpeg_path: str, video_path: Path, audio_path: Path, *, log_p "16000", str(audio_path), ] - _run_logged(cmd, log_path=log_path) + _run_logged(cmd, log_path=log_path, live_upload_cb=live_upload_cb) def transcribe_mock(_audio_path: Path) -> list[Segment]: @@ -609,6 +663,7 @@ def render_burn_in( preset: str | int | None = None, crf: int | None = None, log_path: Path | None = None, + live_upload_cb: Callable[[], None] | None = None, ) -> None: output_path.parent.mkdir(parents=True, exist_ok=True) codec = str(video_codec or "").strip().lower() or "av1" @@ -655,10 +710,18 @@ def render_burn_in( "copy", str(output_path), ] - _run_logged(cmd, log_path=log_path) + _run_logged(cmd, log_path=log_path, live_upload_cb=live_upload_cb) -def mux_soft_sub(ffmpeg_path: str, video_path: Path, srt_path: Path, output_path: Path, *, log_path: Path | None = None) -> None: +def mux_soft_sub( + ffmpeg_path: str, + video_path: Path, + srt_path: Path, + output_path: Path, + *, + log_path: Path | None = None, + live_upload_cb: Callable[[], None] | None = None, +) -> None: output_path.parent.mkdir(parents=True, exist_ok=True) cmd = [ ffmpeg_path, @@ -681,7 +744,7 @@ def mux_soft_sub(ffmpeg_path: str, video_path: Path, srt_path: Path, output_path "language=chi", str(output_path), ] - _run_logged(cmd, log_path=log_path) + _run_logged(cmd, log_path=log_path, live_upload_cb=live_upload_cb) def write_json(path: Path, data: Any) -> None: diff --git a/src/videoroll/apps/subtitle_service/render_queue_store.py b/src/videoroll/apps/subtitle_service/render_queue_store.py index 21d288a..8d9ee0c 100644 --- a/src/videoroll/apps/subtitle_service/render_queue_store.py +++ b/src/videoroll/apps/subtitle_service/render_queue_store.py @@ -7,7 +7,15 @@ from sqlalchemy.orm import Session from videoroll.db.models import AppSetting -RENDER_QUEUE_SETTINGS_KEY = "subtitle.render_queue" +# +# NOTE: This module was originally used for the "render queue" (ffmpeg only). +# The queue is now task-based: max_concurrency limits how many *tasks* (pipelines) +# can be in-flight at the same time. +# +# We keep the filename and legacy key for backward compatibility. +# +LEGACY_RENDER_QUEUE_SETTINGS_KEY = "subtitle.render_queue" +TASK_QUEUE_SETTINGS_KEY = "subtitle.task_queue" _MAX_CONCURRENCY_MIN = 0 _MAX_CONCURRENCY_MAX = 32 @@ -21,11 +29,11 @@ def _as_dict(v: Any) -> dict[str, Any]: return v if isinstance(v, dict) else {} -def _get_row(db: Session) -> AppSetting: - row = db.get(AppSetting, RENDER_QUEUE_SETTINGS_KEY) +def _get_row(db: Session, key: str) -> AppSetting: + row = db.get(AppSetting, key) if row: return row - row = AppSetting(key=RENDER_QUEUE_SETTINGS_KEY, value_json={}) + row = AppSetting(key=key, value_json={}) db.add(row) db.commit() db.refresh(row) @@ -45,8 +53,31 @@ def _normalize_max_concurrency(v: Any) -> int: def get_render_queue_settings(db: Session) -> dict[str, Any]: - row = db.get(AppSetting, RENDER_QUEUE_SETTINGS_KEY) + # Legacy alias. + return get_task_queue_settings(db) + + +def get_task_queue_settings(db: Session) -> dict[str, Any]: + row = db.get(AppSetting, TASK_QUEUE_SETTINGS_KEY) stored = dict(_as_dict(row.value_json)) if row else {} + + # Auto-migrate legacy key on first read (best-effort). + if not stored: + legacy = db.get(AppSetting, LEGACY_RENDER_QUEUE_SETTINGS_KEY) + legacy_stored = dict(_as_dict(legacy.value_json)) if legacy else {} + if legacy_stored: + stored = legacy_stored + try: + new_row = _get_row(db, TASK_QUEUE_SETTINGS_KEY) + new_row.value_json = dict(legacy_stored) + db.add(new_row) + db.commit() + except Exception: + try: + db.rollback() + except Exception: + pass + baseline = _default_settings() merged = {**baseline, **stored} merged["max_concurrency"] = _normalize_max_concurrency(merged.get("max_concurrency")) @@ -54,7 +85,12 @@ def get_render_queue_settings(db: Session) -> dict[str, Any]: def update_render_queue_settings(db: Session, update: dict[str, Any]) -> dict[str, Any]: - row = _get_row(db) + # Legacy alias. + return update_task_queue_settings(db, update) + + +def update_task_queue_settings(db: Session, update: dict[str, Any]) -> dict[str, Any]: + row = _get_row(db, TASK_QUEUE_SETTINGS_KEY) stored = dict(_as_dict(row.value_json)) if "max_concurrency" in update and update["max_concurrency"] is not None: @@ -63,5 +99,4 @@ def update_render_queue_settings(db: Session, update: dict[str, Any]) -> dict[st row.value_json = stored db.add(row) db.commit() - return get_render_queue_settings(db) - + return get_task_queue_settings(db) diff --git a/src/videoroll/apps/subtitle_service/schemas.py b/src/videoroll/apps/subtitle_service/schemas.py index b90de80..2b67f88 100644 --- a/src/videoroll/apps/subtitle_service/schemas.py +++ b/src/videoroll/apps/subtitle_service/schemas.py @@ -218,8 +218,35 @@ class ModelDownloadProxyTestResponse(BaseModel): error: Optional[str] = None +class TaskQueueSettingsRead(BaseModel): + max_concurrency: int = Field(1, description="0=暂停调度；>0 表示最多同时运行多少个任务（Task pipeline）") + + +class TaskQueueSettingsUpdate(BaseModel): + max_concurrency: Optional[int] = Field(default=None, ge=0, le=32) + + +class TaskQueueItemRead(BaseModel): + task_id: uuid.UUID + state: str + stage: str + subtitle_job_id: Optional[uuid.UUID] = None + render_job_id: Optional[uuid.UUID] = None + progress: int = 0 + error_message: Optional[str] = None + created_at: datetime + updated_at: datetime + + +class TaskQueueRead(BaseModel): + settings: TaskQueueSettingsRead + running_count: int = 0 + queued_count: int = 0 + tasks: list[TaskQueueItemRead] = Field(default_factory=list) + + class RenderQueueSettingsRead(BaseModel): - max_concurrency: int = Field(1, description="0=暂停调度；>0 表示最多同时运行多少个 ffmpeg 压制任务") + max_concurrency: int = Field(1, description="(legacy) use TaskQueueSettingsRead instead") class RenderQueueSettingsUpdate(BaseModel): @@ -245,3 +272,4 @@ class RenderQueueRead(BaseModel): running_count: int = 0 queued_count: int = 0 jobs: list[RenderJobRead] = Field(default_factory=list) + history: list[RenderJobRead] = Field(default_factory=list, description="最近已结束（succeeded/failed/canceled）的压制任务列表（用于排查消失/失败原因）") diff --git a/src/videoroll/apps/subtitle_service/task_title_store.py b/src/videoroll/apps/subtitle_service/task_title_store.py index 7fafa25..19574a5 100644 --- a/src/videoroll/apps/subtitle_service/task_title_store.py +++ b/src/videoroll/apps/subtitle_service/task_title_store.py @@ -1,10 +1,13 @@ from __future__ import annotations +import json +import uuid from typing import Any from sqlalchemy.orm import Session -from videoroll.db.models import AppSetting +from videoroll.db.models import AppSetting, Asset, AssetKind +from videoroll.storage.s3 import S3Store def _as_dict(v: Any) -> dict[str, Any]: @@ -29,8 +32,47 @@ def get_task_titles(db: Session, task_id: str) -> dict[str, str]: def get_task_display_title(db: Session, task_id: str) -> str: + return get_task_display_title_with_s3(db, task_id, s3=None) + + +def get_task_display_title_with_s3(db: Session, task_id: str, *, s3: S3Store | None) -> str: t = get_task_titles(db, task_id) - return str(t.get("translated_title") or t.get("source_title") or "").strip() + out = str(t.get("translated_title") or t.get("source_title") or "").strip() + if out: + return out + + if s3 is None: + return "" + + try: + tid = uuid.UUID(str(task_id)) + except Exception: + return "" + + asset = ( + db.query(Asset) + .filter(Asset.task_id == tid, Asset.kind == AssetKind.metadata_json) + .order_by(Asset.created_at.desc()) + .first() + ) + if not asset: + return "" + + try: + obj = s3.get_object(asset.storage_key) + body = obj.get("Body") + raw = body.read() if body else b"" + try: + if body: + body.close() + except Exception: + pass + parsed = json.loads(raw.decode("utf-8")) if raw else {} + info = parsed if isinstance(parsed, dict) else {} + title = str(info.get("title") or info.get("fulltitle") or info.get("alt_title") or "").strip() + return title + except Exception: + return "" def set_task_titles( @@ -53,4 +95,3 @@ def set_task_titles( row.value_json = data db.add(row) db.commit() - diff --git a/src/videoroll/apps/subtitle_service/worker.py b/src/videoroll/apps/subtitle_service/worker.py index 77a9767..04feec7 100644 --- a/src/videoroll/apps/subtitle_service/worker.py +++ b/src/videoroll/apps/subtitle_service/worker.py @@ -4,21 +4,25 @@ import json import logging import re import shutil +import threading import time import traceback import uuid -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any import httpx from celery import Celery from celery.signals import worker_init +from sqlalchemy import or_ from sqlalchemy.orm import Session from videoroll.config import get_subtitle_settings from videoroll.db.base import Base +from videoroll.db.auto_migrate import auto_migrate from videoroll.db.models import ( + AppSetting, Asset, AssetKind, RenderJob, @@ -32,7 +36,9 @@ from videoroll.db.models import ( ) from videoroll.db.session import get_engine, get_sessionmaker from videoroll.storage.s3 import S3Store +from videoroll.utils.internal_api_token import internal_api_token from videoroll.utils.hashing import sha256_file +from videoroll.apps.orchestrator_api.admin_auth_store import INTERNAL_TOKEN_HEADER from videoroll.apps.subtitle_service.processing import ( Segment, extract_audio, @@ -55,12 +61,14 @@ from videoroll.apps.subtitle_service.bilibili_tags_store import set_task_bilibil from videoroll.apps.subtitle_service.task_title_store import set_task_titles from videoroll.apps.subtitle_service.translate_settings_store import get_translate_settings from videoroll.apps.bilibili_publisher.publish_settings_store import get_bilibili_publish_settings -from videoroll.apps.subtitle_service.render_queue_store import get_render_queue_settings +from videoroll.apps.subtitle_service.render_queue_store import TASK_QUEUE_SETTINGS_KEY, get_task_queue_settings from videoroll.utils.cpu import process_cpu_count from videoroll.utils.hf_hub import configure_hf_hub_proxy settings = get_subtitle_settings() +_ORCH_INTERNAL_TOKEN = internal_api_token(settings.s3_secret_access_key) +_ORCH_INTERNAL_HEADERS = {INTERNAL_TOKEN_HEADER: _ORCH_INTERNAL_TOKEN} if _ORCH_INTERNAL_TOKEN else {} logger = logging.getLogger(__name__) celery_app = Celery("subtitle_service", broker=settings.redis_url, backend=settings.redis_url) celery_app.conf.update( @@ -138,12 +146,82 @@ def _db() -> Session: def _ensure_db() -> None: engine = get_engine(settings.database_url) Base.metadata.create_all(engine) + auto_migrate(settings.database_url) def _now() -> datetime: return datetime.now(tz=timezone.utc) +# Task Queue (task-level concurrency) +TASK_QUEUE_LOCK_OWNER = "subtitle_service.task_queue" +_TASK_QUEUE_LOCK_TTL = timedelta(seconds=300) +_TASK_QUEUE_HEARTBEAT_INTERVAL_SECONDS = 30.0 +_TASK_QUEUE_REQUEUE_COUNTDOWN_SECONDS = 10 + + +def _task_queue_expires_at(now: datetime) -> datetime: + return now + _TASK_QUEUE_LOCK_TTL + + +def _task_queue_is_task_locked(task: Task, now: datetime) -> bool: + return bool(task.lock_owner == TASK_QUEUE_LOCK_OWNER and task.lock_until and task.lock_until > now) + + +def _task_queue_unlock(task: Task) -> None: + task.lock_owner = None + task.lock_until = None + + +def _task_queue_lock_settings_row(db: Session) -> None: + """ + Serialize queue ticks by locking the settings row. + This prevents overshooting max_concurrency when multiple ticks run concurrently. + """ + row = db.get(AppSetting, TASK_QUEUE_SETTINGS_KEY) + if not row: + row = AppSetting(key=TASK_QUEUE_SETTINGS_KEY, value_json={}) + db.add(row) + db.commit() + # Best-effort lock (ignored on dialects that don't support it). + db.query(AppSetting).filter(AppSetting.key == TASK_QUEUE_SETTINGS_KEY).with_for_update().first() + + +class _TaskQueueHeartbeat: + def __init__(self, task_id: uuid.UUID): + self._task_id = task_id + self._stop = threading.Event() + self._thr = threading.Thread(target=self._run, name=f"task-queue-hb-{task_id}", daemon=True) + + def start(self) -> None: + self._thr.start() + + def stop(self) -> None: + self._stop.set() + try: + self._thr.join(timeout=5.0) + except Exception: + pass + + def _run(self) -> None: + while not self._stop.wait(_TASK_QUEUE_HEARTBEAT_INTERVAL_SECONDS): + db = _db() + try: + now = _now() + db.query(Task).filter(Task.id == self._task_id, Task.lock_owner == TASK_QUEUE_LOCK_OWNER).update( + {"lock_until": _task_queue_expires_at(now)}, + synchronize_session=False, + ) + db.commit() + except Exception: + try: + db.rollback() + except Exception: + pass + finally: + db.close() + + def _append_log_line(log_path: Path, message: str) -> None: log_path.parent.mkdir(parents=True, exist_ok=True) msg = (message or "").rstrip("\n") @@ -422,6 +500,8 @@ def _recover_interrupted_subtitle_jobs() -> None: db.add(old) task = db.get(Task, old.task_id) if task and task.status not in {TaskStatus.published, TaskStatus.canceled}: + if task.lock_owner == TASK_QUEUE_LOCK_OWNER: + _task_queue_unlock(task) task.status = TaskStatus.failed task.error_code = "SUBTITLE_CRASHED" task.error_message = "subtitle job crashed; manual resume required" @@ -477,7 +557,7 @@ def _on_worker_init(**_kwargs: Any) -> None: try: _recover_interrupted_subtitle_jobs() _recover_interrupted_render_jobs() - celery_app.send_task("subtitle_service.render_queue_tick", args=[], queue="subtitle") + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") except Exception: logger.exception("subtitle job recovery failed") @@ -492,6 +572,7 @@ def process_job(job_id: str) -> dict[str, str]: db = _db() log_path: Path | None = None log_key: str | None = None + hb: _TaskQueueHeartbeat | None = None try: job = db.get(SubtitleJob, jid) if not job: @@ -510,11 +591,32 @@ def process_job(job_id: str) -> dict[str, str]: db.commit() return {"status": "error", "detail": "task not found"} + now = _now() + if task.lock_owner != TASK_QUEUE_LOCK_OWNER: + # Not claimed by the task-level scheduler; put it back and wait. + job.status = SubtitleJobStatus.queued + db.add(job) + db.commit() + celery_app.send_task( + "subtitle_service.task_queue_tick", + args=[], + queue="subtitle", + countdown=_TASK_QUEUE_REQUEUE_COUNTDOWN_SECONDS, + ) + return {"status": "queued", "detail": "waiting for task queue"} + if task.lock_until is None or task.lock_until <= now: + task.lock_until = _task_queue_expires_at(now) + db.add(task) + + # progress=1 is set by the scheduler; bump to >=2 ASAP to mark as claimed by a worker. job.status = SubtitleJobStatus.running - job.progress = 1 + job.progress = max(int(job.progress or 0), 2) db.add(job) db.commit() + hb = _TaskQueueHeartbeat(task.id) + hb.start() + req = job.request_json input_key = (req.get("input") or {}).get("key") if not input_key: @@ -529,10 +631,10 @@ def process_job(job_id: str) -> dict[str, str]: srt_path = work_root / "subtitle_zh.srt" ass_path = work_root / "subtitle_zh.ass" - audio_key = f"work/{task.id}/audio.wav" - segments_key = f"sub/{task.id}/segments.json" - srt_key = f"sub/{task.id}/subtitle_zh.srt" - ass_key = f"sub/{task.id}/subtitle_zh.ass" + audio_key: str | None = None + segments_key: str | None = None + srt_key: str | None = None + ass_key: str | None = None resume = bool(req.get("resume")) output_cfg = (req.get("output") or {}) @@ -565,22 +667,26 @@ def process_job(job_id: str) -> dict[str, str]: ) _safe_upload_log(store, log_path, log_key) - def _download_if_asset_exists(kind: AssetKind, key: str, dest: Path) -> bool: + def _download_latest_asset(kind: AssetKind, dest: Path) -> Asset | None: row = ( db.query(Asset) - .filter(Asset.task_id == task.id, Asset.kind == kind, Asset.storage_key == key) + .filter(Asset.task_id == task.id, Asset.kind == kind) .order_by(Asset.created_at.desc()) .first() ) if not row: - return False + return None try: - store.download_file(key, dest) - return dest.exists() and dest.stat().st_size > 0 + store.download_file(row.storage_key, dest) + if dest.exists() and dest.stat().st_size > 0: + return row except Exception: - return False + return None + return None - if resume and _download_if_asset_exists(AssetKind.subtitle_srt, srt_key, srt_path): + srt_asset = _download_latest_asset(AssetKind.subtitle_srt, srt_path) if resume else None + if resume and srt_asset: + srt_key = srt_asset.storage_key _safe_append_log_line(log_path, f"resume: found existing subtitle_srt asset: {srt_key}") _safe_upload_log(store, log_path, log_key) if task.status in {TaskStatus.failed, TaskStatus.created, TaskStatus.ingested, TaskStatus.downloaded, TaskStatus.audio_extracted, TaskStatus.asr_done, TaskStatus.translated}: @@ -590,32 +696,41 @@ def process_job(job_id: str) -> dict[str, str]: db.add(job) db.commit() - if need_ass and not _download_if_asset_exists(AssetKind.subtitle_ass, ass_key, ass_path): - segs = srt_to_segments(srt_path.read_text(encoding="utf-8")) - ass_text = segments_to_ass(segs, style_name=render_cfg.get("ass_style", "clean_white")) - ass_path.write_text(ass_text, encoding="utf-8") - store.upload_file(ass_path, ass_key, content_type="text/plain") - db.add( - Asset( - task_id=task.id, - kind=AssetKind.subtitle_ass, - storage_key=ass_key, - sha256=sha256_file(ass_path), - size_bytes=ass_path.stat().st_size, + if need_ass: + ass_asset = _download_latest_asset(AssetKind.subtitle_ass, ass_path) + if ass_asset: + ass_key = ass_asset.storage_key + else: + segs = srt_to_segments(srt_path.read_text(encoding="utf-8")) + ass_text = segments_to_ass(segs, style_name=render_cfg.get("ass_style", "clean_white")) + ass_path.write_text(ass_text, encoding="utf-8") + ass_sha = sha256_file(ass_path) + ass_key = f"sub/{task.id}/subtitle_zh_{ass_sha[:16]}.ass" + store.upload_file(ass_path, ass_key, content_type="text/plain") + db.add( + Asset( + task_id=task.id, + kind=AssetKind.subtitle_ass, + storage_key=ass_key, + sha256=ass_sha, + size_bytes=ass_path.stat().st_size, + ) ) - ) - db.add(Subtitle(task_id=task.id, version=1, format=SubtitleFormat.ass, language="zh", storage_key=ass_key)) - db.commit() - _safe_append_log_line(log_path, f"generated ass from resumed srt: {ass_key}") - _safe_upload_log(store, log_path, log_key) + db.add(Subtitle(task_id=task.id, version=1, format=SubtitleFormat.ass, language="zh", storage_key=ass_key)) + db.commit() + _safe_append_log_line(log_path, f"generated ass from resumed srt: {ass_key}") + _safe_upload_log(store, log_path, log_key) if not burn_in and not soft_sub: job.status = SubtitleJobStatus.succeeded job.progress = 100 + _task_queue_unlock(task) + db.add(task) db.add(job) db.commit() _safe_append_log_line(log_path, "subtitle job done (no render configured)") _safe_upload_log(store, log_path, log_key) + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") return {"status": "ok"} after_render = req.get("after_render") if isinstance(req, dict) else None @@ -652,9 +767,9 @@ def process_job(job_id: str) -> dict[str, str]: job.status = SubtitleJobStatus.queued db.add(job) db.commit() - _safe_append_log_line(log_path, "render queued; waiting for render queue") + _safe_append_log_line(log_path, "render queued; waiting for task queue") _safe_upload_log(store, log_path, log_key) - celery_app.send_task("subtitle_service.render_queue_tick", args=[], queue="subtitle") + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") return {"status": "ok", "detail": "render queued"} video_downloaded = False @@ -667,7 +782,9 @@ def process_job(job_id: str) -> dict[str, str]: video_downloaded = True segments: list[Segment] | None = None - if resume and _download_if_asset_exists(AssetKind.segments_json, segments_key, segments_path): + segments_asset = _download_latest_asset(AssetKind.segments_json, segments_path) if resume else None + if resume and segments_asset: + segments_key = segments_asset.storage_key try: data = json.loads(segments_path.read_text(encoding="utf-8")) if isinstance(data, list): @@ -687,20 +804,23 @@ def process_job(job_id: str) -> dict[str, str]: segments = None if segments is None: - if resume and _download_if_asset_exists(AssetKind.audio_wav, audio_key, audio_path): - pass + audio_asset = _download_latest_asset(AssetKind.audio_wav, audio_path) if resume else None + if resume and audio_asset: + audio_key = audio_asset.storage_key else: _ensure_video() _safe_append_log_line(log_path, "ffmpeg: extract audio") extract_audio(settings.ffmpeg_path, video_path, audio_path, log_path=log_path) _safe_upload_log(store, log_path, log_key) + audio_sha = sha256_file(audio_path) + audio_key = f"work/{task.id}/audio_{audio_sha[:16]}.wav" store.upload_file(audio_path, audio_key, content_type="audio/wav") db.add( Asset( task_id=task.id, kind=AssetKind.audio_wav, storage_key=audio_key, - sha256=sha256_file(audio_path), + sha256=audio_sha, size_bytes=audio_path.stat().st_size, ) ) @@ -747,8 +867,18 @@ def process_job(job_id: str) -> dict[str, str]: raise ValueError(f"unsupported ASR engine: {engine}") segments_json = [seg.__dict__ for seg in segments] write_json(segments_path, segments_json) + segments_sha = sha256_file(segments_path) + segments_key = f"sub/{task.id}/segments_{segments_sha[:16]}.json" store.upload_file(segments_path, segments_key, content_type="application/json") - db.add(Asset(task_id=task.id, kind=AssetKind.segments_json, storage_key=segments_key, size_bytes=segments_path.stat().st_size)) + db.add( + Asset( + task_id=task.id, + kind=AssetKind.segments_json, + storage_key=segments_key, + sha256=segments_sha, + size_bytes=segments_path.stat().st_size, + ) + ) task.status = TaskStatus.asr_done db.add(task) @@ -888,16 +1018,36 @@ def process_job(job_id: str) -> dict[str, str]: srt_text = segments_to_srt(segments_out) srt_path.write_text(srt_text, encoding="utf-8") + srt_sha = sha256_file(srt_path) + srt_key = f"sub/{task.id}/subtitle_zh_{srt_sha[:16]}.srt" store.upload_file(srt_path, srt_key, content_type="text/plain") - db.add(Asset(task_id=task.id, kind=AssetKind.subtitle_srt, storage_key=srt_key, sha256=sha256_file(srt_path), size_bytes=srt_path.stat().st_size)) + db.add( + Asset( + task_id=task.id, + kind=AssetKind.subtitle_srt, + storage_key=srt_key, + sha256=srt_sha, + size_bytes=srt_path.stat().st_size, + ) + ) db.add(Subtitle(task_id=task.id, version=1, format=SubtitleFormat.srt, language="zh", storage_key=srt_key)) _safe_append_log_line(log_path, f"subtitle srt uploaded: {srt_key}") if need_ass: ass_text = segments_to_ass(segments_out, style_name=render_cfg.get("ass_style", "clean_white")) ass_path.write_text(ass_text, encoding="utf-8") + ass_sha = sha256_file(ass_path) + ass_key = f"sub/{task.id}/subtitle_zh_{ass_sha[:16]}.ass" store.upload_file(ass_path, ass_key, content_type="text/plain") - db.add(Asset(task_id=task.id, kind=AssetKind.subtitle_ass, storage_key=ass_key, sha256=sha256_file(ass_path), size_bytes=ass_path.stat().st_size)) + db.add( + Asset( + task_id=task.id, + kind=AssetKind.subtitle_ass, + storage_key=ass_key, + sha256=ass_sha, + size_bytes=ass_path.stat().st_size, + ) + ) db.add(Subtitle(task_id=task.id, version=1, format=SubtitleFormat.ass, language="zh", storage_key=ass_key)) _safe_append_log_line(log_path, f"subtitle ass uploaded: {ass_key}") @@ -911,10 +1061,13 @@ def process_job(job_id: str) -> dict[str, str]: if not burn_in and not soft_sub: job.status = SubtitleJobStatus.succeeded job.progress = 100 + _task_queue_unlock(task) + db.add(task) db.add(job) db.commit() _safe_append_log_line(log_path, "subtitle job done (no render configured)") _safe_upload_log(store, log_path, log_key) + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") return {"status": "ok"} after_render = req.get("after_render") if isinstance(req, dict) else None @@ -951,9 +1104,9 @@ def process_job(job_id: str) -> dict[str, str]: job.status = SubtitleJobStatus.queued db.add(job) db.commit() - _safe_append_log_line(log_path, "render queued; waiting for render queue") + _safe_append_log_line(log_path, "render queued; waiting for task queue") _safe_upload_log(store, log_path, log_key) - celery_app.send_task("subtitle_service.render_queue_tick", args=[], queue="subtitle") + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") return {"status": "ok", "detail": "render queued"} except Exception as e: job = db.get(SubtitleJob, uuid.UUID(job_id)) @@ -962,6 +1115,9 @@ def process_job(job_id: str) -> dict[str, str]: job.error_message = str(e) db.add(job) task = db.get(Task, job.task_id) + if task and task.lock_owner == TASK_QUEUE_LOCK_OWNER: + _task_queue_unlock(task) + db.add(task) if task and task.status not in {TaskStatus.published, TaskStatus.canceled}: task.status = TaskStatus.failed task.error_code = task.error_code or "SUBTITLE_FAILED" @@ -971,50 +1127,263 @@ def process_job(job_id: str) -> dict[str, str]: _safe_append_log_line(log_path, f"ERROR: {type(e).__name__}: {e}") _safe_append_log_block(log_path, traceback.format_exc()) _safe_upload_log(store, log_path, log_key) + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") return {"status": "error", "detail": str(e)} finally: + if hb is not None: + hb.stop() db.close() -@celery_app.task(name="subtitle_service.render_queue_tick") -def render_queue_tick() -> dict[str, Any]: +@celery_app.task(name="subtitle_service.task_queue_tick") +def task_queue_tick() -> dict[str, Any]: + """ + Task-level scheduler. + + max_concurrency now limits the number of *tasks* (pipelines) that can be in-flight. + A task occupies a slot from subtitle-job start until render finishes (or subtitle finishes + when no render is configured). + """ _ensure_db() db = _db() - started = 0 + now = _now() + + started_subtitle = 0 + started_render = 0 + recovered_subtitle = 0 + recovered_render = 0 + unlocked_expired = 0 + + to_start: list[tuple[str, str]] = [] # ("subtitle"|"render", job_id) try: - cfg = get_render_queue_settings(db) - max_conc = int(cfg.get("max_concurrency") or 1) + _task_queue_lock_settings_row(db) + cfg = get_task_queue_settings(db) + try: + max_conc = int(cfg.get("max_concurrency", 1)) + except Exception: + max_conc = 1 if max_conc < 0: max_conc = 0 if max_conc == 0: - return {"status": "paused", "max_concurrency": str(max_conc), "started": str(started)} + return {"status": "paused", "max_concurrency": str(max_conc)} - running_count = db.query(RenderJob).filter(RenderJob.status == RenderJobStatus.running).count() - running = int(running_count or 0) + # Clear expired locks to avoid permanent stalls after crashes. + try: + unlocked_expired = int( + db.query(Task) + .filter(Task.lock_owner == TASK_QUEUE_LOCK_OWNER, Task.lock_until.is_not(None), Task.lock_until <= now) + .update({"lock_owner": None, "lock_until": None}, synchronize_session=False) + or 0 + ) + except Exception: + unlocked_expired = 0 - while running + started < max_conc: - j = ( + # Recover render jobs that were marked as running by the scheduler but never claimed by a worker. + cutoff = now - timedelta(seconds=60) + # Subtitle jobs: scheduler sets progress=1; worker bumps it to >=2 once claimed. + stuck_subtitle = ( + db.query(SubtitleJob) + .filter( + SubtitleJob.status == SubtitleJobStatus.running, + SubtitleJob.progress <= 1, + # If the worker actually started, it sets logs_key very early. Treat jobs without logs_key + # as "scheduler started but worker never claimed". + SubtitleJob.logs_key.is_(None), + SubtitleJob.updated_at.is_not(None), + SubtitleJob.updated_at < cutoff, + ) + .order_by(SubtitleJob.updated_at.asc(), SubtitleJob.created_at.asc()) + .with_for_update(skip_locked=True) + .all() + ) + for j in stuck_subtitle: + msg = (j.error_message or "").strip() + detail = "检测到疑似 Worker 未消费：subtitle job 长时间停留在 running(1%)。已自动重新排队。" + j.error_message = f"{msg}\n{detail}" if msg else detail + j.status = SubtitleJobStatus.queued + j.progress = 0 + db.add(j) + recovered_subtitle += 1 + + stuck = ( + db.query(RenderJob) + .filter( + RenderJob.status == RenderJobStatus.running, + RenderJob.progress <= 1, + RenderJob.started_at.is_not(None), + RenderJob.started_at < cutoff, + ) + .order_by(RenderJob.started_at.asc(), RenderJob.created_at.asc()) + .with_for_update(skip_locked=True) + .all() + ) + for j in stuck: + msg = (j.error_message or "").strip() + detail = "检测到疑似 Worker 未消费：render job 长时间停留在 running(1%)。已自动重新排队。" + j.error_message = f"{msg}\n{detail}" if msg else detail + j.retry_count = int(j.retry_count or 0) + 1 + j.status = RenderJobStatus.queued + j.progress = 0 + j.started_at = None + j.finished_at = None + db.add(j) + recovered_render += 1 + + locked_tasks = ( + db.query(Task) + .filter(Task.lock_owner == TASK_QUEUE_LOCK_OWNER, Task.lock_until.is_not(None), Task.lock_until > now) + .order_by(Task.lock_until.asc()) + .all() + ) + + # Phase 1: advance locked tasks (start their next queued job if nothing is running). + for t in locked_tasks: + tid = t.id + has_running = ( + db.query(SubtitleJob).filter(SubtitleJob.task_id == tid, SubtitleJob.status == SubtitleJobStatus.running).count() + + db.query(RenderJob).filter(RenderJob.task_id == tid, RenderJob.status == RenderJobStatus.running).count() + ) + if has_running: + continue + + rj = ( db.query(RenderJob) - .filter(RenderJob.status == RenderJobStatus.queued) + .filter(RenderJob.task_id == tid, RenderJob.status == RenderJobStatus.queued) .order_by(RenderJob.created_at.asc()) .with_for_update(skip_locked=True) .first() ) - if not j: + if rj: + rj.status = RenderJobStatus.running + rj.progress = max(int(rj.progress or 0), 1) + rj.started_at = now + db.add(rj) + to_start.append(("render", str(rj.id))) + started_render += 1 + continue + + sj = ( + db.query(SubtitleJob) + .filter(SubtitleJob.task_id == tid, SubtitleJob.status == SubtitleJobStatus.queued) + .order_by(SubtitleJob.created_at.asc()) + .with_for_update(skip_locked=True) + .first() + ) + if sj: + sj.status = SubtitleJobStatus.running + sj.progress = max(int(sj.progress or 0), 1) + db.add(sj) + to_start.append(("subtitle", str(sj.id))) + started_subtitle += 1 + + # Phase 2: lock and start new tasks up to max_concurrency. + running_tasks = len(locked_tasks) + capacity = max(0, max_conc - running_tasks) + for _ in range(capacity): + # Prefer queued render jobs (rare, usually after an expired lock) so tasks can finish. + rj = ( + db.query(RenderJob) + .join(Task, Task.id == RenderJob.task_id) + .filter( + RenderJob.status == RenderJobStatus.queued, + or_( + Task.lock_owner != TASK_QUEUE_LOCK_OWNER, + Task.lock_until.is_(None), + Task.lock_until <= now, + ), + ) + .order_by(RenderJob.created_at.asc()) + .with_for_update(skip_locked=True) + .first() + ) + if rj: + task = db.query(Task).filter(Task.id == rj.task_id).with_for_update(skip_locked=True).first() + if not task: + continue + if _task_queue_is_task_locked(task, now): + continue + if task.lock_until and task.lock_until > now and task.lock_owner and task.lock_owner != TASK_QUEUE_LOCK_OWNER: + continue + task.lock_owner = TASK_QUEUE_LOCK_OWNER + task.lock_until = _task_queue_expires_at(now) + rj.status = RenderJobStatus.running + rj.progress = max(int(rj.progress or 0), 1) + rj.started_at = now + db.add(task) + db.add(rj) + to_start.append(("render", str(rj.id))) + started_render += 1 + running_tasks += 1 + continue + + sj = ( + db.query(SubtitleJob) + .join(Task, Task.id == SubtitleJob.task_id) + .filter( + SubtitleJob.status == SubtitleJobStatus.queued, + or_( + Task.lock_owner != TASK_QUEUE_LOCK_OWNER, + Task.lock_until.is_(None), + Task.lock_until <= now, + ), + ) + .order_by(SubtitleJob.created_at.asc()) + .with_for_update(skip_locked=True) + .first() + ) + if not sj: break - j.status = RenderJobStatus.running - j.progress = max(int(j.progress or 0), 1) - j.started_at = _now() - db.add(j) - db.commit() - celery_app.send_task("subtitle_service.process_render_job", args=[str(j.id)], queue="subtitle") - started += 1 + task = db.query(Task).filter(Task.id == sj.task_id).with_for_update(skip_locked=True).first() + if not task: + continue + if _task_queue_is_task_locked(task, now): + continue + if task.lock_until and task.lock_until > now and task.lock_owner and task.lock_owner != TASK_QUEUE_LOCK_OWNER: + continue - return {"status": "ok", "max_concurrency": str(max_conc), "started": str(started)} + task.lock_owner = TASK_QUEUE_LOCK_OWNER + task.lock_until = _task_queue_expires_at(now) + sj.status = SubtitleJobStatus.running + sj.progress = max(int(sj.progress or 0), 1) + db.add(task) + db.add(sj) + to_start.append(("subtitle", str(sj.id))) + started_subtitle += 1 + running_tasks += 1 + + db.commit() + except Exception: + try: + db.rollback() + except Exception: + pass + raise finally: db.close() + for kind, jid in to_start: + if kind == "subtitle": + celery_app.send_task("subtitle_service.process_job", args=[jid], queue="subtitle") + else: + celery_app.send_task("subtitle_service.process_render_job", args=[jid], queue="subtitle") + + return { + "status": "ok", + "max_concurrency": str(max_conc), + "started_subtitle": str(started_subtitle), + "started_render": str(started_render), + "recovered_subtitle": str(recovered_subtitle), + "recovered_render": str(recovered_render), + "unlocked_expired": str(unlocked_expired), + } + + +@celery_app.task(name="subtitle_service.render_queue_tick") +def render_queue_tick() -> dict[str, Any]: + # Legacy alias. + return task_queue_tick() + @celery_app.task(name="subtitle_service.process_render_job") def process_render_job(render_job_id: str) -> dict[str, Any]: @@ -1026,6 +1395,7 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: db = _db() log_path: Path | None = None log_key: str | None = None + hb: _TaskQueueHeartbeat | None = None try: rj = db.get(RenderJob, rid) if not rj: @@ -1036,15 +1406,46 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: if rj.status == RenderJobStatus.canceled: return {"status": "skipped", "detail": "canceled"} + task = db.get(Task, rj.task_id) + if not task: + return {"status": "error", "detail": "task not found"} + + now = _now() + if task.lock_owner != TASK_QUEUE_LOCK_OWNER: + # Not claimed by the task-level scheduler; put it back and wait. + if rj.status != RenderJobStatus.succeeded: + rj.status = RenderJobStatus.queued + rj.started_at = None + rj.progress = 0 + db.add(rj) + db.commit() + celery_app.send_task( + "subtitle_service.task_queue_tick", + args=[], + queue="subtitle", + countdown=_TASK_QUEUE_REQUEUE_COUNTDOWN_SECONDS, + ) + return {"status": "queued", "detail": "waiting for task queue"} + if task.lock_until is None or task.lock_until <= now: + task.lock_until = _task_queue_expires_at(now) + db.add(task) + db.commit() + + hb = _TaskQueueHeartbeat(task.id) + hb.start() + # Best-effort: if called directly, try to claim it. if rj.status == RenderJobStatus.queued: rj.status = RenderJobStatus.running rj.started_at = _now() + if rj.status == RenderJobStatus.running and rj.started_at is None: + rj.started_at = _now() if rj.status != RenderJobStatus.running: return {"status": "skipped", "detail": f"unexpected status={rj.status.value}"} - rj.progress = max(int(rj.progress or 0), 1) + # Mark as claimed by a worker ASAP so the scheduler can detect orphaned jobs. + rj.progress = max(int(rj.progress or 0), 2) db.add(rj) db.commit() @@ -1067,10 +1468,6 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: if burn_in and not ass_key: raise ValueError("render job missing ass_key for burn_in") - task = db.get(Task, rj.task_id) - if not task: - raise RuntimeError("task not found") - work_root = Path(settings.work_dir) / "render" / str(rj.id) work_root.mkdir(parents=True, exist_ok=True) log_path = work_root / "job.log" @@ -1087,13 +1484,60 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: f"render job start: render_job_id={rj.id} task_id={task.id} burn_in={burn_in} soft_sub={soft_sub} codec={video_codec} preset={video_preset} crf={video_crf}", ) _safe_upload_log(store, log_path, log_key) + + last_live_upload_at = 0.0 + last_live_upload_size = -1 + last_db_heartbeat_at = 0.0 + + def _heartbeat_db(now: float) -> None: + nonlocal last_db_heartbeat_at + if now - last_db_heartbeat_at < 10.0: + return + try: + rj.updated_at = _now() + db.add(rj) + db.commit() + last_db_heartbeat_at = now + except Exception: + try: + db.rollback() + except Exception: + pass + + def _live_upload_log() -> None: + nonlocal last_live_upload_at, last_live_upload_size + if log_path is None or not log_key: + return + try: + now = time.monotonic() + _heartbeat_db(now) + if now - last_live_upload_at < 2.0: + return + size = log_path.stat().st_size if log_path.exists() else 0 + if last_live_upload_size >= 0 and size - last_live_upload_size < 4096 and now - last_live_upload_at < 10.0: + return + _safe_upload_log(store, log_path, log_key) + last_live_upload_at = now + last_live_upload_size = size + except Exception: + pass + video_path = work_root / "input.mp4" srt_path = work_root / "subtitle_zh.srt" ass_path = work_root / "subtitle_zh.ass" + _safe_append_log_line(log_path, f"download: input_key={input_key}") + _safe_upload_log(store, log_path, log_key) + rj.progress = max(int(rj.progress or 0), 5) + db.add(rj) + db.commit() store.download_file(input_key, video_path) + _safe_append_log_line(log_path, f"download: srt_key={srt_key}") + _safe_upload_log(store, log_path, log_key) store.download_file(srt_key, srt_path) if burn_in and ass_key: + _safe_append_log_line(log_path, f"download: ass_key={ass_key}") + _safe_upload_log(store, log_path, log_key) store.download_file(ass_key, ass_path) _safe_append_log_line(log_path, "inputs downloaded") _safe_upload_log(store, log_path, log_key) @@ -1128,16 +1572,18 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: preset=video_preset, crf=video_crf, log_path=log_path, + live_upload_cb=_live_upload_log, ) _safe_upload_log(store, log_path, log_key) - final_key = f"final/{task.id}/video_burnin.mp4" + final_sha = sha256_file(out_video) + final_key = f"final/{task.id}/video_burnin_{final_sha[:16]}.mp4" store.upload_file(out_video, final_key, content_type="video/mp4") db.add( Asset( task_id=task.id, kind=AssetKind.video_final, storage_key=final_key, - sha256=sha256_file(out_video), + sha256=final_sha, size_bytes=out_video.stat().st_size, ) ) @@ -1152,16 +1598,17 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: out_video = work_root / "video_softsub.mkv" _safe_append_log_line(log_path, "ffmpeg: mux soft subtitles") - mux_soft_sub(settings.ffmpeg_path, video_path, srt_path, out_video, log_path=log_path) + mux_soft_sub(settings.ffmpeg_path, video_path, srt_path, out_video, log_path=log_path, live_upload_cb=_live_upload_log) _safe_upload_log(store, log_path, log_key) - final_key = f"final/{task.id}/video_softsub.mkv" + final_sha = sha256_file(out_video) + final_key = f"final/{task.id}/video_softsub_{final_sha[:16]}.mkv" store.upload_file(out_video, final_key, content_type="video/x-matroska") db.add( Asset( task_id=task.id, kind=AssetKind.video_final, storage_key=final_key, - sha256=sha256_file(out_video), + sha256=final_sha, size_bytes=out_video.stat().st_size, ) ) @@ -1184,12 +1631,17 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: _safe_append_log_line(log_path, "render job done") _safe_upload_log(store, log_path, log_key) + if task.lock_owner == TASK_QUEUE_LOCK_OWNER: + _task_queue_unlock(task) + db.add(task) + db.commit() + # Trigger optional after_render actions (e.g. auto publish). after_render = req.get("after_render") if isinstance(req, dict) else None if isinstance(after_render, dict) and after_render.get("publish"): celery_app.send_task("subtitle_service.after_render_publish", args=[str(rj.id)], queue="subtitle") - celery_app.send_task("subtitle_service.render_queue_tick", args=[], queue="subtitle") + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") return {"status": "ok"} except Exception as e: rj = db.get(RenderJob, rid) @@ -1199,10 +1651,13 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: rj.finished_at = _now() db.add(rj) task = db.get(Task, rj.task_id) - if task and task.status not in {TaskStatus.published, TaskStatus.canceled}: - task.status = TaskStatus.failed - task.error_code = task.error_code or "RENDER_FAILED" - task.error_message = str(e) + if task: + if task.status not in {TaskStatus.published, TaskStatus.canceled}: + task.status = TaskStatus.failed + task.error_code = task.error_code or "RENDER_FAILED" + task.error_message = str(e) + if task.lock_owner == TASK_QUEUE_LOCK_OWNER: + _task_queue_unlock(task) db.add(task) if rj and rj.subtitle_job_id: sj = db.get(SubtitleJob, rj.subtitle_job_id) @@ -1214,9 +1669,11 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: _safe_append_log_line(log_path, f"ERROR: {type(e).__name__}: {e}") _safe_append_log_block(log_path, traceback.format_exc()) _safe_upload_log(store, log_path, log_key) - celery_app.send_task("subtitle_service.render_queue_tick", args=[], queue="subtitle") + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") return {"status": "error", "detail": str(e)} finally: + if hb is not None: + hb.stop() db.close() @@ -1224,7 +1681,7 @@ def process_render_job(render_job_id: str) -> dict[str, Any]: def after_render_publish(render_job_id: str) -> dict[str, Any]: _ensure_db() db = _db() - orch_base = "http://localhost:8000" + orch_base = str(settings.orchestrator_url or "").strip().rstrip("/") or "http://localhost:8000" try: rid = uuid.UUID(render_job_id) rj = db.get(RenderJob, rid) @@ -1250,7 +1707,7 @@ def after_render_publish(render_job_id: str) -> dict[str, Any]: if publish_payload.get("video_key") in {"", None}: publish_payload["video_key"] = None - with httpx.Client(timeout=30.0) as client: + with httpx.Client(timeout=30.0, headers=_ORCH_INTERNAL_HEADERS) as client: resp = client.post(f"{orch_base}/tasks/{task.id}/actions/publish", json=publish_payload) resp.raise_for_status() @@ -1313,9 +1770,14 @@ def cleanup_task(task_id: str) -> dict[str, Any]: if not any(a.kind == AssetKind.video_final for a in assets): # Safety: if there's no final video, keep the latest raw video asset (if any) to avoid deleting the only video. - raw_assets = [a for a in assets if a.kind == AssetKind.video_raw] - if raw_assets: - keep_keys.add(raw_assets[-1].storage_key) + latest_raw = ( + db.query(Asset) + .filter(Asset.task_id == tid, Asset.kind == AssetKind.video_raw) + .order_by(Asset.created_at.desc(), Asset.id.desc()) + .first() + ) + if latest_raw: + keep_keys.add(latest_raw.storage_key) deleted_keys: list[str] = [] for a in assets: @@ -1380,9 +1842,11 @@ def auto_youtube_pipeline(task_id: str) -> dict[str, str]: store = S3Store(settings) store.ensure_bucket() - orch_base = "http://localhost:8000" + orch_base = str(settings.orchestrator_url or "").strip().rstrip("/") or "http://localhost:8000" db = _db() + hb: _TaskQueueHeartbeat | None = None + acquired_lock = False try: tid = uuid.UUID(task_id) task = db.get(Task, tid) @@ -1391,13 +1855,106 @@ def auto_youtube_pipeline(task_id: str) -> dict[str, str]: if task.source_type.value != "youtube": raise RuntimeError("task is not a youtube source") + now = _now() + if not _task_queue_is_task_locked(task, now): + # Claim a task slot before doing anything heavy (download/ASR/render). + _task_queue_lock_settings_row(db) + cfg = get_task_queue_settings(db) + try: + max_conc = int(cfg.get("max_concurrency", 1)) + except Exception: + max_conc = 1 + if max_conc < 0: + max_conc = 0 + if max_conc == 0: + celery_app.send_task( + "subtitle_service.auto_youtube_pipeline", + args=[str(tid)], + queue="subtitle", + countdown=_TASK_QUEUE_REQUEUE_COUNTDOWN_SECONDS, + ) + return {"status": "queued", "task_id": str(tid), "detail": "task queue paused"} + + running = ( + db.query(Task) + .filter(Task.lock_owner == TASK_QUEUE_LOCK_OWNER, Task.lock_until.is_not(None), Task.lock_until > now) + .count() + ) + if int(running or 0) >= int(max_conc): + celery_app.send_task( + "subtitle_service.auto_youtube_pipeline", + args=[str(tid)], + queue="subtitle", + countdown=_TASK_QUEUE_REQUEUE_COUNTDOWN_SECONDS, + ) + return {"status": "queued", "task_id": str(tid), "detail": "waiting for task queue"} + + task.lock_owner = TASK_QUEUE_LOCK_OWNER + task.lock_until = _task_queue_expires_at(now) + db.add(task) + db.commit() + acquired_lock = True + elif task.lock_until is None or task.lock_until <= now: + # Refresh a stale/expired lock to avoid accidental eviction mid-pipeline. + task.lock_until = _task_queue_expires_at(now) + db.add(task) + db.commit() + + hb = _TaskQueueHeartbeat(task.id) + hb.start() + profile = get_auto_profile(db) # Download YouTube video + cover + metadata (idempotent). - with httpx.Client(timeout=None) as client: - resp = client.post(f"{orch_base}/tasks/{tid}/actions/youtube_download") - resp.raise_for_status() - yt = resp.json() if resp.content else {} + yt: dict[str, Any] = {} + yt_attempt = 0 + yt_max_retries = 2 + while True: + try: + timeout_seconds = float(getattr(settings, "orchestrator_timeout_seconds", 1800.0) or 1800.0) + with httpx.Client(timeout=httpx.Timeout(timeout_seconds, connect=10.0), headers=_ORCH_INTERNAL_HEADERS) as client: + resp = client.post(f"{orch_base}/tasks/{tid}/actions/youtube_download") + resp.raise_for_status() + yt = resp.json() if resp.content else {} + break + except httpx.HTTPStatusError as e: + status_code = int(getattr(e.response, "status_code", 0) or 0) + if status_code in {429, 500, 502, 503, 504} and yt_attempt < yt_max_retries: + yt_attempt += 1 + try: + task.retry_count = int(task.retry_count or 0) + 1 + msg = (e.response.text or "").strip() + if len(msg) > 300: + msg = msg[:299] + "…" + task.error_message = f"youtube_download failed; retrying ({yt_attempt}/{yt_max_retries}): {status_code} {msg}".strip() + db.add(task) + db.commit() + except Exception: + db.rollback() + time.sleep(min(30.0, float(3 * (2**yt_attempt)))) + continue + raise + except httpx.HTTPError as e: + if yt_attempt < yt_max_retries: + yt_attempt += 1 + try: + task.retry_count = int(task.retry_count or 0) + 1 + task.error_message = f"youtube_download request failed; retrying ({yt_attempt}/{yt_max_retries}): {type(e).__name__}: {e}" + db.add(task) + db.commit() + except Exception: + db.rollback() + time.sleep(min(30.0, float(3 * (2**yt_attempt)))) + continue + raise + + if yt_attempt: + try: + task.error_message = None + db.add(task) + db.commit() + except Exception: + db.rollback() yt_meta = yt.get("metadata") if isinstance(yt, dict) else {} if not isinstance(yt_meta, dict): @@ -1512,13 +2069,21 @@ def auto_youtube_pipeline(task_id: str) -> dict[str, str]: meta["copyright"] = 1 meta["source"] = "" + # Persist publish meta as a file so it can be edited before publishing. + publish_meta_key = f"meta/{tid}/publish_meta.json" + store.put_bytes( + json.dumps(meta, ensure_ascii=False, indent=2).encode("utf-8"), + publish_meta_key, + content_type="application/json", + ) + # Let orchestrator pick the latest rendered asset when publishing. publish_payload = { "account_id": None, "video_key": None, "cover_key": cover_key, "typeid_mode": profile.get("publish_typeid_mode") or "ai_summary", - "meta": meta, + "meta": None, } req["after_render"] = {"publish": True, "publish_payload": publish_payload} @@ -1526,7 +2091,7 @@ def auto_youtube_pipeline(task_id: str) -> dict[str, str]: db.add(job) db.commit() - celery_app.send_task("subtitle_service.process_job", args=[str(job.id)], queue="subtitle") + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") return {"status": "ok", "task_id": str(tid), "detail": f"queued subtitle job {job.id}"} if profile.get("auto_publish"): @@ -1567,15 +2132,22 @@ def auto_youtube_pipeline(task_id: str) -> dict[str, str]: meta["copyright"] = 1 meta["source"] = "" + publish_meta_key = f"meta/{tid}/publish_meta.json" + store.put_bytes( + json.dumps(meta, ensure_ascii=False, indent=2).encode("utf-8"), + publish_meta_key, + content_type="application/json", + ) + publish_payload = { "account_id": None, "video_key": final_asset.storage_key, "cover_key": cover_key, "typeid_mode": profile.get("publish_typeid_mode") or "ai_summary", - "meta": meta, + "meta": None, } - with httpx.Client(timeout=30.0) as client: + with httpx.Client(timeout=30.0, headers=_ORCH_INTERNAL_HEADERS) as client: resp = client.post(f"{orch_base}/tasks/{tid}/actions/publish", json=publish_payload) resp.raise_for_status() @@ -1587,6 +2159,35 @@ def auto_youtube_pipeline(task_id: str) -> dict[str, str]: task.error_message = str(e) db.add(task) db.commit() + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") raise finally: + if hb is not None: + hb.stop() + try: + if acquired_lock: + tid2 = uuid.UUID(task_id) + task2 = db.get(Task, tid2) + if task2 and task2.lock_owner == TASK_QUEUE_LOCK_OWNER: + inflight = ( + db.query(SubtitleJob) + .filter( + SubtitleJob.task_id == tid2, + SubtitleJob.status.in_([SubtitleJobStatus.queued, SubtitleJobStatus.running]), + ) + .count() + + db.query(RenderJob) + .filter(RenderJob.task_id == tid2, RenderJob.status.in_([RenderJobStatus.queued, RenderJobStatus.running])) + .count() + ) + if not inflight: + _task_queue_unlock(task2) + db.add(task2) + db.commit() + celery_app.send_task("subtitle_service.task_queue_tick", args=[], queue="subtitle") + except Exception: + try: + db.rollback() + except Exception: + pass db.close() diff --git a/src/videoroll/apps/youtube_ingest/main.py b/src/videoroll/apps/youtube_ingest/main.py index da6a987..cb277a9 100644 --- a/src/videoroll/apps/youtube_ingest/main.py +++ b/src/videoroll/apps/youtube_ingest/main.py @@ -7,10 +7,12 @@ from urllib.parse import parse_qs, urlparse from fastapi import Depends, FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from videoroll.config import YouTubeIngestSettings, get_youtube_ingest_settings from videoroll.db.base import Base +from videoroll.db.auto_migrate import auto_migrate from videoroll.db.models import ( IngestedVideo, SourceType, @@ -60,6 +62,7 @@ def _startup() -> None: settings = get_youtube_ingest_settings() engine = get_engine(settings.database_url) Base.metadata.create_all(engine) + auto_migrate(settings.database_url) S3Store(settings).ensure_bucket() @@ -120,13 +123,25 @@ def ingest_single(payload: YouTubeIngestRequest, db: Session = Depends(get_db)) status=TaskStatus.ingested, ) db.add(task) - db.commit() - db.refresh(task) - if video_id: + db.flush() db.add(IngestedVideo(platform="youtube", source_id=video_id, task_id=task.id)) + try: db.commit() + except IntegrityError as e: + # Concurrency-safe dedupe: another request inserted the same IngestedVideo. + db.rollback() + if video_id: + existing = ( + db.query(IngestedVideo) + .filter(IngestedVideo.platform == "youtube", IngestedVideo.source_id == video_id) + .first() + ) + if existing: + return YouTubeIngestResponse(task_id=existing.task_id, deduped=True, source_id=video_id) + raise HTTPException(status_code=500, detail=f"ingest failed: {e}") from e + db.refresh(task) return YouTubeIngestResponse(task_id=task.id, deduped=False, source_id=video_id) @@ -175,10 +190,23 @@ def scan_source( status=TaskStatus.ingested, ) db.add(task) - db.commit() - db.refresh(task) + db.flush() db.add(IngestedVideo(platform="youtube", source_id=e.video_id, task_id=task.id, published_at=e.published_at)) - db.commit() + try: + db.commit() + except IntegrityError: + # Another worker/request beat us to it. + db.rollback() + if ( + db.query(IngestedVideo) + .filter(IngestedVideo.platform == "youtube", IngestedVideo.source_id == e.video_id) + .first() + is not None + ): + skipped += 1 + continue + raise + db.refresh(task) created.append(task.id) if payload.auto_process: diff --git a/src/videoroll/apps/youtube_meta_store.py b/src/videoroll/apps/youtube_meta_store.py new file mode 100644 index 0000000..68f6e6d --- /dev/null +++ b/src/videoroll/apps/youtube_meta_store.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import uuid +from typing import Optional + +from sqlalchemy.orm import Session + +from videoroll.db.models import YouTubeVideoMeta + + +def _as_str(v: object) -> str: + return str(v or "").strip() + + +def get_task_youtube_meta(db: Session, task_id: uuid.UUID) -> Optional[YouTubeVideoMeta]: + return db.get(YouTubeVideoMeta, task_id) + + +def upsert_task_youtube_meta( + db: Session, + task_id: uuid.UUID, + *, + source_id: str | None = None, + title: str, + description: str, + webpage_url: str, + uploader: str | None = None, + upload_date: str | None = None, + duration: int | None = None, +) -> YouTubeVideoMeta: + row = db.get(YouTubeVideoMeta, task_id) + if not row: + row = YouTubeVideoMeta(task_id=task_id) + + row.source_id = _as_str(source_id) or None + row.title = _as_str(title) + row.description = str(description or "") + row.webpage_url = _as_str(webpage_url) + row.uploader = _as_str(uploader) or None + row.upload_date = _as_str(upload_date) or None + row.duration = int(duration) if duration is not None else None + + db.add(row) + return row + + +def delete_task_youtube_meta(db: Session, task_id: uuid.UUID) -> int: + return db.query(YouTubeVideoMeta).filter(YouTubeVideoMeta.task_id == task_id).delete(synchronize_session=False) + diff --git a/src/videoroll/apps/youtube_settings_store.py b/src/videoroll/apps/youtube_settings_store.py index 1c94994..cfe16ed 100644 --- a/src/videoroll/apps/youtube_settings_store.py +++ b/src/videoroll/apps/youtube_settings_store.py @@ -1,15 +1,43 @@ from __future__ import annotations +import io +import http.cookiejar +from datetime import datetime, timezone from typing import Any, Optional from sqlalchemy.orm import Session from videoroll.db.models import AppSetting +from videoroll.utils.fernet import decrypt_str, encrypt_str YOUTUBE_SETTINGS_KEY = "youtube.settings" _MAX_PROXY_LEN = 2048 +_MAX_COOKIES_LEN = 200_000 +_AUTH_COOKIE_NAMES = { + # Common Google/YouTube auth cookies. Presence usually means "logged in". + "SID", + "HSID", + "SSID", + "APISID", + "SAPISID", + "__Secure-1PSID", + "__Secure-3PSID", + "__Secure-1PAPISID", + "__Secure-3PAPISID", + "LOGIN_INFO", +} +_BOT_CHECK_BYPASS_COOKIE_NAMES = { + # Typically set after solving Google's "unusual traffic" CAPTCHA. + "GOOGLE_ABUSE_EXEMPTION", +} +_NETSCAPE_HEADER_TEXT = ( + "# Netscape HTTP Cookie File\n" + "# http://curl.haxx.se/rfc/cookie_spec.html\n" + "# This is a generated file! Do not edit.\n" + "\n" +) def _as_dict(v: Any) -> dict[str, Any]: @@ -27,6 +55,191 @@ def _get_row(db: Session) -> AppSetting: return row +def _decrypt_opt(token: Any) -> str: + if not isinstance(token, str): + return "" + token = token.strip() + if not token: + return "" + try: + return decrypt_str(token).strip() + except Exception: + return "" + + +def get_youtube_cookies_txt(db: Session) -> str: + row = db.get(AppSetting, YOUTUBE_SETTINGS_KEY) + stored = dict(_as_dict(row.value_json)) if row else {} + return _decrypt_opt(stored.get("cookies_txt_enc")) + + +def normalize_and_validate_netscape_cookies_txt(cookies_txt: str) -> str: + raw = str(cookies_txt or "") + raw = raw.lstrip("\ufeff").replace("\r\n", "\n").replace("\r", "\n") + raw_stripped = raw.strip() + if not raw_stripped: + return "" + if raw_stripped.lower().startswith("cookie:"): + raise ValueError("cookies_txt looks like a Cookie request header; please export cookies.txt (Netscape format)") + if raw_stripped.startswith("{") or raw_stripped.startswith("["): + raise ValueError("cookies_txt looks like JSON; please export cookies.txt (Netscape format)") + if "\t" not in raw_stripped and "\n" not in raw_stripped and ";" in raw_stripped and "=" in raw_stripped: + raise ValueError("cookies_txt looks like a Cookie header string; please export cookies.txt (Netscape format)") + + lines_in = raw.split("\n") + # Drop leading blank lines so we can normalize the header deterministically. + while lines_in and not str(lines_in[0] or "").strip(): + lines_in.pop(0) + # Drop an existing header block if present. + if lines_in and (lines_in[0].startswith("# Netscape HTTP Cookie File") or lines_in[0].startswith("# HTTP Cookie File")): + lines_in.pop(0) + while lines_in: + line0 = str(lines_in[0] or "") + if not line0.strip(): + lines_in.pop(0) + break + if line0.lstrip().startswith("#") and not line0.lstrip().startswith("#HttpOnly_"): + lines_in.pop(0) + continue + break + lines_out: list[str] = [] + for line in lines_in: + if not line: + lines_out.append("") + continue + + stripped = line.strip() + if not stripped: + lines_out.append("") + continue + + # Keep normal comments as-is. "#HttpOnly_" lines are data lines. + if stripped.startswith("#") and not stripped.startswith("#HttpOnly_"): + lines_out.append(stripped) + continue + + data_line = stripped + httponly = False + if data_line.startswith("#HttpOnly_"): + httponly = True + data_line = data_line[len("#HttpOnly_") :] + + if "\t" not in data_line: + parts = data_line.split() + if len(parts) == 7: + data_line = "\t".join(parts) + + if "\t" in data_line: + parts = data_line.split("\t") + if len(parts) == 7: + domain = str(parts[0] or "").strip() + include_subdomains = str(parts[1] or "").strip().upper() + if include_subdomains not in {"TRUE", "FALSE"}: + include_subdomains = "TRUE" if domain.startswith(".") else "FALSE" + + # Some exporters omit the leading dot for domain cookies even when include_subdomains is TRUE. + # MozillaCookieJar expects a leading dot for such cookies. Prefer widening rather than shrinking + # cookie scope to avoid breaking auth cookies unexpectedly. + if include_subdomains == "TRUE" and domain and not domain.startswith(".") and "." in domain: + domain = "." + domain + if include_subdomains == "FALSE" and domain.startswith("."): + include_subdomains = "TRUE" + + parts[0] = domain + parts[1] = include_subdomains + data_line = "\t".join(parts) + + if httponly: + data_line = "#HttpOnly_" + data_line + lines_out.append(data_line) + + normalized = _NETSCAPE_HEADER_TEXT + ("\n".join(lines_out).rstrip() + "\n") + try: + jar = http.cookiejar.MozillaCookieJar() + jar._really_load(io.StringIO(normalized), "", ignore_discard=True, ignore_expires=True) + except http.cookiejar.LoadError as e: + msg = str(e) + if "does not look like a Netscape format cookies file" in msg: + raise ValueError("invalid cookies.txt: missing header '# Netscape HTTP Cookie File'") from e + raise ValueError(f"invalid cookies.txt: {msg}") from e + except Exception as e: + raise ValueError(f"invalid cookies.txt: {type(e).__name__}: {e}") from e + + return normalized + + +def summarize_netscape_cookies_txt(cookies_txt: str) -> dict[str, Any]: + """ + Returns a lightweight summary for UI/debugging without exposing cookie values. + """ + raw = str(cookies_txt or "").strip() + if not raw: + return { + "cookies_count": 0, + "cookies_domains_count": 0, + "cookies_has_auth": False, + "cookies_has_bot_check_bypass": False, + "cookies_has_visitor_info": False, + } + + try: + normalized = normalize_and_validate_netscape_cookies_txt(raw) + except Exception: + return { + "cookies_count": 0, + "cookies_domains_count": 0, + "cookies_has_auth": False, + "cookies_has_bot_check_bypass": False, + "cookies_has_visitor_info": False, + } + domains: set[str] = set() + names: set[str] = set() + count = 0 + + for line in normalized.split("\n"): + if not line: + continue + if line.startswith("#") and not line.startswith("#HttpOnly_"): + continue + if line.startswith("#HttpOnly_"): + line = line[len("#HttpOnly_") :] + line = line.strip() + if not line: + continue + + parts = line.split("\t") if "\t" in line else line.split() + if len(parts) != 7: + continue + + domain = str(parts[0] or "").strip().lstrip(".").lower() + name = str(parts[5] or "").strip() + if domain: + domains.add(domain) + if name: + names.add(name) + count += 1 + + has_auth = bool(names & _AUTH_COOKIE_NAMES) + has_bot_check_bypass = bool(names & _BOT_CHECK_BYPASS_COOKIE_NAMES) + has_visitor_info = "VISITOR_INFO1_LIVE" in names + + return { + "cookies_count": count, + "cookies_domains_count": len(domains), + "cookies_has_auth": has_auth, + "cookies_has_bot_check_bypass": has_bot_check_bypass, + "cookies_has_visitor_info": has_visitor_info, + } + + +def _cookies_enabled(stored: dict[str, Any], cookies_txt: str) -> bool: + raw = stored.get("cookies_enabled") + if isinstance(raw, bool): + return raw + # Backward-compatible default: if cookies are set, consider them enabled. + return bool(cookies_txt) + + def get_youtube_settings(db: Session, *, default_proxy: Optional[str] = None) -> dict[str, Any]: row = db.get(AppSetting, YOUTUBE_SETTINGS_KEY) stored = dict(_as_dict(row.value_json)) if row else {} @@ -40,7 +253,16 @@ def get_youtube_settings(db: Session, *, default_proxy: Optional[str] = None) -> if len(proxy_str) > _MAX_PROXY_LEN: proxy_str = proxy_str[:_MAX_PROXY_LEN] - return {"proxy": proxy_str} + cookies_txt = _decrypt_opt(stored.get("cookies_txt_enc")) + summary = summarize_netscape_cookies_txt(cookies_txt) if cookies_txt else summarize_netscape_cookies_txt("") + cookies_updated_at = str(stored.get("cookies_updated_at") or "").strip() or None + return { + "proxy": proxy_str, + "cookies_set": bool(cookies_txt), + "cookies_enabled": _cookies_enabled(stored, cookies_txt), + "cookies_updated_at": cookies_updated_at, + **summary, + } def update_youtube_settings(db: Session, update: dict[str, Any], *, default_proxy: Optional[str] = None) -> dict[str, Any]: @@ -53,9 +275,46 @@ def update_youtube_settings(db: Session, update: dict[str, Any], *, default_prox raise ValueError(f"proxy is too long (max {_MAX_PROXY_LEN} chars)") stored["proxy"] = proxy + if "cookies_txt" in update: + cookies_txt = update.get("cookies_txt") + if cookies_txt is None: + pass + else: + cookies_txt = str(cookies_txt).lstrip("\ufeff") + if not cookies_txt.strip(): + stored.pop("cookies_txt_enc", None) + stored["cookies_enabled"] = False + stored.pop("cookies_updated_at", None) + else: + normalized = normalize_and_validate_netscape_cookies_txt(cookies_txt) + if len(normalized) > _MAX_COOKIES_LEN: + raise ValueError(f"cookies_txt is too long (max {_MAX_COOKIES_LEN} chars)") + stored["cookies_txt_enc"] = encrypt_str(normalized) + stored["cookies_updated_at"] = datetime.now(timezone.utc).isoformat() + if "cookies_enabled" not in update: + stored["cookies_enabled"] = True + + if "cookies_enabled" in update: + enabled_raw = update.get("cookies_enabled") + if enabled_raw is None: + pass + else: + enabled = bool(enabled_raw) + if enabled: + cookies_txt = _decrypt_opt(stored.get("cookies_txt_enc")) + if not cookies_txt: + raise ValueError("cookies_txt is not set; paste cookies.txt first") + # Ensure it still validates (also normalizes legacy stored cookies). + normalized = normalize_and_validate_netscape_cookies_txt(cookies_txt) + if len(normalized) > _MAX_COOKIES_LEN: + raise ValueError(f"cookies_txt is too long (max {_MAX_COOKIES_LEN} chars)") + stored["cookies_txt_enc"] = encrypt_str(normalized) + stored["cookies_enabled"] = True + else: + stored["cookies_enabled"] = False + row.value_json = stored db.add(row) db.commit() return get_youtube_settings(db, default_proxy=default_proxy) - diff --git a/src/videoroll/config.py b/src/videoroll/config.py index 2afd5ae..1ed5b6a 100644 --- a/src/videoroll/config.py +++ b/src/videoroll/config.py @@ -5,6 +5,12 @@ from functools import lru_cache from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict +DEFAULT_YOUTUBE_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/122.0.0.0 Safari/537.36" +) + class CommonSettings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", extra="ignore") @@ -30,13 +36,9 @@ class OrchestratorSettings(CommonSettings): ffmpeg_path: str = Field("ffmpeg", alias="FFMPEG_PATH") # YouTube downloader (yt-dlp) settings. - youtube_user_agent: str = Field("videoroll/0.1", alias="YOUTUBE_USER_AGENT") + youtube_user_agent: str = Field(DEFAULT_YOUTUBE_USER_AGENT, alias="YOUTUBE_USER_AGENT") youtube_cookie_file: str | None = Field(None, alias="YOUTUBE_COOKIE_FILE") youtube_proxy: str | None = Field(None, alias="YOUTUBE_PROXY") - youtube_ytdlp_format: str = Field( - "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best", - alias="YOUTUBE_YTDLP_FORMAT", - ) youtube_extractor_args_json: str | None = Field(None, alias="YOUTUBE_EXTRACTOR_ARGS_JSON") @@ -67,9 +69,13 @@ class SubtitleServiceSettings(CommonSettings): openai_temperature: float = Field(0.2, alias="OPENAI_TEMPERATURE") openai_timeout_seconds: float = Field(180.0, alias="OPENAI_TIMEOUT_SECONDS") + # Orchestrator API (used by subtitle worker for auto pipelines). + orchestrator_url: str = Field("http://localhost:8000", alias="ORCHESTRATOR_URL") + orchestrator_timeout_seconds: float = Field(1800.0, alias="ORCHESTRATOR_TIMEOUT_SECONDS") + class YouTubeIngestSettings(CommonSettings): - user_agent: str = Field("videoroll/0.1", alias="YOUTUBE_USER_AGENT") + user_agent: str = Field(DEFAULT_YOUTUBE_USER_AGENT, alias="YOUTUBE_USER_AGENT") youtube_proxy: str | None = Field(None, alias="YOUTUBE_PROXY") diff --git a/src/videoroll/db/auto_migrate.py b/src/videoroll/db/auto_migrate.py new file mode 100644 index 0000000..ed3f29b --- /dev/null +++ b/src/videoroll/db/auto_migrate.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import logging +from functools import lru_cache + +from sqlalchemy import inspect, text +from sqlalchemy.engine import Engine + +from videoroll.db.session import get_engine + + +logger = logging.getLogger(__name__) + + +def _is_duplicate_column_error(exc: Exception) -> bool: + """ + Best-effort detection across dialects/drivers. + + We want auto-migrations to be safe under concurrent startups where two + processes may race to add the same column. + """ + orig = getattr(exc, "orig", None) + sqlstate = getattr(orig, "sqlstate", None) or getattr(orig, "pgcode", None) + if sqlstate == "42701": # PostgreSQL duplicate_column + return True + + msg = str(exc).lower() + return ("duplicate column" in msg) or ("already exists" in msg and "column" in msg) + + +def _add_column(engine: Engine, table: str, column: str, column_type_sql: str) -> None: + stmt = f"ALTER TABLE {table} ADD COLUMN {column} {column_type_sql}" + with engine.begin() as conn: + try: + conn.execute(text(stmt)) + except Exception as e: + if _is_duplicate_column_error(e): + return + raise + + +def _ensure_tasks_lock_columns(engine: Engine) -> None: + insp = inspect(engine) + if "tasks" not in set(insp.get_table_names()): + return + + cols = {c.get("name") for c in insp.get_columns("tasks")} + dialect = (engine.dialect.name or "").lower() + + if "lock_owner" not in cols: + _add_column(engine, "tasks", "lock_owner", "VARCHAR(128)") + logger.warning("auto-migrated DB: added tasks.lock_owner") + + if "lock_until" not in cols: + ts_type = "TIMESTAMPTZ" if dialect == "postgresql" else "TIMESTAMP" + _add_column(engine, "tasks", "lock_until", ts_type) + logger.warning("auto-migrated DB: added tasks.lock_until") + + +def auto_migrate_engine(engine: Engine) -> None: + """ + Small, targeted "auto migration" for deployments without Alembic. + + Today it only ensures columns required by the task-level queue exist. + """ + _ensure_tasks_lock_columns(engine) + + +@lru_cache +def auto_migrate(database_url: str) -> None: + """ + Idempotent auto migration (cached per database_url). + """ + engine = get_engine(database_url) + auto_migrate_engine(engine) + diff --git a/src/videoroll/db/models.py b/src/videoroll/db/models.py index b30fcb0..8d2f9bc 100644 --- a/src/videoroll/db/models.py +++ b/src/videoroll/db/models.py @@ -320,3 +320,26 @@ class IngestedVideo(Base): UniqueConstraint("platform", "source_id", name="uq_ingested_videos_platform_source_id"), Index("ix_ingested_videos_task", "task_id"), ) + + +class YouTubeVideoMeta(Base): + __tablename__ = "youtube_video_meta" + + task_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("tasks.id", ondelete="CASCADE"), primary_key=True) + + source_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) # youtube videoId + + title: Mapped[str] = mapped_column(Text, nullable=False, default="") + description: Mapped[str] = mapped_column(Text, nullable=False, default="") + webpage_url: Mapped[str] = mapped_column(Text, nullable=False, default="") + + uploader: Mapped[Optional[str]] = mapped_column(String(256), nullable=True) + upload_date: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) # YYYYMMDD + duration: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()) + + __table_args__ = ( + Index("ix_youtube_video_meta_source_id", "source_id"), + ) diff --git a/src/videoroll/storage/s3.py b/src/videoroll/storage/s3.py index 2d7584b..e33ca73 100644 --- a/src/videoroll/storage/s3.py +++ b/src/videoroll/storage/s3.py @@ -30,7 +30,12 @@ class S3Store: aws_secret_access_key=settings.s3_secret_access_key, region_name=settings.s3_region_name, use_ssl=settings.s3_use_ssl, - config=Config(s3={"addressing_style": "path"}), + config=Config( + s3={"addressing_style": "path"}, + retries={"max_attempts": 10, "mode": "adaptive"}, + connect_timeout=10, + read_timeout=120, + ), ) @property diff --git a/src/videoroll/utils/internal_api_token.py b/src/videoroll/utils/internal_api_token.py new file mode 100644 index 0000000..fa5b7e0 --- /dev/null +++ b/src/videoroll/utils/internal_api_token.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import hashlib + + +_SALT = "videoroll-internal-token:v1:" + + +def internal_api_token(secret: str) -> str: + secret = str(secret or "").strip() + if not secret: + return "" + h = hashlib.sha256() + h.update((_SALT + secret).encode("utf-8")) + return h.hexdigest() + diff --git a/web/.dockerignore b/src/web/.dockerignore similarity index 100% rename from web/.dockerignore rename to src/web/.dockerignore diff --git a/web/Dockerfile b/src/web/Dockerfile similarity index 100% rename from web/Dockerfile rename to src/web/Dockerfile diff --git a/web/index.html b/src/web/index.html similarity index 100% rename from web/index.html rename to src/web/index.html diff --git a/web/nginx.conf b/src/web/nginx.conf similarity index 74% rename from web/nginx.conf rename to src/web/nginx.conf index fe477ba..a1950a3 100644 --- a/web/nginx.conf +++ b/src/web/nginx.conf @@ -24,18 +24,10 @@ server { proxy_send_timeout 3600; } - # Expose FastAPI docs on the same port (optional but convenient). - location = /openapi.json { - proxy_pass http://app:8000/openapi.json; - } - - location /docs { - proxy_pass http://app:8000/docs; - } - - location /redoc { - proxy_pass http://app:8000/redoc; - } + # Keep API docs under /api/* to avoid clashing with SPA routes. + location = /openapi.json { return 301 /api/openapi.json; } + location ^~ /docs { return 301 /api$request_uri; } + location ^~ /redoc { return 301 /api$request_uri; } root /usr/share/nginx/html; index index.html; diff --git a/web/package-lock.json b/src/web/package-lock.json similarity index 100% rename from web/package-lock.json rename to src/web/package-lock.json diff --git a/web/package.json b/src/web/package.json similarity index 100% rename from web/package.json rename to src/web/package.json diff --git a/web/postcss.config.js b/src/web/postcss.config.js similarity index 100% rename from web/postcss.config.js rename to src/web/postcss.config.js diff --git a/src/web/src/App.tsx b/src/web/src/App.tsx new file mode 100644 index 0000000..a6c682c --- /dev/null +++ b/src/web/src/App.tsx @@ -0,0 +1,130 @@ +import { useState } from "react"; +import { Link, NavLink, Route, Routes } from "react-router-dom"; +import AuthGate from "./components/AuthGate"; +import { fetchJson } from "./lib/http"; +import { ORCHESTRATOR_URL } from "./lib/urls"; +import DashboardPage from "./pages/DashboardPage"; +import TaskDetailPage from "./pages/TaskDetailPage"; +import TaskNewPage from "./pages/TaskNewPage"; +import TasksPage from "./pages/TasksPage"; +import VideosPage from "./pages/VideosPage"; +import YouTubeSourcesPage from "./pages/YouTubeSourcesPage"; +import SettingsASRPage from "./pages/SettingsASRPage"; +import SettingsYouTubePage from "./pages/SettingsYouTubePage"; +import SettingsStoragePage from "./pages/SettingsStoragePage"; +import SettingsTranslatePage from "./pages/SettingsTranslatePage"; +import SettingsBilibiliPage from "./pages/SettingsBilibiliPage"; +import SettingsAutoPage from "./pages/SettingsAutoPage"; +import RenderQueuePage from "./pages/RenderQueuePage"; + +function NavItem({ to, label }: { to: string; label: string }) { + return ( + + [ + "block rounded px-3 py-2 text-sm", + isActive ? "bg-slate-900 text-white" : "text-slate-700 hover:bg-slate-100", + ].join(" ") + } + > + {label} + + ); +} + +export default function App() { + const orchestratorDisplay = + ORCHESTRATOR_URL.startsWith("http://") || ORCHESTRATOR_URL.startsWith("https://") + ? ORCHESTRATOR_URL + : typeof window !== "undefined" + ? `${window.location.origin}${ORCHESTRATOR_URL}` + : ORCHESTRATOR_URL; + + const [loggingOut, setLoggingOut] = useState(false); + + async function logout() { + if (loggingOut) return; + setLoggingOut(true); + try { + await fetchJson(`${ORCHESTRATOR_URL}/auth/logout`, { method: "POST" }); + window.location.reload(); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + alert(`退出失败：${msg}`); + } finally { + setLoggingOut(false); + } + } + + return ( + +
+
+
+ + VideoRoll + +
+
合规版 · MVP
+ +
+
+
+ 仅用于处理你拥有版权/已获授权/允许再分发的内容。 +
+
+ +
+ + +
+ + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + +
+
+
+ + ); +} diff --git a/src/web/src/components/AuthGate.tsx b/src/web/src/components/AuthGate.tsx new file mode 100644 index 0000000..a9e40e5 --- /dev/null +++ b/src/web/src/components/AuthGate.tsx @@ -0,0 +1,189 @@ +import { PropsWithChildren, useEffect, useMemo, useState } from "react"; +import { fetchJson } from "../lib/http"; +import { ORCHESTRATOR_URL } from "../lib/urls"; + +type AuthStatus = { + password_set: boolean; + trusted: boolean; +}; + +function clampError(err: unknown): string { + const msg = err instanceof Error ? err.message : String(err ?? ""); + const s = msg.trim(); + if (!s) return "Unknown error"; + if (s.length > 400) return s.slice(0, 399) + "…"; + return s; +} + +async function getAuthStatus(): Promise { + return await fetchJson(`${ORCHESTRATOR_URL}/auth/status`); +} + +async function setupPassword(password: string): Promise { + return await fetchJson(`${ORCHESTRATOR_URL}/auth/setup`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ password }), + }); +} + +async function login(password: string): Promise { + return await fetchJson(`${ORCHESTRATOR_URL}/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ password }), + }); +} + +export default function AuthGate({ children }: PropsWithChildren) { + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + const [loadError, setLoadError] = useState(""); + + const passwordSet = Boolean(status?.password_set); + const trusted = Boolean(status?.trusted); + + const [password, setPassword] = useState(""); + const [password2, setPassword2] = useState(""); + const [submitError, setSubmitError] = useState(""); + const [submitting, setSubmitting] = useState(false); + + const mode = useMemo<"setup" | "login" | "app">(() => { + if (!status) return "login"; + if (!status.password_set) return "setup"; + if (!status.trusted) return "login"; + return "app"; + }, [status]); + + useEffect(() => { + let alive = true; + setLoading(true); + setLoadError(""); + getAuthStatus() + .then((s) => { + if (!alive) return; + setStatus(s); + setLoading(false); + }) + .catch((e) => { + if (!alive) return; + setLoadError(clampError(e)); + setLoading(false); + }); + return () => { + alive = false; + }; + }, []); + + async function submit() { + setSubmitError(""); + setSubmitting(true); + try { + if (mode === "setup") { + if (password.length < 8) throw new Error("Password too short (min 8 chars)"); + if (password !== password2) throw new Error("Passwords do not match"); + const s = await setupPassword(password); + setStatus(s); + } else { + const s = await login(password); + setStatus(s); + } + setPassword(""); + setPassword2(""); + } catch (e) { + setSubmitError(clampError(e)); + } finally { + setSubmitting(false); + } + } + + if (loading) { + return ( +
+
+
Loading…
+
+
+ ); + } + + if (loadError) { + return ( +
+
+
+
Failed to load
+
{loadError}
+
+
+
+ ); + } + + if (passwordSet && trusted) { + return <>{children}; + } + + const title = mode === "setup" ? "Set admin password" : "Admin login"; + const subtitle = + mode === "setup" + ? "首次使用/升级后需要设置管理密码。设置后当前设备将被记住。" + : "此设备未被信任，请输入管理密码继续。"; + + return ( +
+
+
+
{title}
+
{subtitle}
+ +
+ + setPassword(e.target.value)} + className="mt-1 w-full rounded border px-3 py-2 text-sm outline-none focus:border-slate-500" + placeholder="••••••••" + autoFocus + /> +
+ + {mode === "setup" ? ( +
+ + setPassword2(e.target.value)} + className="mt-1 w-full rounded border px-3 py-2 text-sm outline-none focus:border-slate-500" + placeholder="••••••••" + /> +
+ ) : null} + + {submitError ?
{submitError}
: null} + + + +
+ 提示：此登录是“设备记住”模式；新设备/清理浏览器数据后需要重新输入密码。 +
+
+
+
+ ); +} + diff --git a/web/src/components/StatusBadge.tsx b/src/web/src/components/StatusBadge.tsx similarity index 100% rename from web/src/components/StatusBadge.tsx rename to src/web/src/components/StatusBadge.tsx diff --git a/web/src/lib/http.ts b/src/web/src/lib/http.ts similarity index 79% rename from web/src/lib/http.ts rename to src/web/src/lib/http.ts index af73717..4cc1569 100644 --- a/web/src/lib/http.ts +++ b/src/web/src/lib/http.ts @@ -1,9 +1,8 @@ export async function fetchJson(url: string, init?: RequestInit): Promise { - const resp = await fetch(url, init); + const resp = await fetch(url, { credentials: "include", ...init }); if (!resp.ok) { const text = await resp.text().catch(() => ""); throw new Error(`${resp.status} ${resp.statusText}${text ? ` - ${text}` : ""}`); } return (await resp.json()) as T; } - diff --git a/web/src/lib/types.ts b/src/web/src/lib/types.ts similarity index 98% rename from web/src/lib/types.ts rename to src/web/src/lib/types.ts index 89fc791..773dcb4 100644 --- a/web/src/lib/types.ts +++ b/src/web/src/lib/types.ts @@ -26,6 +26,7 @@ export type Task = { status: TaskStatus; priority: number; created_by?: string | null; + display_title?: string | null; error_code?: string | null; error_message?: string | null; retry_count: number; diff --git a/web/src/lib/urls.ts b/src/web/src/lib/urls.ts similarity index 67% rename from web/src/lib/urls.ts rename to src/web/src/lib/urls.ts index 141d76f..9184b0e 100644 --- a/web/src/lib/urls.ts +++ b/src/web/src/lib/urls.ts @@ -4,14 +4,14 @@ function env(name: string): string | undefined { return v; } -function hostBase(port: number): string { - if (typeof window !== "undefined" && window.location?.hostname) { - return `http://${window.location.hostname}:${port}`; - } - return `http://localhost:${port}`; +function defaultOrchestratorUrl(): string { + // When served behind nginx (single-port mode), use same-origin proxy. + if (typeof window !== "undefined") return "/api"; + // Fallback (non-browser environments). + return "http://localhost:8000"; } -export const ORCHESTRATOR_URL = env("VITE_ORCHESTRATOR_URL") ?? hostBase(8000); +export const ORCHESTRATOR_URL = env("VITE_ORCHESTRATOR_URL") ?? defaultOrchestratorUrl(); export const SUBTITLE_SERVICE_URL = env("VITE_SUBTITLE_SERVICE_URL") ?? `${ORCHESTRATOR_URL}/subtitle-service`; export const YOUTUBE_INGEST_URL = diff --git a/web/src/main.tsx b/src/web/src/main.tsx similarity index 100% rename from web/src/main.tsx rename to src/web/src/main.tsx diff --git a/web/src/pages/DashboardPage.tsx b/src/web/src/pages/DashboardPage.tsx similarity index 100% rename from web/src/pages/DashboardPage.tsx rename to src/web/src/pages/DashboardPage.tsx diff --git a/web/src/pages/RenderQueuePage.tsx b/src/web/src/pages/RenderQueuePage.tsx similarity index 69% rename from web/src/pages/RenderQueuePage.tsx rename to src/web/src/pages/RenderQueuePage.tsx index 58e0201..695634d 100644 --- a/web/src/pages/RenderQueuePage.tsx +++ b/src/web/src/pages/RenderQueuePage.tsx @@ -1,42 +1,52 @@ import { Link } from "react-router-dom"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { fetchJson } from "../lib/http"; import { SUBTITLE_SERVICE_URL } from "../lib/urls"; -type RenderJob = { - id: string; +type TaskQueueItem = { task_id: string; + state: string; + stage: string; subtitle_job_id?: string | null; - status: string; + render_job_id?: string | null; progress: number; - retry_count: number; error_message?: string | null; created_at: string; updated_at: string; - started_at?: string | null; - finished_at?: string | null; }; -type RenderQueue = { +type TaskQueue = { settings: { max_concurrency: number }; running_count: number; queued_count: number; - jobs: RenderJob[]; + tasks: TaskQueueItem[]; }; export default function RenderQueuePage() { - const [queue, setQueue] = useState(null); + const [queue, setQueue] = useState(null); const [error, setError] = useState(null); const [busy, setBusy] = useState(false); const [maxConcText, setMaxConcText] = useState("1"); const [maxConcDirty, setMaxConcDirty] = useState(false); + const lastKickAtRef = useRef(0); const refresh = useCallback(async () => { setError(null); try { - const q = await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/render_queue`); + const q = await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/task_queue`); setQueue(q); if (!maxConcDirty) setMaxConcText(String(q?.settings?.max_concurrency ?? 1)); + + const maxConc = Number(q?.settings?.max_concurrency ?? 1); + const queued = Number(q?.queued_count ?? 0); + const running = Number(q?.running_count ?? 0); + if (queued > 0 && running < maxConc && maxConc > 0) { + const now = Date.now(); + if (now - (lastKickAtRef.current || 0) > 10_000) { + lastKickAtRef.current = now; + fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/task_queue/tick`, { method: "POST" }).catch(() => {}); + } + } } catch (e: unknown) { setError(e instanceof Error ? e.message : String(e)); } @@ -46,7 +56,7 @@ export default function RenderQueuePage() { refresh(); }, []); - const shouldPoll = useMemo(() => (queue?.jobs ?? []).some((j) => j.status === "queued" || j.status === "running"), [queue]); + const shouldPoll = useMemo(() => (queue?.tasks ?? []).some((t) => t.state === "queued" || t.state === "running"), [queue]); useEffect(() => { if (!shouldPoll) return; @@ -74,7 +84,7 @@ export default function RenderQueuePage() { const n = Number(raw); if (!Number.isFinite(n) || !Number.isInteger(n)) throw new Error("max_concurrency 必须是整数"); if (n < 0 || n > 32) throw new Error("max_concurrency 范围：0..32（0=暂停）"); - await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/render_queue/settings`, { + await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/task_queue/settings`, { method: "PUT", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ max_concurrency: n }), @@ -91,9 +101,9 @@ export default function RenderQueuePage() { return (
-
Queue · Render (ffmpeg)
+
Queue · Task
- 用于限制 “压制硬字幕 / 封装软字幕” 的并发数；ASR/翻译不受该队列影响。 + 用于限制任务并发：一个任务从字幕处理开始到压制结束占用一个并发名额（按 Task 计数）。
{error ?
{error}
: null}
@@ -135,39 +145,42 @@ export default function RenderQueuePage() {
Current Queue
+
仅显示 running/queued（按 Task 去重）。
{!queue ?
加载中…
: null} - {queue && (queue.jobs ?? []).length === 0 ?
暂无
: null} - {queue && (queue.jobs ?? []).length > 0 ? ( + {queue && (queue.tasks ?? []).length === 0 ?
暂无
: null} + {queue && (queue.tasks ?? []).length > 0 ? (
- + + - + - {(queue.jobs ?? []).map((j) => ( - - - + {(queue.tasks ?? []).map((t) => ( + + + + - - - + + +
StatusStateStage Progress Task Subtitle JobRetryRender Job Updated Error
{j.status}{j.progress}%
{t.state}{t.stage}{t.progress}% - - {j.task_id.slice(0, 8)} + + {t.task_id.slice(0, 8)} {j.subtitle_job_id ? j.subtitle_job_id.slice(0, 8) : "-"}{j.retry_count ?? 0}{new Date(j.updated_at).toLocaleString()}{t.subtitle_job_id ? t.subtitle_job_id.slice(0, 8) : "-"}{t.render_job_id ? t.render_job_id.slice(0, 8) : "-"}{new Date(t.updated_at).toLocaleString()} - {j.error_message ? ( -
- {j.error_message} + {t.error_message ? ( +
+ {t.error_message}
) : ( - diff --git a/web/src/pages/SettingsASRPage.tsx b/src/web/src/pages/SettingsASRPage.tsx similarity index 100% rename from web/src/pages/SettingsASRPage.tsx rename to src/web/src/pages/SettingsASRPage.tsx diff --git a/web/src/pages/SettingsAutoPage.tsx b/src/web/src/pages/SettingsAutoPage.tsx similarity index 100% rename from web/src/pages/SettingsAutoPage.tsx rename to src/web/src/pages/SettingsAutoPage.tsx diff --git a/web/src/pages/SettingsBilibiliPage.tsx b/src/web/src/pages/SettingsBilibiliPage.tsx similarity index 100% rename from web/src/pages/SettingsBilibiliPage.tsx rename to src/web/src/pages/SettingsBilibiliPage.tsx diff --git a/web/src/pages/SettingsStoragePage.tsx b/src/web/src/pages/SettingsStoragePage.tsx similarity index 100% rename from web/src/pages/SettingsStoragePage.tsx rename to src/web/src/pages/SettingsStoragePage.tsx diff --git a/web/src/pages/SettingsTranslatePage.tsx b/src/web/src/pages/SettingsTranslatePage.tsx similarity index 100% rename from web/src/pages/SettingsTranslatePage.tsx rename to src/web/src/pages/SettingsTranslatePage.tsx diff --git a/src/web/src/pages/SettingsYouTubePage.tsx b/src/web/src/pages/SettingsYouTubePage.tsx new file mode 100644 index 0000000..d1bc456 --- /dev/null +++ b/src/web/src/pages/SettingsYouTubePage.tsx @@ -0,0 +1,303 @@ +import { useEffect, useState } from "react"; +import { fetchJson } from "../lib/http"; +import { ORCHESTRATOR_URL } from "../lib/urls"; + +type YouTubeSettings = { + proxy: string; + cookies_set?: boolean; + cookies_enabled?: boolean; + cookies_updated_at?: string | null; + cookies_count?: number; + cookies_domains_count?: number; + cookies_has_auth?: boolean; + cookies_has_bot_check_bypass?: boolean; + cookies_has_visitor_info?: boolean; + cookie_file_configured?: boolean; + cookie_file_exists?: boolean; +}; + +type YouTubeProxyTestResponse = { + ok: boolean; + url: string; + used_proxy?: string | null; + status_code?: number | null; + elapsed_ms: number; + error?: string | null; +}; + +export default function SettingsYouTubePage() { + const [settings, setSettings] = useState(null); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + + const [proxy, setProxy] = useState(""); + const [cookiesTxt, setCookiesTxt] = useState(""); + const [cookiesBusy, setCookiesBusy] = useState(false); + const [cookiesEnabled, setCookiesEnabled] = useState(false); + const [testUrl, setTestUrl] = useState("https://www.youtube.com/robots.txt"); + const [testBusy, setTestBusy] = useState(false); + const [testResult, setTestResult] = useState(null); + + async function refresh() { + setError(null); + try { + const s = await fetchJson(`${ORCHESTRATOR_URL}/settings/youtube`); + setSettings(s); + setProxy((s.proxy ?? "").toString()); + setCookiesEnabled(Boolean(s.cookies_enabled)); + setCookiesTxt(""); + } catch (e: unknown) { + setError(e instanceof Error ? e.message : String(e)); + } + } + + useEffect(() => { + refresh(); + }, []); + + return ( +
+
+
Settings · YouTube
+
只影响 YouTube 的下载/元信息/封面/订阅扫描等请求。
+ {error ?
{error}
: null} +
+ +
+
+
当前配置
+ +
+ {!settings ? ( +
加载中…
+ ) : ( +
+
+
proxy
+
{settings.proxy || "-"}
+
+
+
cookies
+
+ {settings.cookies_set ? "已设置" : "未设置"} + {settings.cookies_set ? ` · ${settings.cookies_enabled ? "已启用" : "未启用"}` : ""} +
+ {settings.cookies_set ? ( +
+
+ 条目：{typeof settings.cookies_count === "number" ? settings.cookies_count : "-"} · 域名：{typeof settings.cookies_domains_count === "number" ? settings.cookies_domains_count : "-"} +
+
+ 登录态：{settings.cookies_has_auth ? "看起来有" : "看起来没有"} · 反爬豁免：{settings.cookies_has_bot_check_bypass ? "有" : "没有"} +
+ {settings.cookies_updated_at ?
更新时间：{settings.cookies_updated_at}
: null} + {settings.cookies_enabled && settings.cookies_set && settings.cookies_has_auth === false ? ( +
+ 提示：当前 cookies 可能不包含登录态（缺少 SID/SAPISID 等），遇到“Sign in to confirm you’re not a bot”时通常无效。 +
+ ) : null} +
+ ) : null} +
+
+
cookies 文件（YOUTUBE_COOKIE_FILE）
+
+ {settings.cookie_file_configured + ? `已配置 · ${settings.cookie_file_exists ? "文件存在" : "文件不存在"}` + : "未配置"} +
+ {!settings.cookie_file_configured && settings.cookies_set ? ( +
+ 未配置 YOUTUBE_COOKIE_FILE 也没关系：已保存的 cookies 会在下载/元信息提取时写入临时文件供 yt-dlp 使用。 +
+ ) : null} +
+
+ )} +
+ +
+
代理设置
+
+ 例如：http://127.0.0.1:7890 / socks5://127.0.0.1:1080（留空=不使用代理） +
+
+ + +
+ +
+ + + +
+ + {testResult ? ( +
+
+
{testResult.ok ? "OK" : "FAILED"}
+
status={testResult.status_code ?? "-"}
+
elapsed={testResult.elapsed_ms}ms
+
+
url: {testResult.url}
+
proxy: {testResult.used_proxy ?? "(none)"}
+ {testResult.error ?
{testResult.error}
: null} +
+ ) : null} +
+ +
+
Cookies 设置
+
+ 将浏览器导出的 cookies.txt（Netscape 格式）粘贴到下面。保存后用于 yt-dlp 下载/元信息提取。 +
+
+
建议：用浏览器无痕/新会话登录 YouTube → 导出 cookies.txt → 立刻粘贴保存 → 关闭无痕窗口（避免 cookies 被轮换导致很快失效）。
+
如果配置了代理，请确保浏览器也使用同一个代理/IP 导出并通过验证码，否则服务端仍可能触发风控。
+
+ +