feat: add RAG agent workflow and dashboard monitoring

This commit is contained in:
kelriclink committed 2026-07-01 19:42:48 +08:00
1 parent 750d3371a5
commit 6f872f817b
46 files changed
+6630 -387

No files matched your search

+1
View File
@@ -32,3 +32,4 @@ videoroll-prod-bundle-*.tar
videoroll-prod-bundle-*.tar.sha256
videoroll-deploy-*/
videoroll-prod-deploy-*/
docker_exports/
+4
View File
@@ -3,6 +3,8 @@ FROM ubuntu:24.04
ENV DEBIAN_FRONTEND=noninteractive
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV PIP_DEFAULT_TIMEOUT=300
ENV PIP_RETRIES=20
ENV VIRTUAL_ENV=/opt/venv
ENV PATH="/opt/venv/bin:${PATH}"
@@ -31,11 +33,13 @@ COPY pyproject.toml README.md ./
ARG INSTALL_ASR=0
ARG YTDLP_VERSION=latest
ARG TORCH_CPU_INDEX_URL=https://download.pytorch.org/whl/cpu
# Install dependencies in a cache-friendly layer so editing source code doesn't
# force re-downloading everything on every docker build.
RUN INSTALL_ASR="$INSTALL_ASR" python -c "import os, tomllib; from pathlib import Path; data=tomllib.loads(Path('pyproject.toml').read_text('utf-8')); deps=list(data.get('project', {}).get('dependencies', []) or []); opt=data.get('project', {}).get('optional-dependencies', {}) or {}; deps += list(opt.get('asr', []) or []) if os.getenv('INSTALL_ASR','0')=='1' else []; Path('/tmp/requirements.txt').write_text('\\n'.join(deps) + '\\n', encoding='utf-8')" \
&& pip install --no-cache-dir -U pip \
&& pip install --no-cache-dir --index-url "$TORCH_CPU_INDEX_URL" "torch>=2.3,<3" \
&& pip install --no-cache-dir -r /tmp/requirements.txt
COPY src/videoroll ./src/videoroll
+71 -1
View File
@@ -13,6 +13,9 @@
- 若未配置 `YOUTUBE_COOKIE_FILE`,已保存到数据库的 YouTube cookies 会在下载/元信息提取时写入临时文件供 `yt-dlp` 使用。
- YouTube 触发风控时,接口会返回更明确的 cookies / proxy 提示。
- Bilibili 投稿支持在 Web UI 中保存 Cookie、测试登录、维护默认投稿模板。
- Web UI 支持浅色 / 暗色模式切换,Dashboard 增加 CPU、内存和 Intel GPU 资源监控。
- 字幕翻译支持真正的 RAG 术语知识库:可使用 PostgreSQL + pgvector 检索术语,支持本地 embedding 模型、手动下载模型、重建向量,以及 Agent 自动发现/搜索/验证/入库术语。
- RAG Agent 支持 SearXNG 搜索、网页正文读取、独立 verifier、保守入库策略,并可在 Dashboard 查看每个 Agent 的工具调用 trace。
## 架构概览
@@ -165,7 +168,7 @@ docker compose -f docker-compose.yml --env-file .env up --build -d
- `data/minio`
MinIO 数据目录
- `data/models`
ASR 模型目录(可存放 faster-whisper 或 OpenVINO Whisper 导出模型)
ASR / embedding 模型目录(可存放 faster-whisper、OpenVINO Whisper 导出模型、本地 embedding 模型)
- `data/secrets`
本地密钥和机密目录
- `data/secrets/fernet.key`
@@ -263,6 +266,7 @@ YouTube 风控注意事项:
- `Settings -> ASR`
- `Settings -> Translate`
- `Settings -> Auto`
- `Dashboard`
说明:
@@ -274,6 +278,7 @@ YouTube 风控注意事项:
- 编码参数
- 翻译目标语言与 provider
- 是否自动投稿 Bilibili
- `Dashboard` 会显示任务状态、资源监控、RAG Agent 当前运行状态和历史 trace。
如果只是想做轻量演示,可把:
@@ -298,6 +303,71 @@ SUBTITLE_OPENVINO_MODEL=/models/whisper/whisper-large-v3-ov
- 也支持手工准备模型:可以在宿主机先用 `optimum-cli export openvino --model openai/whisper-large-v3 <output_dir>` 生成,再挂载/上传到 `data/models`。
- 下载或上传后,在 `Settings -> ASR` 中把默认引擎切到 `openvino`,并配置 `device / num_beams / max_new_tokens`。
### 翻译 RAG / Agent
相关页面:
- `Settings -> Translate`
- `Dashboard -> RAG Agent`
RAG 用于解决视频字幕翻译中的上下文和术语问题,例如游戏、动漫、技术讲解中的专有名词、缩写、黑话和社区固定译法。
当前流程:
```text
字幕片段
-> 术语发现
-> 查询 PostgreSQL/pgvector 知识库
-> 未命中时生成多条搜索 query
-> 调用 SearXNG 搜索
-> 必要时读取网页正文
-> LLM 总结候选术语
-> verifier 检查来源和上下文
-> 写入知识库或标记为 context_only / skipped
-> 翻译时注入 rag_context
```
关键行为:
- 单字母变量(如逻辑课里的 `P/Q/R`)默认视为局部变量,不搜索、不写入长期知识库。
- 常见基础术语(如 `truth table`)可以作为本次翻译提示,但默认不自动污染长期知识库。
- 自动正式入库要求 verifier 通过、上下文一致、有有效外部来源,并达到较高置信度;否则进入待审核或跳过。
- Dashboard 中可以点击 RAG Agent 查看每步工具调用、耗时、模型、错误类型和 JSON 输入输出。
PostgreSQL 需要启用 pgvector:
```sql
CREATE EXTENSION IF NOT EXISTS vector;
```
如果应用账号不是数据库超级用户,请用数据库管理员账号先在目标库执行上面的 SQL。
SearXNG 搜索地址建议填写 base URL,例如:
```text
https://search.example.com
```
系统会自动请求 `/search?q=...&format=json`,不需要在配置里手写 `?q=`。
Embedding 支持两种方式:
- OpenAI 兼容 embedding:填写 provider 支持的 embedding 模型名。
- 本地 embedding:在 `Settings -> Translate` 中选择本地 provider,可手动下载模型到 `data/models`,并选择 CPU / OpenVINO GPU 等设备。
更换 embedding 模型后,已有知识库向量不会静默重建;需要在 `Settings -> Translate` 中点击重建按钮,避免模型切换时旧向量和新向量混用。
### Dashboard 资源监控
Dashboard 会定时显示:
- CPU 使用率和 load average
- 内存使用率
- 启用自动模式并配置 Intel GPU 时的 Intel GPU 状态
- 正在运行和最近完成的 RAG Agent
Intel GPU 使用率依赖宿主机和容器能读取对应设备/驱动信息;如果 busy 不可读,页面会显示不可读原因,但不影响其他流程。
### Bilibili 配置
相关页面:
-6
View File
@@ -1,6 +0,0 @@
services:
app:
devices:
- "${INTEL_GPU_RENDER_DEVICE:-/dev/dri/renderD128}:${INTEL_GPU_RENDER_DEVICE:-/dev/dri/renderD128}"
environment:
INTEL_GPU_RENDER_DEVICE: ${INTEL_GPU_RENDER_DEVICE:-/dev/dri/renderD128}
+89
View File
@@ -0,0 +1,89 @@
# This file is intended to be placed at the repository root before deployment.
services:
redis:
image: redis:7
restart: unless-stopped
minio:
image: minio/minio:latest
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-videoroll}
MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-videorollsecret}
ports:
- "${PUBLISH_ADDR:-127.0.0.1}:${MINIO_CONSOLE_PORT:-9001}:9001"
volumes:
- ./data/minio:/data
restart: unless-stopped
minio-init:
image: minio/mc:latest
depends_on:
- minio
entrypoint: /bin/sh
command: >
-c "
until mc alias set local http://minio:9000 ${S3_ACCESS_KEY_ID:-videoroll} ${S3_SECRET_ACCESS_KEY:-videorollsecret}; do
echo 'waiting for minio...';
sleep 1;
done &&
mc mb -p local/${S3_BUCKET:-videoroll} || true
"
restart: "no"
app:
build:
context: .
dockerfile: Dockerfile
args:
INSTALL_ASR: ${INSTALL_ASR:-1}
YTDLP_VERSION: ${YTDLP_VERSION:-latest}
image: videoroll:prod
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
S3_ENDPOINT_URL: ${S3_ENDPOINT_URL:-http://minio:9000}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-videoroll}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-videorollsecret}
S3_BUCKET: ${S3_BUCKET:-videoroll}
S3_REGION_NAME: ${S3_REGION_NAME:-us-east-1}
S3_USE_SSL: ${S3_USE_SSL:-false}
UVICORN_ROOT_PATH: ${UVICORN_ROOT_PATH:-/api}
SUBTITLE_SERVICE_URL: ${SUBTITLE_SERVICE_URL:-http://localhost:8000/subtitle-service}
YOUTUBE_INGEST_URL: ${YOUTUBE_INGEST_URL:-http://localhost:8000/youtube-ingest}
BILIBILI_PUBLISHER_URL: ${BILIBILI_PUBLISHER_URL:-http://localhost:8000/bilibili-publisher}
BILIBILI_PUBLISH_MODE: ${BILIBILI_PUBLISH_MODE:-web}
volumes:
- ./data/models:/models
- ./data/secrets:/secrets
- ./data/work:/work
# Expose the whole Intel DRM device tree so both VAAPI and OpenVINO GPU
# runtimes can discover the render device and initialize compute contexts.
devices:
- "/dev/dri:/dev/dri"
extra_hosts:
- host.docker.internal:host-gateway
depends_on:
redis:
condition: service_started
minio:
condition: service_started
minio-init:
condition: service_completed_successfully
restart: unless-stopped
web:
build:
context: ./src/web
args:
VITE_ORCHESTRATOR_URL: ${VITE_ORCHESTRATOR_URL:-}
VITE_SUBTITLE_SERVICE_URL: ${VITE_SUBTITLE_SERVICE_URL:-}
VITE_YOUTUBE_INGEST_URL: ${VITE_YOUTUBE_INGEST_URL:-}
VITE_BILIBILI_PUBLISHER_URL: ${VITE_BILIBILI_PUBLISHER_URL:-}
image: videoroll-web:prod
ports:
- "${PUBLISH_ADDR:-127.0.0.1}:${WEB_PORT:-3000}:80"
depends_on:
app:
condition: service_started
restart: unless-stopped
+2
View File
@@ -25,6 +25,8 @@ dependencies = [
"defusedxml>=0.7.1",
"yt-dlp[default]>=2026.2.4",
"deno>=2.6.9",
"sentence-transformers>=3.0.0",
"optimum-intel[openvino]>=1.20.0",
]
[project.optional-dependencies]
+91
View File
@@ -9,6 +9,7 @@ from typing import Any, Mapping
import httpx
from videoroll.utils.openai_compat import build_openai_chat_completions_url
from videoroll.utils.openai_compat import build_openai_embeddings_url
_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
@@ -203,3 +204,93 @@ def request_openai_json_object(
format_retries=format_retries,
network_retries=network_retries,
)
def request_openai_embedding(
*,
config: OpenAIChatConfig,
text: str,
client: httpx.Client | None = None,
network_retries: int = 3,
) -> list[float]:
if not config.api_key:
raise RuntimeError("OpenAI API key is not set")
source = str(text or "").strip()
if not source:
raise ValueError("embedding text is empty")
url = build_openai_embeddings_url(config.base_url)
headers = {"Authorization": f"Bearer {config.api_key}"}
req = {"model": config.model, "input": source}
attempts_network = max(1, int(network_retries))
last_err: Exception | None = None
def _with_client(c: httpx.Client) -> list[float]:
nonlocal last_err
for net_attempt in range(attempts_network):
try:
resp = c.post(url, headers=headers, json=req)
try:
resp.raise_for_status()
except httpx.HTTPStatusError as e:
status = resp.status_code
if status in _RETRYABLE_STATUS_CODES and net_attempt < attempts_network - 1:
retry_after = (resp.headers.get("retry-after") or "").strip()
if retry_after:
try:
time.sleep(min(30.0, float(retry_after)))
except Exception:
_sleep_backoff(net_attempt)
else:
_sleep_backoff(net_attempt)
continue
ct = (resp.headers.get("content-type") or "").split(";")[0].strip()
snippet = _resp_snippet(resp)
raise RuntimeError(
f"OpenAI embedding request failed (status={resp.status_code}, content-type={ct}, url={url}). {snippet}"
) from e
try:
resp_json = resp.json()
except Exception as e:
ct = (resp.headers.get("content-type") or "").split(";")[0].strip()
hint = " (check openai_base_url; most providers require it to end with /v1)" if "text/html" in ct else ""
raise RuntimeError(
f"OpenAI embedding endpoint did not return JSON (status={resp.status_code}, content-type={ct}, url={url}){hint}."
) from e
try:
raw = resp_json["data"][0]["embedding"]
except Exception as e:
raise RuntimeError(f"unexpected OpenAI embedding response shape: {resp_json}") from e
if not isinstance(raw, list) or not raw:
raise RuntimeError("OpenAI embedding output is empty")
return [float(x) for x in raw]
except httpx.TimeoutException as e:
last_err = e
if net_attempt < attempts_network - 1:
_sleep_backoff(net_attempt)
continue
break
except httpx.TransportError as e:
last_err = e
if net_attempt < attempts_network - 1:
_sleep_backoff(net_attempt)
continue
break
except Exception as e:
last_err = e
break
if last_err is not None:
raise last_err
raise RuntimeError("OpenAI embedding request failed")
if client is not None:
return _with_client(client)
with create_openai_http_client(config.timeout_seconds) as owned_client:
return _with_client(owned_client)
@@ -52,7 +52,9 @@ from videoroll.storage.s3 import S3Store
from videoroll.utils.auto_youtube import encode_auto_youtube_created_by
from videoroll.utils.hashing import sha256_file
from videoroll.utils.httpx_proxy import HTTPX_PROXY_KWARG_UNSUPPORTED, format_httpx_proxy_error
from videoroll.utils.intel_gpu import sample_intel_gpu_usage
from videoroll.utils.internal_api_token import internal_api_token
from videoroll.utils.resources import process_cpu_summary, read_cgroup_memory_stats, read_memory_stats
from videoroll.utils.workdir_maintenance import (
WorkdirJobState,
cleanup_reclaimable_dirs,
@@ -85,6 +87,10 @@ from videoroll.apps.orchestrator_api.schemas import (
StorageRetentionSettingsUpdate,
SubtitleActionRequest,
SubtitleJobSummary,
SystemCPURead,
SystemIntelGPURead,
SystemMemoryRead,
SystemResourcesRead,
TaskCreate,
TaskPublishReviewRead,
TaskRead,
@@ -1633,6 +1639,74 @@ def health() -> dict[str, str]:
return {"status": "ok"}
def _memory_read(data: dict[str, Any] | None) -> SystemMemoryRead:
data = data or {}
return SystemMemoryRead(
total_bytes=int(data.get("total_bytes") or 0),
used_bytes=int(data.get("used_bytes") or 0),
available_bytes=int(data.get("available_bytes") or 0),
percent=float(data["percent"]) if data.get("percent") is not None else None,
)
@app.get("/system/resources", response_model=SystemResourcesRead)
def system_resources(db: Session = Depends(get_db)) -> SystemResourcesRead:
cpu_data = process_cpu_summary()
mem = read_memory_stats()
cgroup_mem = read_cgroup_memory_stats()
intel_gpu: SystemIntelGPURead | None = None
try:
profile = get_auto_profile(db)
intel_enabled = bool(profile.get("use_intel_gpu"))
except Exception:
intel_enabled = False
if intel_enabled:
subtitle_settings = get_subtitle_settings()
render_device = str(subtitle_settings.intel_gpu_render_device or "").strip() or "/dev/dri/renderD128"
try:
gpu_info = sample_intel_gpu_usage(render_device)
except Exception as e:
gpu_info = {
"enabled": True,
"checked": True,
"available": False,
"render_device": render_device,
"detail": str(e),
}
intel_gpu = SystemIntelGPURead(
enabled=True,
checked=bool(gpu_info.get("checked", True)),
available=bool(gpu_info.get("available", False)),
render_device=str(gpu_info.get("render_device") or render_device),
model_name=gpu_info.get("model_name"),
driver=gpu_info.get("driver"),
pci_slot=gpu_info.get("pci_slot"),
pci_id=gpu_info.get("pci_id"),
usage_supported=bool(gpu_info.get("usage_supported", False)),
usage_percent=float(gpu_info["usage_percent"]) if gpu_info.get("usage_percent") is not None else None,
engines=[
{"name": str(item.get("name") or ""), "percent": float(item["percent"]) if item.get("percent") is not None else None}
for item in list(gpu_info.get("engines") or [])
if isinstance(item, dict)
],
detail=str(gpu_info.get("detail") or ""),
)
return SystemResourcesRead(
sampled_at=datetime.now(tz=timezone.utc).isoformat(),
cpu=SystemCPURead(
percent=float(cpu_data["percent"]) if cpu_data.get("percent") is not None else None,
cores=int(cpu_data.get("cores") or 0),
load_average=cpu_data.get("load_average"),
),
memory=_memory_read(mem),
cgroup_memory=_memory_read(cgroup_mem) if cgroup_mem else None,
intel_gpu=intel_gpu,
)
def _secure_cookie(request: Request) -> bool:
proto = str(request.headers.get("x-forwarded-proto") or request.url.scheme or "").lower()
return proto == "https"
@@ -168,6 +168,47 @@ class RemotePublishResponse(BaseModel):
response: Optional[dict[str, Any]] = None
class SystemCPURead(BaseModel):
percent: Optional[float] = None
cores: int = 0
load_average: Optional[list[float]] = None
class SystemMemoryRead(BaseModel):
total_bytes: int = 0
used_bytes: int = 0
available_bytes: int = 0
percent: Optional[float] = None
class SystemIntelGPUEngineRead(BaseModel):
name: str
percent: Optional[float] = None
class SystemIntelGPURead(BaseModel):
enabled: bool = False
checked: bool = False
available: bool = False
render_device: str = ""
model_name: Optional[str] = None
driver: Optional[str] = None
pci_slot: Optional[str] = None
pci_id: Optional[str] = None
usage_supported: bool = False
usage_percent: Optional[float] = None
engines: list[SystemIntelGPUEngineRead] = Field(default_factory=list)
detail: str = ""
class SystemResourcesRead(BaseModel):
sampled_at: str
cpu: SystemCPURead
memory: SystemMemoryRead
cgroup_memory: Optional[SystemMemoryRead] = None
intel_gpu: Optional[SystemIntelGPURead] = None
class SubtitleJobSummary(BaseModel):
id: uuid.UUID
task_id: uuid.UUID
@@ -0,0 +1,271 @@
from __future__ import annotations
import shutil
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from videoroll.ai.client import OpenAIChatConfig, request_openai_embedding
from videoroll.apps.subtitle_service.model_downloads import download_model_snapshot
_SAFE_MODEL_NAME_REPLACEMENTS = {"/": "--", "\\": "--"}
_LOCAL_MODEL_CACHE: dict[tuple[str, str, str], Any] = {}
_LOCAL_MODEL_CACHE_LOCK = threading.Lock()
@dataclass(frozen=True)
class EmbeddingSettings:
provider: str
model: str
dimensions: int
model_dir: str
device: str
openai_config: OpenAIChatConfig
def normalize_embedding_provider(provider: str | None) -> str:
out = str(provider or "").strip().lower() or "openai"
if out not in {"openai", "local"}:
raise ValueError(f"unsupported embedding provider: {out}")
return out
def safe_embedding_model_name(model: str) -> str:
out = str(model or "").strip()
if not out:
raise ValueError("model is required")
for old, new in _SAFE_MODEL_NAME_REPLACEMENTS.items():
out = out.replace(old, new)
out = "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in out)
return out[:96] or "embedding-model"
def embedding_model_path(model_dir: str | Path, model: str) -> Path:
p = Path(model_dir) / safe_embedding_model_name(model)
return p
def list_local_embedding_models(model_dir: str | Path) -> list[dict[str, Any]]:
root = Path(model_dir)
if not root.exists():
return []
out: list[dict[str, Any]] = []
for p in sorted(root.iterdir(), key=lambda x: x.name.lower()):
if not p.is_dir():
continue
size = 0
for item in p.rglob("*"):
if item.is_file():
try:
size += item.stat().st_size
except OSError:
pass
out.append({"name": p.name, "path": str(p), "size_bytes": size})
return out
def download_local_embedding_model(
*,
model: str,
model_dir: str | Path,
name: str | None = None,
revision: str | None = None,
force: bool = False,
proxy: str | None = None,
) -> Path:
model_name = str(model or "").strip()
if not model_name:
raise ValueError("model is required")
dest_name = safe_embedding_model_name(name or model_name)
return download_model_snapshot(
engine="embedding",
model=model_name,
model_dir=Path(model_dir),
name=dest_name,
revision=revision,
force=force,
proxy=proxy,
)
def delete_local_embedding_model(*, model_dir: str | Path, name: str) -> None:
path = Path(model_dir) / safe_embedding_model_name(name)
if path.exists():
shutil.rmtree(path)
def _load_sentence_transformer(path: Path, *, device: str) -> Any:
try:
from sentence_transformers import SentenceTransformer # type: ignore
except Exception as e:
raise RuntimeError("sentence-transformers is not installed. Rebuild the app image with embedding dependencies.") from e
if not path.exists():
raise RuntimeError(f"local embedding model is not downloaded: {path}")
device_arg = str(device or "cpu").strip() or "cpu"
cache_key = ("sentence-transformers", str(path), device_arg)
with _LOCAL_MODEL_CACHE_LOCK:
model = _LOCAL_MODEL_CACHE.get(cache_key)
if model is None:
model = SentenceTransformer(str(path), device=device_arg)
_LOCAL_MODEL_CACHE[cache_key] = model
return model
def _parse_local_device(device: str) -> tuple[str, str]:
value = str(device or "cpu").strip() or "cpu"
lowered = value.lower()
if lowered.startswith("openvino"):
_, sep, raw_device = value.partition(":")
ov_device = raw_device.strip() if sep else "CPU"
return "openvino", ov_device or "CPU"
return "sentence-transformers", value
def _openvino_export_dir(path: Path) -> Path:
if (path / "openvino_model.xml").exists():
return path
return path / "openvino"
def _load_openvino_feature_extractor(path: Path, *, device: str) -> tuple[Any, Any]:
try:
from optimum.intel.openvino import OVModelForFeatureExtraction # type: ignore
from transformers import AutoTokenizer # type: ignore
except Exception as e:
raise RuntimeError("OpenVINO embedding requires optimum-intel[openvino]. Rebuild the app image with embedding dependencies.") from e
if not path.exists():
raise RuntimeError(f"local embedding model is not downloaded: {path}")
device_arg = str(device or "CPU").strip() or "CPU"
ov_dir = _openvino_export_dir(path)
cache_key = ("openvino-feature-extraction", str(ov_dir), device_arg)
with _LOCAL_MODEL_CACHE_LOCK:
cached = _LOCAL_MODEL_CACHE.get(cache_key)
if cached is not None:
return cached
if (ov_dir / "openvino_model.xml").exists():
model = OVModelForFeatureExtraction.from_pretrained(str(ov_dir), device=device_arg)
tokenizer = AutoTokenizer.from_pretrained(str(path))
else:
ov_dir.mkdir(parents=True, exist_ok=True)
model = OVModelForFeatureExtraction.from_pretrained(str(path), export=True, device=device_arg)
tokenizer = AutoTokenizer.from_pretrained(str(path))
try:
model.save_pretrained(str(ov_dir))
tokenizer.save_pretrained(str(ov_dir))
except Exception:
pass
cached = (model, tokenizer)
_LOCAL_MODEL_CACHE[cache_key] = cached
return cached
def _mean_pool_embedding(model_output: Any, attention_mask: Any) -> list[float]:
try:
import numpy as np # type: ignore
import torch # type: ignore
except Exception as e:
raise RuntimeError("OpenVINO embedding pooling requires numpy and torch.") from e
if isinstance(model_output, dict):
token_embeddings = model_output.get("last_hidden_state")
else:
token_embeddings = getattr(model_output, "last_hidden_state", None)
if token_embeddings is None and isinstance(model_output, (list, tuple)) and model_output:
token_embeddings = model_output[0]
if token_embeddings is None:
raise RuntimeError("OpenVINO embedding output missing last_hidden_state")
if hasattr(token_embeddings, "detach"):
token_embeddings_np = token_embeddings.detach().cpu().numpy()
else:
token_embeddings_np = np.asarray(token_embeddings)
if hasattr(attention_mask, "detach"):
mask_np = attention_mask.detach().cpu().numpy()
else:
mask_np = np.asarray(attention_mask)
mask_np = np.expand_dims(mask_np, axis=-1).astype(np.float32)
summed = (token_embeddings_np * mask_np).sum(axis=1)
counts = np.clip(mask_np.sum(axis=1), 1e-9, None)
vector = summed / counts
tensor = torch.tensor(vector, dtype=torch.float32)
tensor = torch.nn.functional.normalize(tensor, p=2, dim=1)
return _normalize_vector(tensor.detach().cpu().numpy())
def _embed_text_openvino(path: Path, text: str, *, device: str) -> list[float]:
model, tokenizer = _load_openvino_feature_extractor(path, device=device)
inputs = tokenizer(text, padding=True, truncation=True, return_tensors="pt")
outputs = model(**inputs)
return _mean_pool_embedding(outputs, inputs["attention_mask"])
def _normalize_vector(values: Any) -> list[float]:
try:
import numpy as np # type: ignore
if isinstance(values, np.ndarray):
raw = values.tolist()
else:
raw = values
except Exception:
raw = values
if isinstance(raw, list) and raw and isinstance(raw[0], list):
raw = raw[0]
if not isinstance(raw, list) or not raw:
raise RuntimeError("embedding output is empty")
return [float(x) for x in raw]
def embed_text(text: str, *, settings: EmbeddingSettings) -> list[float]:
provider = normalize_embedding_provider(settings.provider)
source = str(text or "").strip()
if not source:
raise ValueError("embedding text is empty")
if provider == "openai":
return request_openai_embedding(config=settings.openai_config, text=source)
model_path = embedding_model_path(settings.model_dir, settings.model)
backend, device = _parse_local_device(settings.device)
if backend == "openvino":
return _embed_text_openvino(model_path, source, device=device)
model = _load_sentence_transformer(model_path, device=device)
try:
encoded = model.encode(source, normalize_embeddings=True)
except TypeError:
encoded = model.encode(source)
return _normalize_vector(encoded)
def assert_embedding_dimensions(vector: list[float], expected: int) -> None:
if expected <= 0:
return
if len(vector) != expected:
raise RuntimeError(f"embedding dimension mismatch: expected {expected}, got {len(vector)}")
def embedding_settings_from_translate_settings(settings: dict[str, Any]) -> EmbeddingSettings:
provider = normalize_embedding_provider(str(settings.get("rag_embedding_provider") or "openai"))
return EmbeddingSettings(
provider=provider,
model=str(settings.get("rag_embedding_model") or "text-embedding-3-small").strip() or "text-embedding-3-small",
dimensions=max(1, min(4096, int(settings.get("rag_embedding_dimensions") or 1536))),
model_dir=str(settings.get("rag_embedding_model_dir") or "/models/embeddings").strip() or "/models/embeddings",
device=str(settings.get("rag_embedding_device") or "cpu").strip() or "cpu",
openai_config=OpenAIChatConfig(
api_key=str(settings.get("openai_api_key") or "").strip() or None,
base_url=str(settings.get("openai_base_url") or "").strip(),
model=str(settings.get("rag_embedding_model") or "text-embedding-3-small").strip() or "text-embedding-3-small",
temperature=0.0,
timeout_seconds=float(settings.get("openai_timeout_seconds") or 60.0),
),
)
+361
View File
@@ -16,6 +16,7 @@ import httpx
from fastapi import Depends, FastAPI, File, HTTPException, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy.orm import Session
from sqlalchemy.exc import ProgrammingError
from videoroll.ai.client import openai_chat_config_from_settings
from videoroll.ai.service import translate_text_openai
@@ -42,6 +43,17 @@ from videoroll.apps.subtitle_service.schemas import (
WhisperSettingsRead,
ModelDownloadProxyTestRequest,
ModelDownloadProxyTestResponse,
KnowledgeItemRead,
KnowledgeEmbeddingRebuildRequest,
KnowledgeEmbeddingRebuildResponse,
KnowledgeItemUpsertRequest,
KnowledgeItemUpsertResponse,
AgentRunRead,
EmbeddingModelDownloadRequest,
EmbeddingModelListRequest,
EmbeddingModelInfo,
EmbeddingTestRequest,
EmbeddingTestResponse,
TaskQueueItemRead,
TaskQueueRead,
TaskQueueSettingsRead,
@@ -55,6 +67,22 @@ from videoroll.apps.subtitle_service.model_downloads import (
normalize_model_download_engine,
)
from videoroll.apps.subtitle_service.render_queue_store import get_task_queue_settings, update_task_queue_settings
from videoroll.apps.subtitle_service.embeddings import (
assert_embedding_dimensions,
delete_local_embedding_model,
download_local_embedding_model,
embedding_settings_from_translate_settings,
embed_text,
list_local_embedding_models,
)
from videoroll.apps.subtitle_service.rag import (
build_knowledge_embedding_text,
list_agent_runs,
rebuild_knowledge_embeddings,
list_knowledge_items,
rag_settings_from_translate_settings,
upsert_knowledge_item,
)
from videoroll.apps.subtitle_service.translate_settings_store import get_translate_settings, update_translate_settings
from videoroll.apps.subtitle_service.worker_concurrency import sync_subtitle_worker_concurrency_for_task_queue_settings
from videoroll.apps.subtitle_service.worker import TASK_QUEUE_LOCK_OWNER, celery_app
@@ -78,6 +106,59 @@ def _models_dir(settings: SubtitleServiceSettings) -> Path:
return Path(settings.whisper_model_dir)
def _embedding_models_dir(settings: SubtitleServiceSettings) -> Path:
return Path(settings.rag_embedding_model_dir)
def _embedding_models_dir_from_translate_settings(cfg: dict[str, Any], settings: SubtitleServiceSettings) -> Path:
path = str(cfg.get("rag_embedding_model_dir") or "").strip()
if path:
return Path(path)
return _embedding_models_dir(settings)
def _is_missing_knowledge_table_error(exc: Exception) -> bool:
orig = getattr(exc, "orig", None)
sqlstate = getattr(orig, "sqlstate", None) or getattr(orig, "pgcode", None)
if sqlstate == "42P01": # PostgreSQL undefined_table
return True
msg = str(exc).lower()
return (
"does not exist" in msg
and any(
table in msg
for table in [
"translation_knowledge_items",
"translation_term_evidence",
"translation_term_matches",
"translation_agent_runs",
]
)
)
def _ensure_rag_schema(settings: SubtitleServiceSettings) -> None:
try:
auto_migrate(settings.database_url, force=True)
except Exception as e:
raise HTTPException(
status_code=503,
detail=(
"RAG knowledge tables are not ready. Ensure PostgreSQL has pgvector installed "
f"and restart the app. Migration error: {e}"
),
) from e
def _handle_knowledge_db_error(exc: Exception, settings: SubtitleServiceSettings) -> None:
if _is_missing_knowledge_table_error(exc):
_ensure_rag_schema(settings)
return
if isinstance(exc, ProgrammingError):
raise HTTPException(status_code=503, detail=f"RAG database schema is not ready: {exc}") from exc
raise HTTPException(status_code=500, detail=f"knowledge database error: {exc}") from exc
_SAFE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
@@ -254,6 +335,19 @@ def get_translate_settings_view(
openai_model=cfg["openai_model"],
openai_temperature=cfg["openai_temperature"],
openai_timeout_seconds=cfg["openai_timeout_seconds"],
rag_enabled=cfg["rag_enabled"],
rag_top_k=cfg["rag_top_k"],
rag_min_score=cfg["rag_min_score"],
rag_embedding_provider=cfg["rag_embedding_provider"],
rag_embedding_model=cfg["rag_embedding_model"],
rag_embedding_dimensions=cfg["rag_embedding_dimensions"],
rag_embedding_model_dir=cfg["rag_embedding_model_dir"],
rag_embedding_device=cfg["rag_embedding_device"],
rag_auto_discover_terms=cfg["rag_auto_discover_terms"],
rag_auto_learn_terms=cfg["rag_auto_learn_terms"],
rag_search_enabled=cfg["rag_search_enabled"],
rag_search_url=cfg["rag_search_url"],
rag_domain=cfg["rag_domain"],
)
@@ -276,6 +370,273 @@ def put_translate_settings_view(
openai_model=cfg["openai_model"],
openai_temperature=cfg["openai_temperature"],
openai_timeout_seconds=cfg["openai_timeout_seconds"],
rag_enabled=cfg["rag_enabled"],
rag_top_k=cfg["rag_top_k"],
rag_min_score=cfg["rag_min_score"],
rag_embedding_provider=cfg["rag_embedding_provider"],
rag_embedding_model=cfg["rag_embedding_model"],
rag_embedding_dimensions=cfg["rag_embedding_dimensions"],
rag_embedding_model_dir=cfg["rag_embedding_model_dir"],
rag_embedding_device=cfg["rag_embedding_device"],
rag_auto_discover_terms=cfg["rag_auto_discover_terms"],
rag_auto_learn_terms=cfg["rag_auto_learn_terms"],
rag_search_enabled=cfg["rag_search_enabled"],
rag_search_url=cfg["rag_search_url"],
rag_domain=cfg["rag_domain"],
)
@app.get("/subtitle/knowledge/items", response_model=list[KnowledgeItemRead])
def list_knowledge_items_view(
item_type: str | None = None,
status: str | None = None,
limit: int = 100,
settings: SubtitleServiceSettings = Depends(get_settings),
db: Session = Depends(get_db),
) -> list[KnowledgeItemRead]:
try:
rows = list_knowledge_items(db, item_type=item_type, status=status, limit=limit)
except Exception as e:
db.rollback()
if not _is_missing_knowledge_table_error(e):
_handle_knowledge_db_error(e, settings)
_ensure_rag_schema(settings)
try:
rows = list_knowledge_items(db, item_type=item_type, status=status, limit=limit)
except Exception as retry_error:
db.rollback()
raise HTTPException(status_code=503, detail=f"RAG knowledge tables are not ready: {retry_error}") from retry_error
return [KnowledgeItemRead(**row) for row in rows]
@app.post("/subtitle/knowledge/items", response_model=KnowledgeItemUpsertResponse)
def upsert_knowledge_item_view(
payload: KnowledgeItemUpsertRequest,
settings: SubtitleServiceSettings = Depends(get_settings),
db: Session = Depends(get_db),
) -> KnowledgeItemUpsertResponse:
cfg = get_translate_settings(db, settings)
rag_cfg = rag_settings_from_translate_settings(cfg)
if payload.item_type == "term" and not payload.term.strip():
raise HTTPException(status_code=400, detail="term is required for term items")
if payload.item_type == "term" and not payload.translation.strip():
raise HTTPException(status_code=400, detail="translation is required for term items")
if payload.item_type == "document" and not (payload.title.strip() or payload.content.strip()):
raise HTTPException(status_code=400, detail="title or content is required for document items")
embedding_text = build_knowledge_embedding_text(
item_type=payload.item_type,
term=payload.term,
translation=payload.translation,
domain=payload.domain,
aliases=payload.aliases,
title=payload.title,
content=payload.content,
description=payload.description,
)
embedding: list[float] | None = None
if embedding_text.strip():
try:
embedding = embed_text(embedding_text, settings=embedding_settings_from_translate_settings(cfg))
assert_embedding_dimensions(embedding, rag_cfg.embedding_dimensions)
except Exception as e:
if payload.item_type == "document":
raise HTTPException(status_code=502, detail=f"embedding failed: {e}") from e
embedding = None
def _save_item() -> str:
item_id = upsert_knowledge_item(
db,
item_type=payload.item_type,
target_lang=payload.target_lang,
term=payload.term,
translation=payload.translation,
domain=payload.domain,
aliases=payload.aliases,
title=payload.title,
content=payload.content,
description=payload.description,
sources=payload.sources,
confidence=payload.confidence,
status=payload.status,
created_by=payload.created_by,
embedding=embedding,
embedding_model=f"{rag_cfg.embedding_provider}:{rag_cfg.embedding_model}" if embedding else "",
)
db.commit()
return item_id
try:
item_id = _save_item()
except Exception as e:
db.rollback()
if not _is_missing_knowledge_table_error(e):
raise HTTPException(status_code=500, detail=f"knowledge item save failed: {e}") from e
_ensure_rag_schema(settings)
try:
item_id = _save_item()
except Exception as retry_error:
db.rollback()
raise HTTPException(status_code=503, detail=f"knowledge item save failed after schema migration: {retry_error}") from retry_error
return KnowledgeItemUpsertResponse(id=uuid.UUID(item_id))
@app.post("/subtitle/knowledge/rebuild-embeddings", response_model=KnowledgeEmbeddingRebuildResponse)
def rebuild_knowledge_embeddings_view(
payload: KnowledgeEmbeddingRebuildRequest,
settings: SubtitleServiceSettings = Depends(get_settings),
db: Session = Depends(get_db),
) -> KnowledgeEmbeddingRebuildResponse:
cfg = get_translate_settings(db, settings)
rag_cfg = rag_settings_from_translate_settings(cfg)
emb_cfg = embedding_settings_from_translate_settings(cfg)
def _rebuild() -> dict[str, Any]:
result = rebuild_knowledge_embeddings(
db,
rag_settings=rag_cfg,
embedding_settings=emb_cfg,
item_type=payload.item_type,
status=payload.status,
limit=payload.limit,
)
db.commit()
return result
try:
result = _rebuild()
except Exception as e:
db.rollback()
if not _is_missing_knowledge_table_error(e):
raise HTTPException(status_code=502, detail=f"knowledge embedding rebuild failed: {e}") from e
_ensure_rag_schema(settings)
try:
result = _rebuild()
except Exception as retry_error:
db.rollback()
raise HTTPException(status_code=503, detail=f"knowledge embedding rebuild failed after schema migration: {retry_error}") from retry_error
return KnowledgeEmbeddingRebuildResponse(**result)
@app.get("/subtitle/agents/runs", response_model=list[AgentRunRead])
def list_agent_runs_view(
status: str | None = None,
limit: int = 30,
settings: SubtitleServiceSettings = Depends(get_settings),
db: Session = Depends(get_db),
) -> list[AgentRunRead]:
try:
rows = list_agent_runs(db, status=status, limit=limit)
except Exception as e:
db.rollback()
if not _is_missing_knowledge_table_error(e):
_handle_knowledge_db_error(e, settings)
_ensure_rag_schema(settings)
try:
rows = list_agent_runs(db, status=status, limit=limit)
except Exception as retry_error:
db.rollback()
raise HTTPException(status_code=503, detail=f"RAG agent tables are not ready: {retry_error}") from retry_error
return [AgentRunRead(**row) for row in rows]
@app.get("/subtitle/embedding/models", response_model=list[EmbeddingModelInfo])
def list_embedding_models(
settings: SubtitleServiceSettings = Depends(get_settings),
db: Session = Depends(get_db),
) -> list[EmbeddingModelInfo]:
cfg = get_translate_settings(db, settings)
return [EmbeddingModelInfo(**item) for item in list_local_embedding_models(_embedding_models_dir_from_translate_settings(cfg, settings))]
@app.post("/subtitle/embedding/models/list", response_model=list[EmbeddingModelInfo])
def list_embedding_models_for_dir(
payload: EmbeddingModelListRequest,
settings: SubtitleServiceSettings = Depends(get_settings),
db: Session = Depends(get_db),
) -> list[EmbeddingModelInfo]:
cfg = get_translate_settings(db, settings)
if payload.model_dir is not None:
cfg["rag_embedding_model_dir"] = payload.model_dir
return [EmbeddingModelInfo(**item) for item in list_local_embedding_models(_embedding_models_dir_from_translate_settings(cfg, settings))]
@app.post("/subtitle/embedding/models/download", response_model=EmbeddingModelInfo)
def download_embedding_model(
payload: EmbeddingModelDownloadRequest,
settings: SubtitleServiceSettings = Depends(get_settings),
db: Session = Depends(get_db),
) -> EmbeddingModelInfo:
asr_cfg = get_asr_settings(db, settings)
cfg = get_translate_settings(db, settings)
if payload.model_dir is not None:
cfg["rag_embedding_model_dir"] = payload.model_dir
proxy = str(asr_cfg.get("model_download_proxy") or "").strip() or None
try:
dest = download_local_embedding_model(
model=payload.model,
model_dir=_embedding_models_dir_from_translate_settings(cfg, settings),
name=payload.name,
revision=payload.revision,
force=payload.force,
proxy=proxy,
)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e)) from e
size = sum(p.stat().st_size for p in dest.rglob("*") if p.is_file())
return EmbeddingModelInfo(name=dest.name, path=str(dest), size_bytes=size)
@app.delete("/subtitle/embedding/models/{name}")
def delete_embedding_model(
name: str,
settings: SubtitleServiceSettings = Depends(get_settings),
db: Session = Depends(get_db),
) -> dict[str, bool]:
model_name = _validate_model_name(name)
cfg = get_translate_settings(db, settings)
delete_local_embedding_model(model_dir=_embedding_models_dir_from_translate_settings(cfg, settings), name=model_name)
return {"deleted": True}
@app.post("/subtitle/embedding/test", response_model=EmbeddingTestResponse)
def test_embedding(
payload: EmbeddingTestRequest,
settings: SubtitleServiceSettings = Depends(get_settings),
db: Session = Depends(get_db),
) -> EmbeddingTestResponse:
text = str(payload.text or "").strip()
if not text:
raise HTTPException(status_code=400, detail="text is required")
if len(text) > 2000:
raise HTTPException(status_code=400, detail="text too long (max 2000 chars)")
cfg = get_translate_settings(db, settings)
if payload.provider is not None:
cfg["rag_embedding_provider"] = payload.provider
if payload.model is not None:
cfg["rag_embedding_model"] = payload.model
if payload.model_dir is not None:
cfg["rag_embedding_model_dir"] = payload.model_dir
if payload.dimensions is not None:
cfg["rag_embedding_dimensions"] = payload.dimensions
if payload.device is not None:
cfg["rag_embedding_device"] = payload.device
emb_settings = embedding_settings_from_translate_settings(cfg)
try:
vector = embed_text(text, settings=emb_settings)
except Exception as e:
raise HTTPException(status_code=502, detail=str(e)) from e
expected = int(cfg.get("rag_embedding_dimensions") or len(vector))
return EmbeddingTestResponse(
provider=emb_settings.provider,
model=emb_settings.model,
dimensions=len(vector),
expected_dimensions=expected,
ok=len(vector) == expected,
)
@@ -24,7 +24,12 @@ OPENVINO_WHISPER_SIZE_TO_REPO: dict[str, str] = {
"large-v3": "OpenVINO/whisper-large-v3-fp16-ov",
}
_SUPPORTED_MODEL_DOWNLOAD_ENGINES = {"faster-whisper", "openvino"}
EMBEDDING_MODEL_TO_REPO: dict[str, str] = {
"bge-small-zh-v1.5": "BAAI/bge-small-zh-v1.5",
"bge-m3": "BAAI/bge-m3",
}
_SUPPORTED_MODEL_DOWNLOAD_ENGINES = {"faster-whisper", "openvino", "embedding"}
def normalize_model_download_engine(engine: str | None) -> str:
@@ -35,6 +40,8 @@ def normalize_model_download_engine(engine: str | None) -> str:
def _engine_size_to_repo(engine: str) -> dict[str, str]:
if engine == "embedding":
return EMBEDDING_MODEL_TO_REPO
if engine == "openvino":
return OPENVINO_WHISPER_SIZE_TO_REPO
return FASTER_WHISPER_SIZE_TO_REPO
@@ -60,6 +67,9 @@ def default_model_dir_name(engine: str, model: str) -> str:
if engine_n == "openvino" and model_s in OPENVINO_WHISPER_SIZE_TO_REPO:
return OPENVINO_WHISPER_SIZE_TO_REPO[model_s].split("/", 1)[1]
if engine_n == "embedding" and model_s in EMBEDDING_MODEL_TO_REPO:
return EMBEDDING_MODEL_TO_REPO[model_s].split("/", 1)[1]
return model_s.replace("/", "--")
@@ -686,6 +686,7 @@ def translate_segments_openai_with_summary(
batch_size: int = 50,
enable_summary: bool = True,
glossary: dict[str, str] | None = None,
rag_context_provider: Callable[[list[Segment], int, str], dict[str, Any] | None] | None = None,
resume_from: Iterable[Segment] | None = None,
initial_summary: str = "",
on_batch_done: Callable[[list[Segment], str, int], None] | None = None,
@@ -726,6 +727,10 @@ def translate_segments_openai_with_summary(
payload_in["summary"] = summary
if glossary:
payload_in["glossary"] = glossary
if rag_context_provider is not None:
rag_context = rag_context_provider(batch, start_idx, summary)
if rag_context:
payload_in["rag_context"] = rag_context
user_prompt = (
"你将收到一批字幕 block。请按 block 为单位翻译。\n"
@@ -733,6 +738,8 @@ def translate_segments_openai_with_summary(
"- 保留每个 block 的 idx 不变;不得增删 block,不得改变顺序;\n"
"- 只翻译 text 字段;同一 block 内多行先合并理解再翻译;\n"
"- 术语、人名保持一致;数字/单位尽量保留原格式;\n"
"- 如果输入包含 rag_context,请优先参考其中的 term_cards/knowledge_cards 来理解专有名词、梗、作品设定和技术背景;\n"
"- term_cards 中的 translation 是推荐译法,除非明显不符合当前上下文,否则保持一致;\n"
"- 输出必须是 JSON 对象,且必须包含 translations 数组;不要输出任何解释。\n"
f"- 目标语言:{tgt}\n"
f"- 风格:{tone}\n\n"
@@ -859,6 +866,7 @@ def translate_segments_openai(
batch_size: int = 50,
enable_summary: bool = True,
glossary: dict[str, str] | None = None,
rag_context_provider: Callable[[list[Segment], int, str], dict[str, Any] | None] | None = None,
) -> list[Segment]:
out, _summary = translate_segments_openai_with_summary(
segments,
@@ -872,6 +880,7 @@ def translate_segments_openai(
batch_size=batch_size,
enable_summary=enable_summary,
glossary=glossary,
rag_context_provider=rag_context_provider,
)
return out
def _srt_ts(seconds: float) -> str:
File diff suppressed because it is too large. Load diff
@@ -202,6 +202,20 @@ class TranslateSettingsRead(BaseModel):
openai_temperature: float
openai_timeout_seconds: float
rag_enabled: bool = False
rag_top_k: int = 8
rag_min_score: float = 0.68
rag_embedding_provider: str = "openai"
rag_embedding_model: str = "text-embedding-3-small"
rag_embedding_dimensions: int = 1536
rag_embedding_model_dir: str = "/models/embeddings"
rag_embedding_device: str = "cpu"
rag_auto_discover_terms: bool = False
rag_auto_learn_terms: bool = False
rag_search_enabled: bool = False
rag_search_url: str = ""
rag_domain: str = ""
class TranslateSettingsUpdate(BaseModel):
default_provider: Optional[str] = None
@@ -218,6 +232,20 @@ class TranslateSettingsUpdate(BaseModel):
openai_temperature: Optional[float] = None
openai_timeout_seconds: Optional[float] = None
rag_enabled: Optional[bool] = None
rag_top_k: Optional[int] = Field(default=None, ge=0, le=30)
rag_min_score: Optional[float] = Field(default=None, ge=0.0, le=1.0)
rag_embedding_provider: Optional[str] = None
rag_embedding_model: Optional[str] = None
rag_embedding_dimensions: Optional[int] = Field(default=None, ge=1, le=4096)
rag_embedding_model_dir: Optional[str] = None
rag_embedding_device: Optional[str] = None
rag_auto_discover_terms: Optional[bool] = None
rag_auto_learn_terms: Optional[bool] = None
rag_search_enabled: Optional[bool] = None
rag_search_url: Optional[str] = None
rag_domain: Optional[str] = None
class TranslateTestRequest(BaseModel):
text: str = "Hello world."
@@ -229,6 +257,119 @@ class TranslateTestResponse(BaseModel):
translated_text: str
class KnowledgeItemRead(BaseModel):
id: uuid.UUID
item_type: str
term: str = ""
translation: str = ""
target_lang: str = "zh"
domain: str = ""
aliases: list[Any] = Field(default_factory=list)
title: str = ""
content: str = ""
description: str = ""
sources: list[Any] = Field(default_factory=list)
confidence: float = 0.0
status: str = "approved"
created_by: str = "manual"
usage_count: int = 0
embedding_model: str = ""
last_verified_at: Optional[datetime] = None
created_at: datetime
updated_at: datetime
class KnowledgeItemUpsertRequest(BaseModel):
item_type: Literal["term", "document"] = "term"
target_lang: str = "zh"
term: str = ""
translation: str = ""
domain: str = ""
aliases: list[str] = Field(default_factory=list)
title: str = ""
content: str = ""
description: str = ""
sources: list[dict[str, Any]] = Field(default_factory=list)
confidence: float = Field(default=1.0, ge=0.0, le=1.0)
status: str = "approved"
created_by: str = "manual"
class KnowledgeItemUpsertResponse(BaseModel):
id: uuid.UUID
class KnowledgeEmbeddingRebuildRequest(BaseModel):
item_type: Optional[Literal["term", "document"]] = None
status: Optional[str] = None
limit: int = Field(default=1000, ge=1, le=10000)
class KnowledgeEmbeddingRebuildResponse(BaseModel):
total: int
updated: int
failed: int
skipped: int
embedding_model: str
dimensions: int
errors: list[dict[str, str]] = Field(default_factory=list)
class AgentRunRead(BaseModel):
id: uuid.UUID
agent_type: str
status: str
term: str = ""
domain: str = ""
target_lang: str = "zh"
task_id: Optional[uuid.UUID] = None
subtitle_job_id: Optional[uuid.UUID] = None
query: str = ""
steps: list[Any] = Field(default_factory=list)
result: dict[str, Any] = Field(default_factory=dict)
error: str = ""
knowledge_item_id: Optional[uuid.UUID] = None
started_at: datetime
finished_at: Optional[datetime] = None
created_at: datetime
updated_at: datetime
class EmbeddingModelInfo(BaseModel):
name: str
path: str
size_bytes: Optional[int] = None
class EmbeddingModelListRequest(BaseModel):
model_dir: Optional[str] = None
class EmbeddingModelDownloadRequest(BaseModel):
model: str = "BAAI/bge-small-zh-v1.5"
name: Optional[str] = None
model_dir: Optional[str] = None
revision: Optional[str] = None
force: bool = False
class EmbeddingTestRequest(BaseModel):
text: str = "hello world"
provider: Optional[str] = None
model: Optional[str] = None
model_dir: Optional[str] = None
dimensions: Optional[int] = None
device: Optional[str] = None
class EmbeddingTestResponse(BaseModel):
provider: str
model: str
dimensions: int
expected_dimensions: int
ok: bool
class WhisperModelInfo(BaseModel):
name: str
path: str
@@ -56,6 +56,21 @@ def get_translate_settings(db: Session, defaults: SubtitleServiceSettings) -> di
"openai_model": str(openai.get("model") or defaults.openai_model),
"openai_temperature": float(openai.get("temperature") or defaults.openai_temperature),
"openai_timeout_seconds": float(openai.get("timeout_seconds") or defaults.openai_timeout_seconds),
"rag_enabled": bool(stored.get("rag_enabled") if "rag_enabled" in stored else defaults.rag_enabled),
"rag_top_k": int(stored.get("rag_top_k") or defaults.rag_top_k),
"rag_min_score": float(stored.get("rag_min_score") or defaults.rag_min_score),
"rag_embedding_provider": str(stored.get("rag_embedding_provider") or defaults.rag_embedding_provider),
"rag_embedding_model": str(stored.get("rag_embedding_model") or defaults.rag_embedding_model),
"rag_embedding_dimensions": int(stored.get("rag_embedding_dimensions") or defaults.rag_embedding_dimensions),
"rag_embedding_model_dir": str(stored.get("rag_embedding_model_dir") or defaults.rag_embedding_model_dir),
"rag_embedding_device": str(stored.get("rag_embedding_device") or defaults.rag_embedding_device),
"rag_auto_discover_terms": bool(
stored.get("rag_auto_discover_terms") if "rag_auto_discover_terms" in stored else defaults.rag_auto_discover_terms
),
"rag_auto_learn_terms": bool(stored.get("rag_auto_learn_terms") if "rag_auto_learn_terms" in stored else defaults.rag_auto_learn_terms),
"rag_search_enabled": bool(stored.get("rag_search_enabled") if "rag_search_enabled" in stored else defaults.rag_search_enabled),
"rag_search_url": str(stored.get("rag_search_url") or defaults.rag_search_url),
"rag_domain": str(stored.get("rag_domain") or defaults.rag_domain),
}
@@ -70,6 +85,19 @@ def update_translate_settings(db: Session, defaults: SubtitleServiceSettings, up
"default_batch_size",
"default_max_retries",
"default_enable_summary",
"rag_enabled",
"rag_top_k",
"rag_min_score",
"rag_embedding_provider",
"rag_embedding_model",
"rag_embedding_dimensions",
"rag_embedding_model_dir",
"rag_embedding_device",
"rag_auto_discover_terms",
"rag_auto_learn_terms",
"rag_search_enabled",
"rag_search_url",
"rag_domain",
]:
if key not in update:
continue
@@ -120,6 +148,26 @@ def update_translate_settings(db: Session, defaults: SubtitleServiceSettings, up
stored["default_max_retries"] = max(0, min(10, int(stored["default_max_retries"])))
except Exception:
stored["default_max_retries"] = defaults.translate_max_retries
try:
stored["rag_top_k"] = max(0, min(30, int(stored.get("rag_top_k") or defaults.rag_top_k)))
except Exception:
stored["rag_top_k"] = defaults.rag_top_k
try:
stored["rag_min_score"] = max(0.0, min(1.0, float(stored.get("rag_min_score") or defaults.rag_min_score)))
except Exception:
stored["rag_min_score"] = defaults.rag_min_score
try:
stored["rag_embedding_dimensions"] = max(1, min(4096, int(stored.get("rag_embedding_dimensions") or defaults.rag_embedding_dimensions)))
except Exception:
stored["rag_embedding_dimensions"] = defaults.rag_embedding_dimensions
for bool_key in ["rag_enabled", "rag_auto_discover_terms", "rag_auto_learn_terms", "rag_search_enabled"]:
if bool_key in stored:
stored[bool_key] = bool(stored[bool_key])
provider = str(stored.get("rag_embedding_provider") or defaults.rag_embedding_provider).strip().lower()
stored["rag_embedding_provider"] = provider if provider in {"openai", "local"} else defaults.rag_embedding_provider
for str_key in ["rag_embedding_model", "rag_embedding_model_dir", "rag_embedding_device", "rag_search_url", "rag_domain"]:
if str_key in stored and stored[str_key] is not None:
stored[str_key] = str(stored[str_key]).strip()
try:
if "openai_temperature" in update and update["openai_temperature"] is not None:
openai["temperature"] = float(openai.get("temperature"))
@@ -74,6 +74,8 @@ from videoroll.apps.subtitle_service.model_downloads import (
download_model_snapshot,
resolve_model_repo_id,
)
from videoroll.apps.subtitle_service.rag import build_rag_context, rag_settings_from_translate_settings
from videoroll.apps.subtitle_service.embeddings import embedding_settings_from_translate_settings
from videoroll.apps.subtitle_service.task_title_store import set_task_titles
from videoroll.apps.subtitle_service.translate_settings_store import get_translate_settings
from videoroll.apps.publish_meta_draft import apply_publish_source_overrides, default_publish_meta
@@ -1358,6 +1360,44 @@ def process_job(self: Any, job_id: str) -> dict[str, str]:
)
checkpoint_segments = list(resume_prefix)
rag_settings = rag_settings_from_translate_settings(translate_settings)
rag_chat_config = openai_chat_config_from_settings(translate_settings)
rag_embedding_settings = embedding_settings_from_translate_settings(translate_settings)
if rag_settings.enabled:
_safe_append_log_line(
log_path,
"translate rag: "
f"enabled top_k={rag_settings.top_k} min_score={rag_settings.min_score} "
f"embedding_provider={rag_settings.embedding_provider} embedding_model={rag_settings.embedding_model} "
f"domain={rag_settings.domain or '(any)'}",
)
def _rag_context_provider(batch_segments: list[Segment], start_idx: int, summary: str) -> dict[str, Any] | None:
del start_idx, summary
if not rag_settings.enabled:
return None
ctx = build_rag_context(
db,
segments=batch_segments,
target_lang=target_lang,
rag_settings=rag_settings,
embedding_settings=rag_embedding_settings,
chat_config=rag_chat_config,
task_id=str(task.id),
subtitle_job_id=str(job.id),
)
if ctx.hits:
try:
db.commit()
except Exception:
db.rollback()
if not ctx.term_cards and not ctx.knowledge_cards:
return None
return {
"term_cards": ctx.term_cards,
"knowledge_cards": ctx.knowledge_cards,
}
def _on_translate_batch_done(batch_segments: list[Segment], updated_summary: str, completed_count: int) -> None:
del completed_count
@@ -1375,6 +1415,7 @@ def process_job(self: Any, job_id: str) -> dict[str, str]:
timeout_seconds=translate_settings["openai_timeout_seconds"],
batch_size=batch_size,
enable_summary=enable_summary,
rag_context_provider=_rag_context_provider,
resume_from=resume_prefix,
initial_summary=resume_summary,
on_batch_done=_on_translate_batch_done,
+14
View File
@@ -81,6 +81,20 @@ class SubtitleServiceSettings(CommonSettings):
openai_temperature: float = Field(0.2, alias="OPENAI_TEMPERATURE")
openai_timeout_seconds: float = Field(180.0, alias="OPENAI_TIMEOUT_SECONDS")
rag_enabled: bool = Field(False, alias="SUBTITLE_RAG_ENABLED")
rag_top_k: int = Field(8, alias="SUBTITLE_RAG_TOP_K")
rag_min_score: float = Field(0.68, alias="SUBTITLE_RAG_MIN_SCORE")
rag_embedding_provider: str = Field("openai", alias="SUBTITLE_RAG_EMBEDDING_PROVIDER")
rag_embedding_model: str = Field("text-embedding-3-small", alias="SUBTITLE_RAG_EMBEDDING_MODEL")
rag_embedding_dimensions: int = Field(1536, alias="SUBTITLE_RAG_EMBEDDING_DIMENSIONS")
rag_embedding_model_dir: str = Field("/models/embeddings", alias="SUBTITLE_RAG_EMBEDDING_MODEL_DIR")
rag_embedding_device: str = Field("cpu", alias="SUBTITLE_RAG_EMBEDDING_DEVICE")
rag_auto_discover_terms: bool = Field(False, alias="SUBTITLE_RAG_AUTO_DISCOVER_TERMS")
rag_auto_learn_terms: bool = Field(False, alias="SUBTITLE_RAG_AUTO_LEARN_TERMS")
rag_search_enabled: bool = Field(False, alias="SUBTITLE_RAG_SEARCH_ENABLED")
rag_search_url: str = Field("", alias="SUBTITLE_RAG_SEARCH_URL")
rag_domain: str = Field("", alias="SUBTITLE_RAG_DOMAIN")
# Orchestrator API (used by subtitle worker for auto pipelines).
orchestrator_url: str = Field("http://localhost:8000", alias="ORCHESTRATOR_URL")
orchestrator_timeout_seconds: float = Field(1800.0, alias="ORCHESTRATOR_TIMEOUT_SECONDS")
+175 -1
View File
@@ -91,6 +91,175 @@ def _ensure_youtube_sources_columns(engine: Engine) -> None:
logger.warning("auto-migrated DB: added youtube_sources.%s", column)
def _ensure_pgvector_rag_tables(engine: Engine) -> None:
dialect = (engine.dialect.name or "").lower()
if dialect != "postgresql":
logger.warning("RAG tables require PostgreSQL/pgvector; skipping for dialect=%s", dialect)
return
with engine.connect() as conn:
vector_type = conn.execute(text("SELECT to_regtype('vector')")).scalar()
try:
conn.commit()
except Exception:
pass
if vector_type is None:
try:
with engine.begin() as ext_conn:
ext_conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
except Exception as e:
logger.warning("pgvector extension is unavailable; skipping RAG table migration: %s", e)
return
with engine.begin() as conn:
conn.execute(
text(
f"""
CREATE TABLE IF NOT EXISTS translation_knowledge_items (
id UUID PRIMARY KEY,
item_type VARCHAR(32) NOT NULL DEFAULT 'document',
term TEXT NOT NULL DEFAULT '',
normalized_term TEXT NOT NULL DEFAULT '',
translation TEXT NOT NULL DEFAULT '',
target_lang VARCHAR(16) NOT NULL DEFAULT 'zh',
domain TEXT NOT NULL DEFAULT '',
aliases JSONB NOT NULL DEFAULT '[]'::jsonb,
title TEXT NOT NULL DEFAULT '',
content TEXT NOT NULL DEFAULT '',
description TEXT NOT NULL DEFAULT '',
sources JSONB NOT NULL DEFAULT '[]'::jsonb,
confidence DOUBLE PRECISION NOT NULL DEFAULT 0,
status VARCHAR(32) NOT NULL DEFAULT 'approved',
created_by VARCHAR(32) NOT NULL DEFAULT 'manual',
usage_count INTEGER NOT NULL DEFAULT 0,
embedding vector,
embedding_model TEXT NOT NULL DEFAULT '',
embedding_text_hash VARCHAR(64) NOT NULL DEFAULT '',
last_verified_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""
)
)
try:
conn.execute(text("ALTER TABLE translation_knowledge_items ALTER COLUMN embedding TYPE vector"))
except Exception:
pass
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS translation_term_evidence (
id UUID PRIMARY KEY,
term TEXT NOT NULL DEFAULT '',
normalized_term TEXT NOT NULL DEFAULT '',
domain TEXT NOT NULL DEFAULT '',
query TEXT NOT NULL DEFAULT '',
results JSONB NOT NULL DEFAULT '[]'::jsonb,
summary TEXT NOT NULL DEFAULT '',
sources JSONB NOT NULL DEFAULT '[]'::jsonb,
confidence DOUBLE PRECISION NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""
)
)
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS translation_term_matches (
id UUID PRIMARY KEY,
task_id UUID,
subtitle_job_id UUID,
knowledge_item_id UUID REFERENCES translation_knowledge_items(id) ON DELETE SET NULL,
term TEXT NOT NULL DEFAULT '',
normalized_term TEXT NOT NULL DEFAULT '',
segment_start DOUBLE PRECISION,
segment_end DOUBLE PRECISION,
raw_context TEXT NOT NULL DEFAULT '',
decision TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""
)
)
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS translation_agent_runs (
id UUID PRIMARY KEY,
agent_type VARCHAR(64) NOT NULL DEFAULT 'rag_term_research',
status VARCHAR(32) NOT NULL DEFAULT 'running',
term TEXT NOT NULL DEFAULT '',
normalized_term TEXT NOT NULL DEFAULT '',
domain TEXT NOT NULL DEFAULT '',
target_lang VARCHAR(16) NOT NULL DEFAULT 'zh',
task_id UUID,
subtitle_job_id UUID,
query TEXT NOT NULL DEFAULT '',
steps JSONB NOT NULL DEFAULT '[]'::jsonb,
result JSONB NOT NULL DEFAULT '{}'::jsonb,
error TEXT NOT NULL DEFAULT '',
knowledge_item_id UUID REFERENCES translation_knowledge_items(id) ON DELETE SET NULL,
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
finished_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""
)
)
conn.execute(
text(
"""
CREATE UNIQUE INDEX IF NOT EXISTS uq_translation_terms_lang_domain_norm
ON translation_knowledge_items (target_lang, domain, normalized_term)
WHERE item_type = 'term' AND normalized_term <> ''
"""
)
)
conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS ix_translation_agent_runs_status_updated
ON translation_agent_runs (status, updated_at DESC)
"""
)
)
conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS ix_translation_agent_runs_term
ON translation_agent_runs (target_lang, domain, normalized_term)
"""
)
)
conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS ix_translation_knowledge_type_status
ON translation_knowledge_items (item_type, status, target_lang)
"""
)
)
conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS ix_translation_knowledge_domain_norm
ON translation_knowledge_items (domain, normalized_term)
"""
)
)
conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS ix_translation_matches_task
ON translation_term_matches (task_id, subtitle_job_id)
"""
)
)
def auto_migrate_engine(engine: Engine) -> None:
"""
Small, targeted "auto migration" for deployments without Alembic.
@@ -100,6 +269,7 @@ def auto_migrate_engine(engine: Engine) -> None:
"""
_ensure_tasks_lock_columns(engine)
_ensure_youtube_sources_columns(engine)
_ensure_pgvector_rag_tables(engine)
@lru_cache
@@ -111,5 +281,9 @@ def _auto_migrate_cached(database_url: str, pid: int) -> None:
auto_migrate_engine(engine)
def auto_migrate(database_url: str) -> None:
def auto_migrate(database_url: str, *, force: bool = False) -> None:
if force:
engine = get_engine(database_url)
auto_migrate_engine(engine)
return
_auto_migrate_cached(database_url, os.getpid())
+78
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import os
import shutil
import subprocess
import time
from pathlib import Path
from typing import Any
@@ -36,6 +37,56 @@ def detect_intel_hardware(render_device: str) -> dict[str, Any]:
return _probe_intel_sysfs(sysfs_dir, device_path)
def sample_intel_gpu_usage(render_device: str, *, interval_seconds: float = 0.1) -> dict[str, Any]:
hardware = detect_intel_hardware(render_device)
out: dict[str, Any] = {
**hardware,
"usage_supported": False,
"usage_percent": None,
"engines": [],
}
if not bool(hardware.get("available")):
return out
sysfs_dir = _render_sysfs_device_dir(Path(str(render_device or "").strip() or "/dev/dri/renderD128"))
if sysfs_dir is None:
out["detail"] = "未找到渲染设备的 sysfs 信息"
return out
first = _read_engine_busy_snapshot(sysfs_dir)
if not first:
out["detail"] = f"{hardware.get('detail') or '已检测到 Intel 硬件'};未找到 GPU busy 计数器"
return out
interval = max(0.02, float(interval_seconds))
time.sleep(interval)
second = _read_engine_busy_snapshot(sysfs_dir)
if not second:
out["detail"] = f"{hardware.get('detail') or '已检测到 Intel 硬件'};GPU busy 计数器不可读"
return out
elapsed_us = interval * 1_000_000.0
engines: list[dict[str, Any]] = []
max_percent = 0.0
for name, start_busy in first.items():
end_busy = second.get(name)
if end_busy is None:
continue
delta = max(0, end_busy - start_busy)
percent = max(0.0, min(100.0, (delta / elapsed_us) * 100.0))
max_percent = max(max_percent, percent)
engines.append({"name": name, "percent": percent})
if not engines:
out["detail"] = f"{hardware.get('detail') or '已检测到 Intel 硬件'};GPU busy 计数器无有效数据"
return out
out["usage_supported"] = True
out["usage_percent"] = max_percent
out["engines"] = sorted(engines, key=lambda item: str(item.get("name") or ""))
return out
def _probe_intel_sysfs(sysfs_dir: Path, render_device: str) -> dict[str, Any]:
result: dict[str, Any] = {
"checked": True,
@@ -72,6 +123,33 @@ def _probe_intel_sysfs(sysfs_dir: Path, render_device: str) -> dict[str, Any]:
return result
def _read_engine_busy_snapshot(sysfs_dir: Path) -> dict[str, int]:
engine_root = sysfs_dir / "engine"
out: dict[str, int] = {}
if engine_root.exists():
for busy_path in engine_root.glob("*/busy"):
raw = _read_text(busy_path)
try:
value = int(str(raw or "").strip())
except Exception:
continue
name = busy_path.parent.name.strip() or "engine"
out[name] = value
if out:
return out
# Some kernels expose aggregate busy counters under the device directory.
for name in ["busy", "gt/gt0/rps_busy", "gt/gt0/busy"]:
path = sysfs_dir / name
raw = _read_text(path)
try:
value = int(str(raw or "").strip())
except Exception:
continue
out[name.replace("/", ".")] = value
return out
def _render_sysfs_device_dir(node: Path) -> Path | None:
name = node.name
if name:
+17
View File
@@ -55,3 +55,20 @@ def build_openai_chat_completions_url(base_url: str) -> str:
base = normalize_openai_base_url(raw)
return base.rstrip("/") + "/chat/completions"
def build_openai_embeddings_url(base_url: str) -> str:
"""
Accepts either:
- a base URL (e.g. https://api.openai.com/v1), or
- a full endpoint URL ending with /embeddings.
"""
raw = (base_url or "").strip()
if not raw:
raw = DEFAULT_OPENAI_BASE_URL
raw = _ensure_scheme(raw).rstrip("/")
if raw.endswith("/embeddings"):
return raw
base = normalize_openai_base_url(raw)
return base.rstrip("/") + "/embeddings"
+147
View File
@@ -0,0 +1,147 @@
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from videoroll.utils.cpu import process_cpu_count
def _read_text(path: Path) -> str | None:
try:
return path.read_text(encoding="utf-8").strip()
except Exception:
return None
def read_cpu_snapshot() -> tuple[int, int] | None:
raw = _read_text(Path("/proc/stat"))
if not raw:
return None
first = raw.splitlines()[0].split()
if len(first) < 5 or first[0] != "cpu":
return None
try:
values = [int(x) for x in first[1:]]
except Exception:
return None
idle = values[3] + (values[4] if len(values) > 4 else 0)
return sum(values), idle
def cpu_percent_between(prev: tuple[int, int], curr: tuple[int, int]) -> float | None:
total_delta = curr[0] - prev[0]
idle_delta = curr[1] - prev[1]
if total_delta <= 0:
return None
busy = max(0, total_delta - max(0, idle_delta))
return max(0.0, min(100.0, (busy / total_delta) * 100.0))
def sample_cpu_percent(*, interval_seconds: float = 0.05) -> float | None:
prev = read_cpu_snapshot()
if prev is None:
return None
time.sleep(max(0.0, float(interval_seconds)))
curr = read_cpu_snapshot()
if curr is None:
return None
return cpu_percent_between(prev, curr)
def read_load_average() -> list[float] | None:
try:
return [float(x) for x in os.getloadavg()]
except Exception:
return None
def _parse_meminfo(raw: str) -> dict[str, int]:
out: dict[str, int] = {}
for line in raw.splitlines():
if ":" not in line:
continue
key, rest = line.split(":", 1)
parts = rest.strip().split()
if not parts:
continue
try:
value = int(parts[0])
except Exception:
continue
unit = parts[1].lower() if len(parts) > 1 else ""
out[key] = value * 1024 if unit == "kb" else value
return out
def memory_stats_from_meminfo(raw: str) -> dict[str, int | float] | None:
data = _parse_meminfo(raw)
total = int(data.get("MemTotal") or 0)
if total <= 0:
return None
available = int(data.get("MemAvailable") or 0)
if available <= 0:
available = int(data.get("MemFree") or 0) + int(data.get("Buffers") or 0) + int(data.get("Cached") or 0)
available = max(0, min(total, available))
used = max(0, total - available)
return {
"total_bytes": total,
"used_bytes": used,
"available_bytes": available,
"percent": (used / total) * 100.0,
}
def read_memory_stats() -> dict[str, int | float] | None:
raw = _read_text(Path("/proc/meminfo"))
if not raw:
return None
return memory_stats_from_meminfo(raw)
def read_cgroup_memory_stats() -> dict[str, int | float] | None:
# cgroup v2
current = _read_int(Path("/sys/fs/cgroup/memory.current"))
max_raw = _read_text(Path("/sys/fs/cgroup/memory.max"))
total = None if max_raw in {None, "", "max"} else _parse_positive_int(max_raw)
if current is not None and total is not None and total > 0:
return _memory_stats_from_usage(current, total)
# cgroup v1
current = _read_int(Path("/sys/fs/cgroup/memory/memory.usage_in_bytes"))
total = _read_int(Path("/sys/fs/cgroup/memory/memory.limit_in_bytes"))
if current is not None and total is not None and 0 < total < 1 << 60:
return _memory_stats_from_usage(current, total)
return None
def _read_int(path: Path) -> int | None:
return _parse_positive_int(_read_text(path))
def _parse_positive_int(raw: str | None) -> int | None:
try:
n = int(str(raw or "").strip())
except Exception:
return None
return n if n >= 0 else None
def _memory_stats_from_usage(used: int, total: int) -> dict[str, int | float]:
used = max(0, min(total, int(used)))
available = max(0, total - used)
return {
"total_bytes": total,
"used_bytes": used,
"available_bytes": available,
"percent": (used / total) * 100.0 if total > 0 else 0.0,
}
def process_cpu_summary() -> dict[str, Any]:
return {
"percent": sample_cpu_percent(),
"cores": process_cpu_count(),
"load_average": read_load_average(),
}
+10 -1
View File
@@ -3,11 +3,20 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="color-scheme" content="light dark" />
<title>VideoRoll</title>
<script>
(() => {
try {
const stored = localStorage.getItem("videoroll-theme");
const dark = stored === "dark" || (!stored && matchMedia("(prefers-color-scheme: dark)").matches);
document.documentElement.classList.toggle("dark", dark);
} catch {}
})();
</script>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>
+67 -25
View File
@@ -1,6 +1,7 @@
import { useState } from "react";
import { ReactNode, useEffect, useState } from "react";
import { Link, NavLink, Route, Routes } from "react-router-dom";
import AuthGate from "./components/AuthGate";
import { FeedbackProvider } from "./components/Feedback";
import { fetchJson } from "./lib/http";
import { ORCHESTRATOR_URL } from "./lib/urls";
import DashboardPage from "./pages/DashboardPage";
@@ -25,8 +26,10 @@ function NavItem({ to, label }: { to: string; label: string }) {
to={to}
className={({ isActive }) =>
[
"block rounded px-3 py-2 text-sm",
isActive ? "bg-slate-900 text-white" : "text-slate-700 hover:bg-slate-100",
"block rounded-md px-3 py-2 text-sm",
isActive
? "bg-slate-900 text-white dark:bg-slate-100 dark:text-slate-950"
: "text-slate-700 hover:bg-slate-100 dark:text-slate-300 dark:hover:bg-slate-800",
].join(" ")
}
>
@@ -35,6 +38,15 @@ function NavItem({ to, label }: { to: string; label: string }) {
);
}
function NavGroup({ title, children }: { title: string; children: ReactNode }) {
return (
<div>
<div className="px-3 pb-1 pt-2 text-[11px] font-semibold uppercase tracking-normal text-slate-400">{title}</div>
<div className="space-y-1">{children}</div>
</div>
);
}
export default function App() {
const orchestratorDisplay =
ORCHESTRATOR_URL.startsWith("http://") || ORCHESTRATOR_URL.startsWith("https://")
@@ -44,6 +56,19 @@ export default function App() {
: ORCHESTRATOR_URL;
const [loggingOut, setLoggingOut] = useState(false);
const [darkMode, setDarkMode] = useState(() => {
if (typeof window === "undefined") return false;
const stored = window.localStorage.getItem("videoroll-theme");
if (stored === "dark") return true;
if (stored === "light") return false;
return window.matchMedia?.("(prefers-color-scheme: dark)").matches ?? false;
});
useEffect(() => {
const root = document.documentElement;
root.classList.toggle("dark", darkMode);
window.localStorage.setItem("videoroll-theme", darkMode ? "dark" : "light");
}, [darkMode]);
async function logout() {
if (loggingOut) return;
@@ -60,22 +85,32 @@ export default function App() {
}
return (
<AuthGate>
<div className="min-h-screen bg-slate-50">
<header className="border-b bg-white">
<FeedbackProvider>
<AuthGate>
<div className="min-h-screen bg-slate-50 transition-colors dark:bg-slate-950">
<header className="border-b bg-white transition-colors dark:border-slate-800 dark:bg-slate-950">
<div className="mx-auto flex max-w-6xl items-center justify-between px-4 py-3">
<Link to="/" className="font-semibold">
VideoRoll
</Link>
<div className="flex items-center gap-3">
<div className="text-xs text-slate-500">合规版 · MVP</div>
<button
type="button"
onClick={() => setDarkMode((value) => !value)}
className="inline-flex h-8 w-8 items-center justify-center rounded-md border border-slate-300 text-sm text-slate-700 hover:bg-slate-50 dark:border-slate-700 dark:text-slate-200 dark:hover:bg-slate-900"
title={darkMode ? "切换到浅色模式" : "切换到黑暗模式"}
aria-label={darkMode ? "切换到浅色模式" : "切换到黑暗模式"}
>
{darkMode ? "☀" : "☾"}
</button>
<button
type="button"
onClick={logout}
disabled={loggingOut}
className={[
"rounded border px-2 py-1 text-xs",
loggingOut ? "cursor-not-allowed bg-slate-100 text-slate-400" : "hover:bg-slate-50",
"rounded border px-2 py-1 text-xs dark:border-slate-700 dark:text-slate-200",
loggingOut ? "cursor-not-allowed bg-slate-100 text-slate-400 dark:bg-slate-900 dark:text-slate-500" : "hover:bg-slate-50 dark:hover:bg-slate-900",
].join(" ")}
>
{loggingOut ? "退出中…" : "退出"}
@@ -89,23 +124,29 @@ export default function App() {
<div className="mx-auto grid max-w-6xl grid-cols-12 gap-4 px-4 py-4">
<aside className="col-span-12 md:col-span-3">
<div className="rounded border bg-white p-2">
<NavItem to="/" label="Dashboard" />
<NavItem to="/tasks" label="Tasks" />
<NavItem to="/videos" label="Videos" />
<NavItem to="/tasks/new" label="New Task" />
<NavItem to="/youtube/sources" label="YouTube Sources" />
<NavItem to="/queue/render" label="Queue · Task" />
<NavItem to="/settings/asr" label="Settings · ASR" />
<NavItem to="/settings/youtube" label="Settings · YouTube" />
<NavItem to="/settings/storage" label="Settings · Storage" />
<NavItem to="/settings/api" label="Settings · API" />
<NavItem to="/settings/review" label="Settings · Review" />
<NavItem to="/settings/auto" label="Settings · Auto" />
<NavItem to="/settings/translate" label="Settings · Translate" />
<NavItem to="/settings/bilibili" label="Settings · Bilibili" />
<div className="rounded-md border border-slate-200 bg-white p-2 shadow-sm dark:border-slate-800 dark:bg-slate-900">
<NavGroup title="工作台">
<NavItem to="/" label="Dashboard" />
<NavItem to="/tasks" label="Tasks" />
<NavItem to="/videos" label="Videos" />
<NavItem to="/queue/render" label="Queue · Task" />
</NavGroup>
<NavGroup title="来源">
<NavItem to="/tasks/new" label="New Task" />
<NavItem to="/youtube/sources" label="YouTube Sources" />
</NavGroup>
<NavGroup title="配置">
<NavItem to="/settings/auto" label="Settings · Auto" />
<NavItem to="/settings/youtube" label="Settings · YouTube" />
<NavItem to="/settings/bilibili" label="Settings · Bilibili" />
<NavItem to="/settings/asr" label="Settings · ASR" />
<NavItem to="/settings/translate" label="Settings · Translate" />
<NavItem to="/settings/review" label="Settings · Review" />
<NavItem to="/settings/storage" label="Settings · Storage" />
<NavItem to="/settings/api" label="Settings · API" />
</NavGroup>
</div>
<div className="mt-3 rounded border bg-white p-3 text-xs text-slate-600">
<div className="mt-3 rounded-md border border-slate-200 bg-white p-3 text-xs text-slate-600 shadow-sm dark:border-slate-800 dark:bg-slate-900 dark:text-slate-400">
后端(Orchestrator):{orchestratorDisplay}
</div>
</aside>
@@ -131,6 +172,7 @@ export default function App() {
</main>
</div>
</div>
</AuthGate>
</AuthGate>
</FeedbackProvider>
);
}
+104
View File
@@ -0,0 +1,104 @@
import { PropsWithChildren, ReactNode, useCallback, useMemo, useState } from "react";
import { ConfirmOptions, FeedbackContext, Toast, ToastKind } from "./feedbackContext";
type ConfirmState = ConfirmOptions & {
resolve: (value: boolean) => void;
};
const toastClasses: Record<ToastKind, string> = {
success: "border-emerald-200 bg-emerald-50 text-emerald-900",
error: "border-rose-200 bg-rose-50 text-rose-900",
info: "border-sky-200 bg-sky-50 text-sky-900",
warning: "border-amber-200 bg-amber-50 text-amber-950",
};
function clampToastText(value: ReactNode): ReactNode {
if (typeof value !== "string") return value;
const text = value.trim();
return text.length > 700 ? `${text.slice(0, 699)}...` : text;
}
export function FeedbackProvider({ children }: PropsWithChildren) {
const [toasts, setToasts] = useState<Toast[]>([]);
const [confirmState, setConfirmState] = useState<ConfirmState | null>(null);
const dismissToast = useCallback((id: number) => {
setToasts((items) => items.filter((item) => item.id !== id));
}, []);
const toast = useCallback(
(next: Omit<Toast, "id">) => {
const id = Date.now() + Math.floor(Math.random() * 10000);
setToasts((items) => [...items, { ...next, id }].slice(-4));
window.setTimeout(() => dismissToast(id), next.kind === "error" ? 9000 : 5000);
},
[dismissToast],
);
const confirm = useCallback((options: ConfirmOptions) => {
return new Promise<boolean>((resolve) => {
setConfirmState({ ...options, resolve });
});
}, []);
const value = useMemo(() => ({ toast, confirm }), [toast, confirm]);
function closeConfirm(result: boolean) {
const current = confirmState;
if (!current) return;
setConfirmState(null);
current.resolve(result);
}
const confirmTone = confirmState?.tone ?? "default";
const confirmButtonClass =
confirmTone === "danger"
? "bg-rose-700 text-white hover:bg-rose-800"
: confirmTone === "warning"
? "bg-amber-600 text-white hover:bg-amber-700"
: "bg-slate-900 text-white hover:bg-slate-800";
return (
<FeedbackContext.Provider value={value}>
{children}
<div className="pointer-events-none fixed right-4 top-4 z-50 flex w-[min(24rem,calc(100vw-2rem))] flex-col gap-2">
{toasts.map((item) => (
<div key={item.id} className={`pointer-events-auto rounded-md border p-3 shadow-lg ${toastClasses[item.kind]}`}>
<div className="flex items-start justify-between gap-3">
<div>
<div className="text-sm font-semibold">{item.title}</div>
{item.message ? <div className="mt-1 whitespace-pre-wrap break-words text-xs">{clampToastText(item.message)}</div> : null}
</div>
<button
type="button"
className="rounded px-1 text-xs opacity-70 hover:bg-white/60 hover:opacity-100"
onClick={() => dismissToast(item.id)}
aria-label="Dismiss"
>
x
</button>
</div>
</div>
))}
</div>
{confirmState ? (
<div className="fixed inset-0 z-40 flex items-center justify-center bg-slate-950/40 px-4 py-6">
<div className="w-full max-w-md rounded-md border border-slate-200 bg-white p-4 shadow-xl">
<div className="text-base font-semibold text-slate-950">{confirmState.title}</div>
{confirmState.message ? <div className="mt-2 whitespace-pre-wrap break-words text-sm text-slate-700">{confirmState.message}</div> : null}
<div className="mt-4 flex justify-end gap-2">
<button type="button" className="rounded-md border px-3 py-2 text-sm hover:bg-slate-50" onClick={() => closeConfirm(false)}>
{confirmState.cancelLabel ?? "取消"}
</button>
<button type="button" className={`rounded-md px-3 py-2 text-sm ${confirmButtonClass}`} onClick={() => closeConfirm(true)}>
{confirmState.confirmLabel ?? "确认"}
</button>
</div>
</div>
</div>
) : null}
</FeedbackContext.Provider>
);
}
+37
View File
@@ -0,0 +1,37 @@
import { createContext, ReactNode, useContext } from "react";
export type ToastKind = "success" | "error" | "info" | "warning";
export type Toast = {
id: number;
kind: ToastKind;
title: string;
message?: ReactNode;
};
export type ConfirmOptions = {
title: string;
message?: ReactNode;
confirmLabel?: string;
cancelLabel?: string;
tone?: "default" | "danger" | "warning";
};
export type FeedbackContextValue = {
toast: (toast: Omit<Toast, "id">) => void;
confirm: (options: ConfirmOptions) => Promise<boolean>;
};
export const FeedbackContext = createContext<FeedbackContextValue | null>(null);
export function useToast() {
const ctx = useContext(FeedbackContext);
if (!ctx) throw new Error("useToast must be used inside FeedbackProvider");
return ctx.toast;
}
export function useConfirm() {
const ctx = useContext(FeedbackContext);
if (!ctx) throw new Error("useConfirm must be used inside FeedbackProvider");
return ctx.confirm;
}
+74
View File
@@ -0,0 +1,74 @@
import { ButtonHTMLAttributes, PropsWithChildren, ReactNode } from "react";
type ButtonTone = "primary" | "secondary" | "danger" | "warning" | "ghost";
type ButtonSize = "sm" | "xs";
const toneClasses: Record<ButtonTone, string> = {
primary: "border-slate-900 bg-slate-900 text-white hover:bg-slate-800",
secondary: "border-slate-300 bg-white text-slate-800 hover:bg-slate-50",
danger: "border-rose-300 bg-white text-rose-700 hover:bg-rose-50",
warning: "border-amber-300 bg-white text-amber-800 hover:bg-amber-50",
ghost: "border-transparent bg-transparent text-slate-700 hover:bg-slate-100",
};
const sizeClasses: Record<ButtonSize, string> = {
sm: "px-3 py-2 text-sm",
xs: "px-2 py-1 text-xs",
};
type ButtonProps = ButtonHTMLAttributes<HTMLButtonElement> & {
tone?: ButtonTone;
size?: ButtonSize;
};
export function Button({ tone = "secondary", size = "sm", className = "", ...props }: ButtonProps) {
return (
<button
{...props}
className={[
"inline-flex items-center justify-center rounded-md border font-medium transition-colors disabled:cursor-not-allowed disabled:opacity-50",
toneClasses[tone],
sizeClasses[size],
className,
].join(" ")}
/>
);
}
export function PageHeader({
title,
description,
actions,
}: {
title: ReactNode;
description?: ReactNode;
actions?: ReactNode;
}) {
return (
<div className="vr-section">
<div className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
<div>
<div className="text-lg font-semibold text-slate-950">{title}</div>
{description ? <div className="mt-1 text-sm text-slate-600">{description}</div> : null}
</div>
{actions ? <div className="flex flex-wrap items-center gap-2">{actions}</div> : null}
</div>
</div>
);
}
export function Section({ children, className = "" }: PropsWithChildren<{ className?: string }>) {
return <div className={`vr-section ${className}`}>{children}</div>;
}
export function DataTable({ children }: PropsWithChildren) {
return (
<div className="vr-table-wrap">
<table className="vr-table">{children}</table>
</div>
);
}
export function EmptyState({ children }: PropsWithChildren) {
return <div className="py-6 text-center text-sm text-slate-500">{children}</div>;
}
+538 -21
View File
@@ -1,9 +1,10 @@
import { useEffect, useMemo, useState } from "react";
import { Link } from "react-router-dom";
import { fetchJson } from "../lib/http";
import { ORCHESTRATOR_URL } from "../lib/urls";
import { ORCHESTRATOR_URL, SUBTITLE_SERVICE_URL } from "../lib/urls";
import StatusBadge from "../components/StatusBadge";
import { Asset, Task } from "../lib/types";
import { PageHeader } from "../components/ui";
type ConvertedVideoItem = {
task: Task;
@@ -11,16 +12,311 @@ type ConvertedVideoItem = {
cover_asset?: Asset | null;
};
type ResourceMemory = {
total_bytes: number;
used_bytes: number;
available_bytes: number;
percent?: number | null;
};
type ResourceSnapshot = {
sampled_at: string;
cpu: {
percent?: number | null;
cores: number;
load_average?: number[] | null;
};
memory: ResourceMemory;
cgroup_memory?: ResourceMemory | null;
intel_gpu?: {
enabled: boolean;
checked: boolean;
available: boolean;
render_device: string;
model_name?: string | null;
driver?: string | null;
usage_supported: boolean;
usage_percent?: number | null;
detail?: string | null;
engines: Array<{ name: string; percent?: number | null }>;
} | null;
};
type AgentRun = {
id: string;
agent_type: string;
status: string;
term: string;
domain: string;
target_lang: string;
task_id?: string | null;
subtitle_job_id?: string | null;
query: string;
steps: Array<Record<string, unknown>>;
result: Record<string, unknown>;
error: string;
knowledge_item_id?: string | null;
started_at: string;
finished_at?: string | null;
created_at: string;
updated_at: string;
};
function fileNameFromKey(key: string): string {
const parts = (key ?? "").split("/");
return parts[parts.length - 1] || key || "-";
}
function formatDate(value: string): string {
return new Date(value).toLocaleString();
}
function clampPercent(value?: number | null): number {
if (!Number.isFinite(Number(value))) return 0;
return Math.max(0, Math.min(100, Number(value)));
}
function formatPercent(value?: number | null): string {
if (!Number.isFinite(Number(value))) return "-";
return `${clampPercent(value).toFixed(1)}%`;
}
function formatBytes(value?: number | null): string {
const units = ["B", "KB", "MB", "GB", "TB"];
let size = Math.max(0, Number(value || 0));
let unit = 0;
while (size >= 1024 && unit < units.length - 1) {
size /= 1024;
unit += 1;
}
return `${size >= 100 || unit === 0 ? size.toFixed(0) : size.toFixed(1)} ${units[unit]}`;
}
function textValue(value: unknown): string {
return typeof value === "string" ? value : "";
}
function numberValue(value: unknown): number | null {
const n = Number(value);
return Number.isFinite(n) ? n : null;
}
function formatDurationMs(value: unknown): string {
const n = Number(value);
if (!Number.isFinite(n)) return "";
if (n < 1000) return `${Math.max(0, Math.round(n))} ms`;
return `${(n / 1000).toFixed(1)} s`;
}
function agentStatusClass(status: string): string {
if (status === "running") return "border-sky-200 bg-sky-50 text-sky-800";
if (status === "succeeded") return "border-emerald-200 bg-emerald-50 text-emerald-800";
if (status === "skipped") return "border-amber-200 bg-amber-50 text-amber-800";
if (status === "failed") return "border-rose-200 bg-rose-50 text-rose-800";
return "border-slate-200 bg-slate-50 text-slate-700";
}
function stepToneClass(kind: unknown, action: unknown): string {
const k = String(kind || "");
const a = String(action || "");
if (a.includes("failed")) return "border-rose-200 bg-rose-50 text-rose-800";
if (k === "llm") return "border-violet-200 bg-violet-50 text-violet-800";
if (k === "tool") return "border-sky-200 bg-sky-50 text-sky-800";
if (k === "policy") return "border-amber-200 bg-amber-50 text-amber-800";
return "border-slate-200 bg-slate-50 text-slate-700";
}
function prettyJson(value: unknown): string {
if (value === undefined || value === null || value === "") return "";
if (typeof value === "string") return value;
try {
return JSON.stringify(value, null, 2);
} catch {
return String(value);
}
}
function AgentStep({ step, index }: { step: Record<string, unknown>; index: number }) {
const kind = textValue(step.kind) || "event";
const action = textValue(step.action) || "step";
const at = textValue(step.at);
const toolName = textValue(step.tool_name) || textValue(step.tool);
const model = textValue(step.model);
const errorType = textValue(step.error_type);
const duration = formatDurationMs(step.duration_ms);
const ok = typeof step.ok === "boolean" ? step.ok : null;
const body = Object.fromEntries(
Object.entries(step).filter(([key]) => !["kind", "action", "at", "tool", "tool_name", "model", "tokens", "duration_ms", "ok", "error_type"].includes(key)),
);
const hasBody = Object.keys(body).length > 0;
const defaultOpen = index < 2 || action.includes("failed") || Boolean(errorType);
return (
<div className="min-w-0 overflow-hidden rounded-md border border-slate-200 p-3">
<div className="flex flex-wrap items-center justify-between gap-2">
<div className="flex min-w-0 items-center gap-2">
<span className="font-mono text-xs text-slate-500">#{index + 1}</span>
<span className={`rounded border px-2 py-0.5 text-xs ${stepToneClass(kind, action)}`}>{kind}</span>
<span className="truncate text-sm font-medium text-slate-900">{action}</span>
</div>
<div className="flex flex-wrap items-center justify-end gap-2 text-xs text-slate-500">
{toolName ? <span className="rounded bg-slate-100 px-1.5 py-0.5 font-mono">{toolName}</span> : null}
{model ? <span className="max-w-40 truncate rounded bg-slate-100 px-1.5 py-0.5 font-mono">{model}</span> : null}
{duration ? <span>{duration}</span> : null}
{ok !== null ? <span className={ok ? "text-emerald-700" : "text-rose-700"}>{ok ? "ok" : "error"}</span> : null}
{errorType ? <span className="text-rose-700">{errorType}</span> : null}
{at ? <span>{new Date(at).toLocaleTimeString()}</span> : null}
</div>
</div>
{hasBody ? (
<details className="mt-2 min-w-0" open={defaultOpen}>
<summary className="cursor-pointer select-none text-xs font-medium text-slate-600">JSON</summary>
<pre className="mt-2 max-h-72 max-w-full overflow-auto whitespace-pre rounded bg-slate-950 p-3 text-xs leading-relaxed text-slate-100">{prettyJson(body)}</pre>
</details>
) : null}
</div>
);
}
function AgentRunDetail({ run, onClose }: { run: AgentRun; onClose: () => void }) {
const translation = textValue(run.result?.translation);
const confidence = numberValue(run.result?.confidence);
const knowledgeStatus = textValue(run.result?.knowledge_status);
const failureCategory = textValue(run.result?.failure_category);
return (
<div className="fixed inset-0 z-50 flex items-start justify-center overflow-auto bg-slate-950/45 p-4">
<div className="mt-8 w-full max-w-[min(64rem,calc(100vw-2rem))] overflow-hidden rounded-md bg-white shadow-xl">
<div className="flex items-start justify-between gap-4 border-b border-slate-200 p-4">
<div className="min-w-0">
<div className="text-lg font-semibold text-slate-950">{run.term || run.query || run.id}</div>
<div className="mt-1 truncate font-mono text-xs text-slate-500">{run.id}</div>
</div>
<button className="rounded-md border border-slate-300 px-3 py-2 text-sm hover:bg-slate-50" onClick={onClose}>
关闭
</button>
</div>
<div className="grid min-w-0 gap-4 p-4 lg:grid-cols-[280px_minmax(0,1fr)]">
<div className="min-w-0 space-y-3">
<div className="min-w-0 rounded-md border border-slate-200 p-3">
<div className="mb-2 text-xs font-semibold text-slate-600">概览</div>
<div className="grid gap-1 text-xs text-slate-600">
<div>status: <span className={`rounded border px-1.5 py-0.5 ${agentStatusClass(run.status)}`}>{run.status}</span></div>
<div className="truncate">type: {run.agent_type}</div>
<div className="truncate">domain: {run.domain || "-"}</div>
<div className="truncate">target: {run.target_lang}</div>
<div className="truncate">query: {run.query || "-"}</div>
<div>started: {formatDate(run.started_at)}</div>
<div>updated: {formatDate(run.updated_at)}</div>
{run.finished_at ? <div>finished: {formatDate(run.finished_at)}</div> : null}
</div>
</div>
<div className="min-w-0 rounded-md border border-slate-200 p-3">
<div className="mb-2 text-xs font-semibold text-slate-600">结果</div>
{translation ? (
<div className="grid gap-1 text-xs text-slate-600">
<div className="text-sm font-medium text-slate-900">{translation}</div>
{confidence !== null ? <div>confidence: {(confidence * 100).toFixed(0)}%</div> : null}
{knowledgeStatus ? <div>knowledge: {knowledgeStatus}</div> : null}
{failureCategory ? <div>failure: {failureCategory}</div> : null}
{run.knowledge_item_id ? (
<Link to={`/settings/translate?knowledgeItem=${run.knowledge_item_id}`} className="truncate font-mono text-sky-700 hover:underline">
item: {run.knowledge_item_id}
</Link>
) : null}
</div>
) : (
<div className="text-sm text-slate-500">暂无结果。</div>
)}
{run.error ? <div className="mt-2 max-h-32 overflow-auto rounded bg-rose-50 p-2 text-xs text-rose-700">{run.error}</div> : null}
</div>
{Object.keys(run.result || {}).length ? (
<div className="min-w-0 rounded-md border border-slate-200 p-3">
<div className="mb-2 text-xs font-semibold text-slate-600">Result JSON</div>
<pre className="max-h-64 max-w-full overflow-auto whitespace-pre rounded bg-slate-950 p-3 text-xs leading-relaxed text-slate-100">{prettyJson(run.result)}</pre>
</div>
) : null}
</div>
<div className="min-w-0">
<div className="mb-2 text-sm font-semibold text-slate-900">对话流</div>
<div className="min-w-0 space-y-2">
{run.steps.length === 0 ? <div className="text-sm text-slate-500">暂无步骤记录。</div> : null}
{run.steps.map((step, index) => <AgentStep key={`${run.id}-${index}`} step={step} index={index} />)}
</div>
</div>
</div>
</div>
</div>
);
}
function AgentRunCard({ run, onSelect }: { run: AgentRun; onSelect: (run: AgentRun) => void }) {
const translation = textValue(run.result?.translation);
const confidence = numberValue(run.result?.confidence);
const knowledgeStatus = textValue(run.result?.knowledge_status);
const failureCategory = textValue(run.result?.failure_category);
const opened = run.steps.filter((step) => step.action === "open_url" || step.action === "read_url").length;
const latestStep = run.steps[run.steps.length - 1];
return (
<button type="button" className="block w-full rounded-md border border-slate-200 p-3 text-left hover:bg-slate-50" onClick={() => onSelect(run)}>
<div className="flex items-start justify-between gap-3">
<div className="min-w-0">
<div className="truncate text-sm font-medium text-slate-950">{run.term || run.query || run.id}</div>
<div className="mt-1 truncate font-mono text-xs text-slate-500">{run.agent_type} · {formatDate(run.updated_at)}</div>
</div>
<span className={`shrink-0 rounded border px-2 py-0.5 text-xs ${agentStatusClass(run.status)}`}>{run.status}</span>
</div>
<div className="mt-2 grid gap-1 text-xs text-slate-600">
{run.domain ? <div className="truncate">domain: {run.domain}</div> : null}
{run.query ? <div className="truncate">query: {run.query}</div> : null}
{opened ? <div>opened/read pages: {opened}</div> : null}
{latestStep?.action ? <div className="truncate">latest: {String(latestStep.action)}</div> : null}
{translation ? (
<div className="truncate">
result: {translation}
{confidence !== null ? ` · ${(confidence * 100).toFixed(0)}%` : ""}
{knowledgeStatus ? ` · ${knowledgeStatus}` : ""}
</div>
) : null}
{failureCategory ? <div className="truncate text-amber-700">failure: {failureCategory}</div> : null}
{run.error ? <div className="line-clamp-2 text-rose-700">{run.error}</div> : null}
</div>
</button>
);
}
function ResourceBar({ label, value, detail, tone = "sky" }: { label: string; value?: number | null; detail?: string; tone?: "sky" | "emerald" | "amber" }) {
const percent = clampPercent(value);
const color =
tone === "emerald"
? "bg-emerald-500"
: tone === "amber"
? "bg-amber-500"
: "bg-sky-500";
return (
<div>
<div className="mb-1 flex items-center justify-between gap-3 text-xs">
<span className="font-medium text-slate-700">{label}</span>
<span className="font-mono text-slate-600">{formatPercent(value)}</span>
</div>
<div className="h-3 overflow-hidden rounded-sm bg-slate-200 dark:bg-slate-800">
<div className={`h-full ${color}`} style={{ width: `${percent}%` }} />
</div>
{detail ? <div className="mt-1 truncate text-xs text-slate-500">{detail}</div> : null}
</div>
);
}
const runningStatuses = new Set(["INGESTED", "DOWNLOADED", "AUDIO_EXTRACTED", "ASR_DONE", "TRANSLATED", "SUBTITLE_READY", "RENDERED", "READY_FOR_REVIEW", "APPROVED", "PUBLISHING"]);
export default function DashboardPage() {
const [tasks, setTasks] = useState<Task[] | null>(null);
const [error, setError] = useState<string | null>(null);
const [videos, setVideos] = useState<ConvertedVideoItem[] | null>(null);
const [videosError, setVideosError] = useState<string | null>(null);
const [resources, setResources] = useState<ResourceSnapshot | null>(null);
const [resourcesError, setResourcesError] = useState<string | null>(null);
const [agentRuns, setAgentRuns] = useState<AgentRun[] | null>(null);
const [agentRunsError, setAgentRunsError] = useState<string | null>(null);
const [selectedAgentId, setSelectedAgentId] = useState<string | null>(null);
useEffect(() => {
fetchJson<Task[]>(`${ORCHESTRATOR_URL}/tasks?limit=200`)
@@ -34,43 +330,263 @@ export default function DashboardPage() {
.catch((e: unknown) => setVideosError(e instanceof Error ? e.message : String(e)));
}, []);
useEffect(() => {
let cancelled = false;
let timer: number | undefined;
const load = async () => {
try {
const data = await fetchJson<ResourceSnapshot>(`${ORCHESTRATOR_URL}/system/resources`);
if (cancelled) return;
setResources(data);
setResourcesError(null);
} catch (e: unknown) {
if (cancelled) return;
setResourcesError(e instanceof Error ? e.message : String(e));
} finally {
if (!cancelled) timer = window.setTimeout(load, 3000);
}
};
load();
return () => {
cancelled = true;
if (timer) window.clearTimeout(timer);
};
}, []);
useEffect(() => {
let cancelled = false;
let timer: number | undefined;
const load = async () => {
try {
const data = await fetchJson<AgentRun[]>(`${SUBTITLE_SERVICE_URL}/subtitle/agents/runs?limit=12`);
if (cancelled) return;
setAgentRuns(data);
setAgentRunsError(null);
} catch (e: unknown) {
if (cancelled) return;
setAgentRunsError(e instanceof Error ? e.message : String(e));
} finally {
if (!cancelled) timer = window.setTimeout(load, 5000);
}
};
load();
return () => {
cancelled = true;
if (timer) window.clearTimeout(timer);
};
}, []);
const counts = useMemo(() => {
const out = new Map<string, number>();
for (const t of tasks ?? []) out.set(t.status, (out.get(t.status) ?? 0) + 1);
return Array.from(out.entries()).sort((a, b) => b[1] - a[1]);
}, [tasks]);
const total = tasks?.length ?? 0;
const failedTasks = useMemo(() => (tasks ?? []).filter((t) => t.status === "FAILED").slice(0, 6), [tasks]);
const runningTasks = useMemo(() => (tasks ?? []).filter((t) => runningStatuses.has(t.status)).slice(0, 6), [tasks]);
const runningAgents = useMemo(() => (agentRuns ?? []).filter((run) => run.status === "running"), [agentRuns]);
const recentFinishedAgents = useMemo(() => (agentRuns ?? []).filter((run) => run.status !== "running").slice(0, 6), [agentRuns]);
const selectedAgent = useMemo(() => (agentRuns ?? []).find((run) => run.id === selectedAgentId) ?? null, [agentRuns, selectedAgentId]);
const publishedCount = counts.find(([status]) => status === "PUBLISHED")?.[1] ?? 0;
const failedCount = counts.find(([status]) => status === "FAILED")?.[1] ?? 0;
const runningCount = (tasks ?? []).filter((t) => runningStatuses.has(t.status)).length;
return (
<div className="space-y-4">
<div className="rounded border bg-white p-4">
<div className="flex items-center justify-between">
<div>
<div className="text-lg font-semibold">Dashboard</div>
<div className="text-sm text-slate-600">快速查看任务状态与入口</div>
</div>
<Link to="/tasks/new" className="rounded bg-slate-900 px-3 py-2 text-sm text-white hover:bg-slate-800">
<PageHeader
title="Dashboard"
description="快速查看任务状态、失败任务和最新产物。"
actions={
<>
<Link to="/tasks?status=FAILED" className="rounded-md border border-slate-300 px-3 py-2 text-sm text-slate-800 hover:bg-slate-50">
查看失败
</Link>
<Link to="/tasks/new" className="rounded-md bg-slate-900 px-3 py-2 text-sm text-white hover:bg-slate-800">
新建任务
</Link>
</div>
</div>
</Link>
</>
}
/>
<div className="rounded border bg-white p-4">
<div className="mb-2 text-sm font-semibold">最近任务(200 条内)状态统计</div>
<div className="vr-section">
<div className="mb-3 flex items-center justify-between gap-3">
<div className="text-sm font-semibold">最近任务(200 条内)</div>
{tasks ? <div className="text-xs text-slate-500">共 {total} 条</div> : null}
</div>
{error ? <div className="text-sm text-rose-700">{error}</div> : null}
{!tasks ? <div className="text-sm text-slate-500">加载中…</div> : null}
{tasks ? (
<div className="grid grid-cols-2 gap-2 md:grid-cols-3">
<>
<div className="grid grid-cols-2 gap-3 md:grid-cols-4">
<Link to="/tasks" className="rounded-md border border-slate-200 bg-slate-50 p-3 hover:bg-slate-100">
<div className="text-xs text-slate-500">全部任务</div>
<div className="mt-1 text-2xl font-semibold text-slate-950">{total}</div>
</Link>
<Link to="/queue/render" className="rounded-md border border-sky-200 bg-sky-50 p-3 hover:bg-sky-100">
<div className="text-xs text-sky-700">运行中</div>
<div className="mt-1 text-2xl font-semibold text-sky-950">{runningCount}</div>
</Link>
<Link to="/tasks?status=FAILED" className="rounded-md border border-rose-200 bg-rose-50 p-3 hover:bg-rose-100">
<div className="text-xs text-rose-700">失败</div>
<div className="mt-1 text-2xl font-semibold text-rose-950">{failedCount}</div>
</Link>
<Link to="/tasks?status=PUBLISHED" className="rounded-md border border-emerald-200 bg-emerald-50 p-3 hover:bg-emerald-100">
<div className="text-xs text-emerald-700">已发布</div>
<div className="mt-1 text-2xl font-semibold text-emerald-950">{publishedCount}</div>
</Link>
</div>
<div className="mt-3 grid grid-cols-2 gap-2 md:grid-cols-4">
{counts.map(([status, n]) => (
<div key={status} className="rounded border p-3">
<Link key={status} to={`/tasks?status=${status}`} className="rounded-md border border-slate-200 p-3 hover:bg-slate-50">
<div className="text-xs text-slate-500">{status}</div>
<div className="text-2xl font-semibold">{n}</div>
</div>
<div className="text-xl font-semibold text-slate-900">{n}</div>
</Link>
))}
</div>
</>
) : null}
</div>
<div className="vr-section">
<div className="flex items-center justify-between gap-3">
<div>
<div className="text-sm font-semibold">资源监控</div>
<div className="mt-1 text-xs text-slate-500">每 3 秒刷新一次,数值来自当前运行后端的容器/主机视角。</div>
</div>
{resources?.sampled_at ? <div className="text-xs text-slate-500">{new Date(resources.sampled_at).toLocaleTimeString()}</div> : null}
</div>
{resourcesError ? <div className="mt-3 text-sm text-rose-700">{resourcesError}</div> : null}
{!resources ? <div className="mt-3 text-sm text-slate-500">加载中…</div> : null}
{resources ? (
<div className="mt-4 grid gap-4 lg:grid-cols-3">
<div className="rounded-md border border-slate-200 p-3 dark:border-slate-700">
<ResourceBar
label={`CPU${resources.cpu.cores ? ` · ${resources.cpu.cores} cores` : ""}`}
value={resources.cpu.percent}
detail={resources.cpu.load_average?.length ? `load ${resources.cpu.load_average.map((n) => n.toFixed(2)).join(" / ")}` : undefined}
/>
</div>
<div className="rounded-md border border-slate-200 p-3 dark:border-slate-700">
<ResourceBar
label={resources.cgroup_memory ? "Memory · cgroup" : "Memory"}
value={(resources.cgroup_memory ?? resources.memory).percent}
detail={`${formatBytes((resources.cgroup_memory ?? resources.memory).used_bytes)} / ${formatBytes((resources.cgroup_memory ?? resources.memory).total_bytes)}`}
tone="emerald"
/>
{resources.cgroup_memory ? (
<div className="mt-2 text-xs text-slate-500">host: {formatBytes(resources.memory.used_bytes)} / {formatBytes(resources.memory.total_bytes)}</div>
) : null}
</div>
{resources.intel_gpu?.enabled ? (
<div className="rounded-md border border-slate-200 p-3 dark:border-slate-700">
<ResourceBar
label="Intel GPU"
value={resources.intel_gpu.usage_percent}
detail={
resources.intel_gpu.available
? `${resources.intel_gpu.model_name || resources.intel_gpu.render_device}${resources.intel_gpu.usage_supported ? "" : " · busy 不可读"}`
: resources.intel_gpu.detail || "未检测到可用 Intel GPU"
}
tone="amber"
/>
{resources.intel_gpu.engines?.length ? (
<div className="mt-2 grid gap-1">
{resources.intel_gpu.engines.slice(0, 4).map((engine) => (
<div key={engine.name} className="flex items-center justify-between gap-2 text-[11px] text-slate-500">
<span className="truncate">{engine.name}</span>
<span className="font-mono">{formatPercent(engine.percent)}</span>
</div>
))}
</div>
) : null}
</div>
) : null}
</div>
) : null}
</div>
<div className="rounded border bg-white p-4">
<div className="vr-section">
<div className="flex items-center justify-between gap-3">
<div>
<div className="text-sm font-semibold">RAG Agent</div>
<div className="mt-1 text-xs text-slate-500">显示术语发现、搜索、网页读取、总结入库的运行记录。</div>
</div>
{agentRuns ? <div className="text-xs text-slate-500">运行中 {runningAgents.length}</div> : null}
</div>
{agentRunsError ? <div className="mt-3 text-sm text-rose-700">{agentRunsError}</div> : null}
{!agentRuns ? <div className="mt-3 text-sm text-slate-500">加载中…</div> : null}
{agentRuns ? (
<div className="mt-4 grid gap-4 lg:grid-cols-2">
<div>
<div className="mb-2 text-xs font-semibold text-slate-600">正在工作</div>
<div className="space-y-2">
{runningAgents.length === 0 ? <div className="text-sm text-slate-500">暂无运行中的 agent。</div> : null}
{runningAgents.map((run) => <AgentRunCard key={run.id} run={run} onSelect={(item) => setSelectedAgentId(item.id)} />)}
</div>
</div>
<div>
<div className="mb-2 text-xs font-semibold text-slate-600">最近结果</div>
<div className="space-y-2">
{recentFinishedAgents.length === 0 ? <div className="text-sm text-slate-500">暂无 agent 结果。</div> : null}
{recentFinishedAgents.map((run) => <AgentRunCard key={run.id} run={run} onSelect={(item) => setSelectedAgentId(item.id)} />)}
</div>
</div>
</div>
) : null}
</div>
<div className="grid gap-4 lg:grid-cols-2">
<div className="vr-section">
<div className="flex items-center justify-between gap-3">
<div className="text-sm font-semibold">正在处理</div>
<Link to="/queue/render" className="text-sm text-slate-700 hover:underline">
队列 →
</Link>
</div>
<div className="mt-3 space-y-2">
{!tasks ? <div className="text-sm text-slate-500">加载中…</div> : null}
{tasks && runningTasks.length === 0 ? <div className="text-sm text-slate-500">暂无运行中任务。</div> : null}
{runningTasks.map((t) => (
<Link key={t.id} to={`/tasks/${t.id}`} className="block rounded-md border border-slate-200 p-3 hover:bg-slate-50">
<div className="flex items-center justify-between gap-3">
<div className="min-w-0">
<div className="truncate text-sm font-medium text-slate-950">{t.display_title?.trim() || t.source_url || t.id}</div>
<div className="mt-1 font-mono text-xs text-slate-500">{t.id.slice(0, 8)} · {formatDate(t.updated_at)}</div>
</div>
<StatusBadge status={t.status} />
</div>
</Link>
))}
</div>
</div>
<div className="vr-section">
<div className="flex items-center justify-between gap-3">
<div className="text-sm font-semibold">最近失败</div>
<Link to="/tasks?status=FAILED" className="text-sm text-slate-700 hover:underline">
全部失败 →
</Link>
</div>
<div className="mt-3 space-y-2">
{!tasks ? <div className="text-sm text-slate-500">加载中…</div> : null}
{tasks && failedTasks.length === 0 ? <div className="text-sm text-slate-500">暂无失败任务。</div> : null}
{failedTasks.map((t) => (
<Link key={t.id} to={`/tasks/${t.id}`} className="block rounded-md border border-rose-100 p-3 hover:bg-rose-50">
<div className="flex items-start justify-between gap-3">
<div className="min-w-0">
<div className="truncate text-sm font-medium text-slate-950">{t.display_title?.trim() || t.source_url || t.id}</div>
<div className="mt-1 line-clamp-2 text-xs text-rose-700">{t.error_message || t.error_code || "FAILED"}</div>
</div>
<StatusBadge status={t.status} />
</div>
</Link>
))}
</div>
</div>
</div>
<div className="vr-section">
<div className="flex items-center justify-between gap-3">
<div className="text-sm font-semibold">已转换视频(video_final)</div>
<Link to="/videos" className="text-sm text-slate-700 hover:underline">
@@ -81,7 +597,7 @@ export default function DashboardPage() {
{videosError ? <div className="mt-2 text-sm text-rose-700">{videosError}</div> : null}
{!videos ? <div className="mt-2 text-sm text-slate-500">加载中…</div> : null}
{videos ? (
<div className="mt-3 overflow-auto">
<div className="mt-3 overflow-auto rounded-md border border-slate-200">
<table className="min-w-full text-left text-sm">
<thead className="text-xs text-slate-500">
<tr>
@@ -108,12 +624,12 @@ export default function DashboardPage() {
<td className="py-2 pr-3">
<div className="flex flex-wrap items-center gap-2">
<a
className="rounded border px-2 py-1 text-xs hover:bg-slate-50"
className="rounded-md border border-slate-300 px-2 py-1 text-xs hover:bg-slate-50"
href={`${ORCHESTRATOR_URL}/tasks/${it.task.id}/assets/${it.final_asset.id}/download`}
>
Download
</a>
<Link className="rounded border px-2 py-1 text-xs hover:bg-slate-50" to={`/tasks/${it.task.id}`}>
<Link className="rounded-md border border-slate-300 px-2 py-1 text-xs hover:bg-slate-50" to={`/tasks/${it.task.id}`}>
Detail
</Link>
</div>
@@ -132,6 +648,7 @@ export default function DashboardPage() {
</div>
) : null}
</div>
{selectedAgent ? <AgentRunDetail run={selectedAgent} onClose={() => setSelectedAgentId(null)} /> : null}
</div>
);
}
+1 -1
View File
@@ -50,7 +50,7 @@ export default function RenderQueuePage() {
useEffect(() => {
refresh();
}, []);
}, [refresh]);
const shouldPoll = useMemo(() => (queue?.tasks ?? []).some((t) => t.state === "queued" || t.state === "running"), [queue]);
+9 -1
View File
@@ -1,4 +1,5 @@
import { useEffect, useMemo, useState } from "react";
import { useConfirm } from "../components/feedbackContext";
import { fetchJson } from "../lib/http";
import { SUBTITLE_SERVICE_URL } from "../lib/urls";
@@ -58,6 +59,7 @@ function formatBytes(n?: number | null): string {
}
export default function SettingsASRPage() {
const confirm = useConfirm();
const [settings, setSettings] = useState<WhisperSettings | null>(null);
const [asrDefaults, setAsrDefaults] = useState<ASRDefaults | null>(null);
const [models, setModels] = useState<WhisperModelInfo[] | null>(null);
@@ -417,7 +419,13 @@ export default function SettingsASRPage() {
disabled={busy}
className="rounded border border-rose-300 px-2 py-1 text-xs text-rose-700 hover:bg-rose-50 disabled:opacity-50"
onClick={async () => {
if (!confirm(`确定删除模型:${m.name} ?`)) return;
const ok = await confirm({
title: "删除模型",
message: `确定删除模型:${m.name} ?`,
confirmLabel: "删除",
tone: "danger",
});
if (!ok) return;
setBusy(true);
setError(null);
try {
+9 -1
View File
@@ -1,4 +1,5 @@
import { useEffect, useMemo, useState } from "react";
import { useConfirm } from "../components/feedbackContext";
import { fetchJson } from "../lib/http";
import { ORCHESTRATOR_URL } from "../lib/urls";
@@ -30,6 +31,7 @@ function orchestratorBaseUrl(): string {
}
export default function SettingsApiPage() {
const confirm = useConfirm();
const [settings, setSettings] = useState<RemoteApiSettings | null>(null);
const [tokenInput, setTokenInput] = useState("");
const [error, setError] = useState<string | null>(null);
@@ -148,7 +150,13 @@ export default function SettingsApiPage() {
disabled={busy || !settings?.token_set}
className="rounded border px-3 py-2 text-sm hover:bg-slate-50 disabled:opacity-50"
onClick={async () => {
if (!window.confirm("确认清空远程管理 token?清空后外部将无法再调用该接口。")) return;
const ok = await confirm({
title: "清空远程管理 token",
message: "清空后外部将无法再调用该接口。",
confirmLabel: "清空",
tone: "danger",
});
if (!ok) return;
setBusy(true);
setError(null);
try {
+10 -2
View File
@@ -1,5 +1,6 @@
import { Link } from "react-router-dom";
import { useEffect, useMemo, useState } from "react";
import { useConfirm } from "../components/feedbackContext";
import { fetchJson } from "../lib/http";
import { SUBTITLE_SERVICE_URL } from "../lib/urls";
@@ -57,6 +58,7 @@ function normalizeYouTubeSubtitleMode(value: unknown, legacyPrefer?: boolean | n
}
export default function SettingsAutoPage() {
const confirm = useConfirm();
const [error, setError] = useState<string | null>(null);
const [busy, setBusy] = useState(false);
@@ -605,8 +607,14 @@ export default function SettingsAutoPage() {
<button
disabled={busy}
className="rounded border px-3 py-2 text-sm hover:bg-slate-50 disabled:opacity-50"
onClick={() => {
if (!confirm("确定恢复默认配置吗?")) return;
onClick={async () => {
const ok = await confirm({
title: "恢复默认配置",
message: "确定恢复默认配置吗?当前页面中尚未保存的修改会被覆盖。",
confirmLabel: "恢复默认",
tone: "warning",
});
if (!ok) return;
setFormats({ srt: true, ass: true });
setBurnIn(true);
setSoftSub(false);
+16 -2
View File
@@ -1,4 +1,5 @@
import { useEffect, useState } from "react";
import { useConfirm } from "../components/feedbackContext";
import { fetchJson } from "../lib/http";
import { BILIBILI_PUBLISHER_URL } from "../lib/urls";
@@ -21,6 +22,7 @@ type BilibiliMe = {
};
export default function SettingsBilibiliPage() {
const confirm = useConfirm();
const [settings, setSettings] = useState<PublishSettings | null>(null);
const [auth, setAuth] = useState<AuthSettings | null>(null);
const [metaText, setMetaText] = useState<string>("{}");
@@ -117,7 +119,13 @@ export default function SettingsBilibiliPage() {
disabled={busy || !auth?.cookie_set}
className="rounded border border-rose-300 px-3 py-2 text-sm text-rose-700 hover:bg-rose-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("确定清除已保存的 Cookies 吗?")) return;
const ok = await confirm({
title: "清除 Bilibili Cookies",
message: "确定清除已保存的 Cookies 吗?清除后真实投稿不可用。",
confirmLabel: "清除",
tone: "danger",
});
if (!ok) return;
setBusy(true);
setError(null);
try {
@@ -221,7 +229,13 @@ export default function SettingsBilibiliPage() {
disabled={busy}
className="rounded border border-rose-300 px-3 py-2 text-sm text-rose-700 hover:bg-rose-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("确定恢复内置默认模板吗?")) return;
const ok = await confirm({
title: "恢复默认模板",
message: "确定恢复内置默认模板吗?当前 default_meta.json 会被覆盖。",
confirmLabel: "恢复默认",
tone: "warning",
});
if (!ok) return;
setBusy(true);
setError(null);
try {
+546 -172
View File
@@ -1,4 +1,6 @@
import { useEffect, useState } from "react";
import { useConfirm } from "../components/feedbackContext";
import { Button, DataTable, EmptyState, PageHeader, Section } from "../components/ui";
import { fetchJson } from "../lib/http";
import { SUBTITLE_SERVICE_URL } from "../lib/urls";
@@ -9,16 +11,57 @@ type TranslateSettings = {
default_batch_size: number;
default_max_retries: number;
default_enable_summary: boolean;
openai_api_key_set: boolean;
openai_base_url: string;
openai_model: string;
openai_temperature: number;
openai_timeout_seconds: number;
rag_enabled: boolean;
rag_top_k: number;
rag_min_score: number;
rag_embedding_provider: string;
rag_embedding_model: string;
rag_embedding_dimensions: number;
rag_embedding_model_dir: string;
rag_embedding_device: string;
rag_auto_discover_terms: boolean;
rag_auto_learn_terms: boolean;
rag_search_enabled: boolean;
rag_search_url: string;
rag_domain: string;
};
type KnowledgeItem = {
id: string;
item_type: string;
term: string;
translation: string;
target_lang: string;
domain: string;
title: string;
description: string;
confidence: number;
status: string;
created_by: string;
usage_count: number;
};
type EmbeddingModelInfo = {
name: string;
path: string;
size_bytes?: number | null;
};
function safeEmbeddingModelName(raw: string) {
const value = raw.trim().replace(/[\\/]/g, "--").replace(/[^A-Za-z0-9._-]/g, "-");
return value.slice(0, 96) || "embedding-model";
}
export default function SettingsTranslatePage() {
const confirm = useConfirm();
const [settings, setSettings] = useState<TranslateSettings | null>(null);
const [items, setItems] = useState<KnowledgeItem[]>([]);
const [embeddingModels, setEmbeddingModels] = useState<EmbeddingModelInfo[]>([]);
const [error, setError] = useState<string | null>(null);
const [busy, setBusy] = useState(false);
@@ -35,15 +78,49 @@ export default function SettingsTranslatePage() {
const [openaiTimeoutSeconds, setOpenaiTimeoutSeconds] = useState(60);
const [openaiApiKey, setOpenaiApiKey] = useState("");
const [ragEnabled, setRagEnabled] = useState(false);
const [ragTopK, setRagTopK] = useState(8);
const [ragMinScore, setRagMinScore] = useState(0.68);
const [ragEmbeddingProvider, setRagEmbeddingProvider] = useState("openai");
const [ragEmbeddingModel, setRagEmbeddingModel] = useState("text-embedding-3-small");
const [ragEmbeddingDimensions, setRagEmbeddingDimensions] = useState(1536);
const [ragEmbeddingModelDir, setRagEmbeddingModelDir] = useState("/models/embeddings");
const [ragEmbeddingDevice, setRagEmbeddingDevice] = useState("cpu");
const [ragAutoDiscoverTerms, setRagAutoDiscoverTerms] = useState(false);
const [ragAutoLearnTerms, setRagAutoLearnTerms] = useState(false);
const [ragSearchEnabled, setRagSearchEnabled] = useState(false);
const [ragSearchUrl, setRagSearchUrl] = useState("");
const [ragDomain, setRagDomain] = useState("");
const [testText, setTestText] = useState("Hello world. This is a translation test.");
const [testTargetLang, setTestTargetLang] = useState("zh");
const [testStyle, setTestStyle] = useState("口语自然");
const [testResult, setTestResult] = useState<string | null>(null);
const [knowledgeType, setKnowledgeType] = useState<"term" | "document">("term");
const [knowledgeTerm, setKnowledgeTerm] = useState("");
const [knowledgeTranslation, setKnowledgeTranslation] = useState("");
const [knowledgeDomain, setKnowledgeDomain] = useState("");
const [knowledgeTitle, setKnowledgeTitle] = useState("");
const [knowledgeContent, setKnowledgeContent] = useState("");
const [knowledgeDescription, setKnowledgeDescription] = useState("");
const [embeddingDownloadModel, setEmbeddingDownloadModel] = useState("BAAI/bge-small-zh-v1.5");
const [embeddingDownloadName, setEmbeddingDownloadName] = useState("");
const [embeddingTestResult, setEmbeddingTestResult] = useState<string | null>(null);
const [embeddingRebuildResult, setEmbeddingRebuildResult] = useState<string | null>(null);
async function refresh() {
setError(null);
try {
const s = await fetchJson<TranslateSettings>(`${SUBTITLE_SERVICE_URL}/subtitle/translate/settings`);
const [s, knowledge] = await Promise.all([
fetchJson<TranslateSettings>(`${SUBTITLE_SERVICE_URL}/subtitle/translate/settings`),
fetchJson<KnowledgeItem[]>(`${SUBTITLE_SERVICE_URL}/subtitle/knowledge/items?limit=50`).catch(() => []),
]);
const localModels = await fetchJson<EmbeddingModelInfo[]>(`${SUBTITLE_SERVICE_URL}/subtitle/embedding/models/list`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ model_dir: s.rag_embedding_model_dir }),
}).catch(() => []);
setSettings(s);
setDefaultProvider(s.default_provider);
setDefaultTargetLang(s.default_target_lang);
@@ -55,8 +132,23 @@ export default function SettingsTranslatePage() {
setOpenaiModel(s.openai_model);
setOpenaiTemperature(s.openai_temperature);
setOpenaiTimeoutSeconds(s.openai_timeout_seconds);
setRagEnabled(s.rag_enabled);
setRagTopK(s.rag_top_k);
setRagMinScore(s.rag_min_score);
setRagEmbeddingProvider(s.rag_embedding_provider);
setRagEmbeddingModel(s.rag_embedding_model);
setRagEmbeddingDimensions(s.rag_embedding_dimensions);
setRagEmbeddingModelDir(s.rag_embedding_model_dir);
setRagEmbeddingDevice(s.rag_embedding_device);
setRagAutoDiscoverTerms(s.rag_auto_discover_terms);
setRagAutoLearnTerms(s.rag_auto_learn_terms);
setRagSearchEnabled(s.rag_search_enabled);
setRagSearchUrl(s.rag_search_url);
setRagDomain(s.rag_domain);
setTestTargetLang(s.default_target_lang || "zh");
setTestStyle(s.default_style || "口语自然");
setItems(knowledge);
setEmbeddingModels(localModels);
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
}
@@ -66,63 +158,209 @@ export default function SettingsTranslatePage() {
refresh();
}, []);
async function persistSettings() {
const payload: Partial<TranslateSettings> & { openai_api_key?: string } = {
default_provider: defaultProvider,
default_target_lang: defaultTargetLang,
default_style: defaultStyle,
default_batch_size: defaultBatchSize,
default_max_retries: defaultMaxRetries,
default_enable_summary: defaultEnableSummary,
openai_base_url: openaiBaseUrl,
openai_model: openaiModel,
openai_temperature: openaiTemperature,
openai_timeout_seconds: openaiTimeoutSeconds,
rag_enabled: ragEnabled,
rag_top_k: ragTopK,
rag_min_score: ragMinScore,
rag_embedding_provider: ragEmbeddingProvider,
rag_embedding_model: ragEmbeddingModel,
rag_embedding_dimensions: ragEmbeddingDimensions,
rag_embedding_model_dir: ragEmbeddingModelDir,
rag_embedding_device: ragEmbeddingDevice,
rag_auto_discover_terms: ragAutoDiscoverTerms,
rag_auto_learn_terms: ragAutoLearnTerms,
rag_search_enabled: ragSearchEnabled,
rag_search_url: ragSearchUrl,
rag_domain: ragDomain,
};
if (openaiApiKey.trim()) payload.openai_api_key = openaiApiKey.trim();
await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/translate/settings`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
}
async function saveSettings() {
setBusy(true);
setError(null);
try {
await persistSettings();
setOpenaiApiKey("");
await refresh();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}
async function rebuildKnowledgeEmbeddings() {
const ok = await confirm({
title: "重建知识库向量",
message: "会使用当前页面里的 embedding 配置重新生成知识库向量。知识条目较多时会比较慢。",
confirmLabel: "开始重建",
});
if (!ok) return;
setBusy(true);
setError(null);
setEmbeddingRebuildResult(null);
try {
await persistSettings();
setOpenaiApiKey("");
const resp = await fetchJson<{
total: number;
updated: number;
failed: number;
skipped: number;
embedding_model: string;
dimensions: number;
}>(`${SUBTITLE_SERVICE_URL}/subtitle/knowledge/rebuild-embeddings`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ limit: 10000 }),
});
setEmbeddingRebuildResult(
`${resp.embedding_model} / ${resp.dimensions} dims:共 ${resp.total} 条,更新 ${resp.updated},跳过 ${resp.skipped},失败 ${resp.failed}`,
);
await refresh();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}
async function saveKnowledgeItem() {
setBusy(true);
setError(null);
try {
await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/knowledge/items`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
item_type: knowledgeType,
target_lang: defaultTargetLang || "zh",
term: knowledgeTerm,
translation: knowledgeTranslation,
domain: knowledgeDomain,
title: knowledgeTitle,
content: knowledgeContent,
description: knowledgeDescription,
confidence: 1,
status: "approved",
created_by: "manual",
}),
});
setKnowledgeTerm("");
setKnowledgeTranslation("");
setKnowledgeTitle("");
setKnowledgeContent("");
setKnowledgeDescription("");
await refresh();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}
async function downloadEmbeddingModel() {
setBusy(true);
setError(null);
try {
await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/embedding/models/download`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model: embeddingDownloadModel,
name: embeddingDownloadName.trim() || null,
model_dir: ragEmbeddingModelDir,
force: false,
}),
});
await refresh();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}
async function testEmbedding() {
setBusy(true);
setError(null);
setEmbeddingTestResult(null);
try {
const resp = await fetchJson<{ provider: string; model: string; dimensions: number; expected_dimensions: number; ok: boolean }>(
`${SUBTITLE_SERVICE_URL}/subtitle/embedding/test`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
text: "Rush B with an AWP",
provider: ragEmbeddingProvider,
model: ragEmbeddingModel,
model_dir: ragEmbeddingModelDir,
dimensions: ragEmbeddingDimensions,
device: ragEmbeddingDevice,
}),
},
);
setEmbeddingTestResult(`${resp.provider}:${resp.model} -> ${resp.dimensions} dims${resp.ok ? "" : `,期望 ${resp.expected_dimensions}`}`);
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}
return (
<div className="space-y-4">
<div className="rounded border bg-white p-4">
<div className="text-lg font-semibold">Settings · Translate</div>
<div className="mt-1 text-sm text-slate-600">配置字幕翻译(可选接入 OpenAI 标准接口)。</div>
{error ? <div className="mt-3 text-sm text-rose-700">{error}</div> : null}
</div>
<PageHeader title="Settings · Translate" description="配置 OpenAI 翻译、pgvector RAG 和字幕术语知识库。" actions={<Button onClick={() => refresh()}>刷新</Button>} />
{error ? <div className="rounded-md border border-rose-200 bg-rose-50 p-3 text-sm text-rose-700">{error}</div> : null}
<div className="rounded border bg-white p-4">
<Section>
<div className="flex items-center justify-between">
<div className="text-sm font-semibold">当前配置(保存于后端配置/数据库)</div>
<button onClick={() => refresh()} className="rounded border px-3 py-2 text-sm hover:bg-slate-50">
刷新
</button>
</div>
{!settings ? (
<div className="mt-2 text-sm text-slate-500">加载中…</div>
) : (
<div className="mt-3 grid gap-3 md:grid-cols-2">
<div className="rounded border p-3">
<div className="text-xs text-slate-500">default_provider</div>
<div className="mt-1 font-mono text-sm">{settings.default_provider}</div>
</div>
<div className="rounded border p-3">
<div className="text-xs text-slate-500">default_target_lang</div>
<div className="mt-1 font-mono text-sm">{settings.default_target_lang}</div>
</div>
<div className="rounded border p-3">
<div className="text-xs text-slate-500">default_style</div>
<div className="mt-1 font-mono text-sm">{settings.default_style}</div>
</div>
<div className="rounded border p-3">
<div className="text-xs text-slate-500">default_batch_size</div>
<div className="mt-1 font-mono text-sm">{settings.default_batch_size}</div>
</div>
<div className="rounded border p-3">
<div className="text-xs text-slate-500">default_max_retries</div>
<div className="mt-1 font-mono text-sm">{settings.default_max_retries}</div>
</div>
<div className="rounded border p-3 md:col-span-2">
<div className="text-xs text-slate-500">default_enable_summary</div>
<div className="mt-1 text-sm">{settings.default_enable_summary ? "true" : "false"}</div>
</div>
</div>
)}
<div className="mt-3 text-xs text-slate-500">提示:默认值用于前端初始选择;任务里仍可按需覆盖。</div>
</div>
<div className="rounded border bg-white p-4">
<div className="flex items-center justify-between">
<div className="text-sm font-semibold">保存配置</div>
<div className="text-sm font-semibold text-slate-900">当前配置</div>
<div className="text-xs text-slate-500">
OpenAI API Key:{settings?.openai_api_key_set ? <span className="text-emerald-700">已设置</span> : <span className="text-rose-700">未设置</span>}
</div>
</div>
{!settings ? (
<div className="mt-2 text-sm text-slate-500">加载中...</div>
) : (
<div className="mt-3 grid gap-3 md:grid-cols-3">
{[
["provider", settings.default_provider],
["target", settings.default_target_lang],
["batch", settings.default_batch_size],
["summary", settings.default_enable_summary ? "true" : "false"],
["rag", settings.rag_enabled ? "enabled" : "disabled"],
["embedding", `${settings.rag_embedding_provider}:${settings.rag_embedding_model}`],
].map(([label, value]) => (
<div key={label} className="rounded-md border border-slate-200 p-3">
<div className="text-xs text-slate-500">{label}</div>
<div className="mt-1 break-all font-mono text-sm text-slate-900">{value}</div>
</div>
))}
</div>
)}
</Section>
<Section>
<div className="text-sm font-semibold text-slate-900">翻译与模型</div>
<div className="mt-3 grid gap-3 md:grid-cols-2">
<label className="block">
<div className="mb-1 text-xs text-slate-600">default_provider</div>
@@ -132,12 +370,10 @@ export default function SettingsTranslatePage() {
<option value="noop">noop</option>
</select>
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">default_target_lang</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={defaultTargetLang} onChange={(e) => setDefaultTargetLang(e.target.value)} />
</label>
<label className="block md:col-span-2">
<div className="mb-1 text-xs text-slate-600">default_style</div>
<select className="w-full rounded border px-3 py-2 text-sm" value={defaultStyle} onChange={(e) => setDefaultStyle(e.target.value)}>
@@ -146,159 +382,297 @@ export default function SettingsTranslatePage() {
<option value="电商营销">电商营销</option>
</select>
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">default_batch_size</div>
<input
type="number"
min={1}
className="w-full rounded border px-3 py-2 text-sm"
value={defaultBatchSize}
onChange={(e) => setDefaultBatchSize(parseInt(e.target.value || "1", 10))}
/>
<input type="number" min={1} className="w-full rounded border px-3 py-2 text-sm" value={defaultBatchSize} onChange={(e) => setDefaultBatchSize(parseInt(e.target.value || "1", 10))} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">default_max_retries</div>
<input
type="number"
min={0}
max={10}
className="w-full rounded border px-3 py-2 text-sm"
value={defaultMaxRetries}
onChange={(e) => setDefaultMaxRetries(parseInt(e.target.value || "0", 10))}
/>
<input type="number" min={0} max={10} className="w-full rounded border px-3 py-2 text-sm" value={defaultMaxRetries} onChange={(e) => setDefaultMaxRetries(parseInt(e.target.value || "0", 10))} />
</label>
<label className="flex items-center gap-2 text-sm">
<input type="checkbox" checked={defaultEnableSummary} onChange={(e) => setDefaultEnableSummary(e.target.checked)} />
default_enable_summary
</label>
<div className="md:col-span-2 pt-2 text-xs font-semibold text-slate-700">OpenAI(标准接口)</div>
<label className="block md:col-span-2">
<div className="mb-1 text-xs text-slate-600">openai_api_key(仅保存,不回显)</div>
<input
type="password"
className="w-full rounded border px-3 py-2 text-sm"
placeholder={settings?.openai_api_key_set ? "已设置(留空则不修改)" : "sk-..."}
value={openaiApiKey}
onChange={(e) => setOpenaiApiKey(e.target.value)}
/>
<input type="password" className="w-full rounded border px-3 py-2 text-sm" placeholder={settings?.openai_api_key_set ? "已设置(留空则不修改)" : "sk-..."} value={openaiApiKey} onChange={(e) => setOpenaiApiKey(e.target.value)} />
</label>
<label className="block md:col-span-2">
<div className="mb-1 text-xs text-slate-600">openai_base_url</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={openaiBaseUrl} onChange={(e) => setOpenaiBaseUrl(e.target.value)} />
<div className="mt-1 text-xs text-slate-500">
提示:多数 OpenAI 兼容接口需要 base_url 以 <span className="font-mono">/v1</span> 结尾(例如{" "}
<span className="font-mono">https://api.openai.com/v1</span>)。如果只填域名根(例如{" "}
<span className="font-mono">https://ai.example.com</span>),后端会自动补全为{" "}
<span className="font-mono">https://ai.example.com/v1</span>。
</div>
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">openai_model</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={openaiModel} onChange={(e) => setOpenaiModel(e.target.value)} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">openai_temperature</div>
<input
type="number"
step="0.1"
min={0}
max={2}
className="w-full rounded border px-3 py-2 text-sm"
value={openaiTemperature}
onChange={(e) => setOpenaiTemperature(parseFloat(e.target.value || "0"))}
/>
<input type="number" step="0.1" min={0} max={2} className="w-full rounded border px-3 py-2 text-sm" value={openaiTemperature} onChange={(e) => setOpenaiTemperature(parseFloat(e.target.value || "0"))} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">openai_timeout_seconds</div>
<input
type="number"
min={1}
className="w-full rounded border px-3 py-2 text-sm"
value={openaiTimeoutSeconds}
onChange={(e) => setOpenaiTimeoutSeconds(parseFloat(e.target.value || "1"))}
/>
<input type="number" min={1} className="w-full rounded border px-3 py-2 text-sm" value={openaiTimeoutSeconds} onChange={(e) => setOpenaiTimeoutSeconds(parseFloat(e.target.value || "1"))} />
</label>
</div>
</Section>
<Section>
<div className="text-sm font-semibold text-slate-900">RAG / pgvector</div>
<div className="mt-3 grid gap-3 md:grid-cols-2">
<label className="flex items-center gap-2 text-sm">
<input type="checkbox" checked={ragEnabled} onChange={(e) => setRagEnabled(e.target.checked)} />
启用 RAG 翻译增强
</label>
<label className="flex items-center gap-2 text-sm">
<input type="checkbox" checked={ragAutoDiscoverTerms} onChange={(e) => setRagAutoDiscoverTerms(e.target.checked)} />
LLM 自动发现术语
</label>
<label className="flex items-center gap-2 text-sm">
<input type="checkbox" checked={ragAutoLearnTerms} onChange={(e) => setRagAutoLearnTerms(e.target.checked)} />
允许自动学习术语
</label>
<label className="flex items-center gap-2 text-sm">
<input type="checkbox" checked={ragSearchEnabled} onChange={(e) => setRagSearchEnabled(e.target.checked)} />
允许调用搜索服务
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">rag_top_k</div>
<input type="number" min={0} max={30} className="w-full rounded border px-3 py-2 text-sm" value={ragTopK} onChange={(e) => setRagTopK(parseInt(e.target.value || "0", 10))} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">rag_min_score</div>
<input type="number" step="0.01" min={0} max={1} className="w-full rounded border px-3 py-2 text-sm" value={ragMinScore} onChange={(e) => setRagMinScore(parseFloat(e.target.value || "0"))} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">embedding_model</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={ragEmbeddingModel} onChange={(e) => setRagEmbeddingModel(e.target.value)} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">embedding_provider</div>
<select
className="w-full rounded border px-3 py-2 text-sm"
value={ragEmbeddingProvider}
onChange={(e) => {
const provider = e.target.value;
setRagEmbeddingProvider(provider);
if (provider === "local" && ragEmbeddingModel === "text-embedding-3-small") {
setRagEmbeddingModel("BAAI/bge-small-zh-v1.5");
setRagEmbeddingDimensions(512);
}
if (provider === "openai" && ragEmbeddingModel === "BAAI/bge-small-zh-v1.5") {
setRagEmbeddingModel("text-embedding-3-small");
setRagEmbeddingDimensions(1536);
}
}}
>
<option value="openai">openai</option>
<option value="local">local</option>
</select>
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">embedding_dimensions</div>
<input type="number" min={1} max={4096} className="w-full rounded border px-3 py-2 text-sm" value={ragEmbeddingDimensions} onChange={(e) => setRagEmbeddingDimensions(parseInt(e.target.value || "1536", 10))} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">embedding_device</div>
<select className="w-full rounded border px-3 py-2 text-sm" value={ragEmbeddingDevice} onChange={(e) => setRagEmbeddingDevice(e.target.value)}>
<option value="cpu">CPU(PyTorch)</option>
<option value="openvino:CPU">CPU(OpenVINO)</option>
<option value="openvino:GPU">Intel GPU(OpenVINO)</option>
</select>
</label>
<label className="block md:col-span-2">
<div className="mb-1 text-xs text-slate-600">embedding_model_dir</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={ragEmbeddingModelDir} onChange={(e) => setRagEmbeddingModelDir(e.target.value)} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">domain</div>
<input className="w-full rounded border px-3 py-2 text-sm" placeholder="例如 Minecraft / CS2 / Anime" value={ragDomain} onChange={(e) => setRagDomain(e.target.value)} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">SearXNG Base URL</div>
<input className="w-full rounded border px-3 py-2 text-sm" placeholder="https://search.linvk.com" value={ragSearchUrl} onChange={(e) => setRagSearchUrl(e.target.value)} />
</label>
</div>
<div className="mt-3 flex flex-wrap items-center gap-2">
<button
<Button disabled={busy} onClick={testEmbedding}>测试 Embedding</Button>
<Button tone="primary" disabled={busy} onClick={rebuildKnowledgeEmbeddings}>{busy ? "处理中..." : "重建知识库向量"}</Button>
{embeddingTestResult ? <div className="text-sm text-slate-700">{embeddingTestResult}</div> : null}
{embeddingRebuildResult ? <div className="text-sm text-slate-700">{embeddingRebuildResult}</div> : null}
</div>
</Section>
<Section>
<div className="text-sm font-semibold text-slate-900">本地 Embedding 模型</div>
<div className="mt-3 grid gap-3 md:grid-cols-2">
<label className="block">
<div className="mb-1 text-xs text-slate-600">HuggingFace repo / alias</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={embeddingDownloadModel} onChange={(e) => setEmbeddingDownloadModel(e.target.value)} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">保存名称(可空)</div>
<input className="w-full rounded border px-3 py-2 text-sm" placeholder="默认用 repo 名称" value={embeddingDownloadName} onChange={(e) => setEmbeddingDownloadName(e.target.value)} />
</label>
</div>
<div className="mt-3 flex flex-wrap items-center gap-2">
<Button tone="primary" disabled={busy} onClick={downloadEmbeddingModel}>{busy ? "下载中..." : "下载模型"}</Button>
<Button
disabled={busy}
className="rounded bg-slate-900 px-3 py-2 text-sm text-white hover:bg-slate-800 disabled:opacity-50"
onClick={async () => {
setBusy(true);
setError(null);
try {
const payload: any = {
default_provider: defaultProvider,
default_target_lang: defaultTargetLang,
default_style: defaultStyle,
default_batch_size: defaultBatchSize,
default_max_retries: defaultMaxRetries,
default_enable_summary: defaultEnableSummary,
openai_base_url: openaiBaseUrl,
openai_model: openaiModel,
openai_temperature: openaiTemperature,
openai_timeout_seconds: openaiTimeoutSeconds,
};
if (openaiApiKey.trim()) payload.openai_api_key = openaiApiKey.trim();
await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/translate/settings`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
setOpenaiApiKey("");
await refresh();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
onClick={() => {
setRagEmbeddingProvider("local");
setRagEmbeddingModel(safeEmbeddingModelName(embeddingDownloadName.trim() || embeddingDownloadModel));
if (embeddingDownloadModel.includes("bge-small")) setRagEmbeddingDimensions(512);
}}
>
{busy ? "保存中…" : "保存"}
</button>
<button
disabled={busy || !settings?.openai_api_key_set}
className="rounded border border-rose-300 px-3 py-2 text-sm text-rose-700 hover:bg-rose-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("确定清除 OpenAI API Key 吗?")) return;
setBusy(true);
setError(null);
try {
await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/translate/settings`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ openai_api_key: "" }),
});
await refresh();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}}
>
清除 Key
</button>
使用该模型
</Button>
</div>
{embeddingModels.length === 0 ? (
<EmptyState>暂无本地 embedding 模型</EmptyState>
) : (
<DataTable>
<thead>
<tr>
<th className="py-2 pr-3 text-left">Name</th>
<th className="py-2 pr-3 text-left">Path</th>
<th className="py-2 pr-3 text-left">Size</th>
<th className="py-2 pr-3 text-left">Action</th>
</tr>
</thead>
<tbody>
{embeddingModels.map((model) => (
<tr key={model.name}>
<td className="py-2 pr-3 font-mono text-xs">{model.name}</td>
<td className="py-2 pr-3 font-mono text-xs">{model.path}</td>
<td className="py-2 pr-3">{model.size_bytes ? `${Math.round(model.size_bytes / 1024 / 1024)} MB` : "-"}</td>
<td className="py-2 pr-3">
<Button
size="xs"
onClick={() => {
setRagEmbeddingProvider("local");
setRagEmbeddingModel(model.name);
if (model.name.includes("bge-small")) setRagEmbeddingDimensions(512);
}}
>
使用
</Button>
</td>
</tr>
))}
</tbody>
</DataTable>
)}
</Section>
<div className="mt-3 text-xs text-slate-500">
提示:API Key 会加密存储在 DB 中,解密密钥保存在本地 `data/secrets/fernet.key`(容器内 `/secrets/fernet.key`)。
</div>
<div className="flex flex-wrap items-center gap-2">
<Button tone="primary" disabled={busy} onClick={saveSettings}>{busy ? "保存中..." : "保存配置"}</Button>
<Button
tone="danger"
disabled={busy || !settings?.openai_api_key_set}
onClick={async () => {
const ok = await confirm({ title: "清除 OpenAI API Key", message: "确定清除 OpenAI API Key 吗?", confirmLabel: "清除", tone: "danger" });
if (!ok) return;
setBusy(true);
setError(null);
try {
await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/translate/settings`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ openai_api_key: "" }),
});
await refresh();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}}
>
清除 Key
</Button>
</div>
<div className="rounded border bg-white p-4">
<div className="text-sm font-semibold">测试翻译</div>
<Section>
<div className="text-sm font-semibold text-slate-900">知识库导入</div>
<div className="mt-3 grid gap-3 md:grid-cols-2">
<label className="block">
<div className="mb-1 text-xs text-slate-600">type</div>
<select className="w-full rounded border px-3 py-2 text-sm" value={knowledgeType} onChange={(e) => setKnowledgeType(e.target.value as "term" | "document")}>
<option value="term">term</option>
<option value="document">document</option>
</select>
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">domain</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={knowledgeDomain} onChange={(e) => setKnowledgeDomain(e.target.value)} />
</label>
{knowledgeType === "term" ? (
<>
<label className="block">
<div className="mb-1 text-xs text-slate-600">term</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={knowledgeTerm} onChange={(e) => setKnowledgeTerm(e.target.value)} />
</label>
<label className="block">
<div className="mb-1 text-xs text-slate-600">translation</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={knowledgeTranslation} onChange={(e) => setKnowledgeTranslation(e.target.value)} />
</label>
</>
) : (
<label className="block md:col-span-2">
<div className="mb-1 text-xs text-slate-600">title</div>
<input className="w-full rounded border px-3 py-2 text-sm" value={knowledgeTitle} onChange={(e) => setKnowledgeTitle(e.target.value)} />
</label>
)}
<label className="block md:col-span-2">
<div className="mb-1 text-xs text-slate-600">description</div>
<textarea className="h-20 w-full rounded border px-3 py-2 text-sm" value={knowledgeDescription} onChange={(e) => setKnowledgeDescription(e.target.value)} />
</label>
<label className="block md:col-span-2">
<div className="mb-1 text-xs text-slate-600">content</div>
<textarea className="h-28 w-full rounded border px-3 py-2 text-sm" value={knowledgeContent} onChange={(e) => setKnowledgeContent(e.target.value)} />
</label>
</div>
<div className="mt-3">
<Button tone="primary" disabled={busy} onClick={saveKnowledgeItem}>{busy ? "保存中..." : "写入知识库"}</Button>
</div>
</Section>
<Section>
<div className="text-sm font-semibold text-slate-900">最近知识条目</div>
{items.length === 0 ? (
<EmptyState>暂无知识条目</EmptyState>
) : (
<DataTable>
<thead>
<tr>
<th className="py-2 pr-3 text-left">Type</th>
<th className="py-2 pr-3 text-left">Term / Title</th>
<th className="py-2 pr-3 text-left">Translation</th>
<th className="py-2 pr-3 text-left">Domain</th>
<th className="py-2 pr-3 text-left">Status</th>
<th className="py-2 pr-3 text-left">Used</th>
</tr>
</thead>
<tbody>
{items.map((item) => (
<tr key={item.id}>
<td className="py-2 pr-3">{item.item_type}</td>
<td className="py-2 pr-3">{item.term || item.title || "-"}</td>
<td className="py-2 pr-3">{item.translation || "-"}</td>
<td className="py-2 pr-3">{item.domain || "-"}</td>
<td className="py-2 pr-3">{item.status}</td>
<td className="py-2 pr-3">{item.usage_count}</td>
</tr>
))}
</tbody>
</DataTable>
)}
</Section>
<Section>
<div className="text-sm font-semibold text-slate-900">测试翻译</div>
<div className="mt-2 grid gap-3 md:grid-cols-2">
<label className="block">
<div className="mb-1 text-xs text-slate-600">target_lang</div>
@@ -318,9 +692,9 @@ export default function SettingsTranslatePage() {
</label>
</div>
<div className="mt-3 flex items-center gap-3">
<button
<Button
tone="primary"
disabled={busy}
className="rounded bg-slate-900 px-3 py-2 text-sm text-white hover:bg-slate-800 disabled:opacity-50"
onClick={async () => {
setBusy(true);
setError(null);
@@ -339,8 +713,8 @@ export default function SettingsTranslatePage() {
}
}}
>
{busy ? "测试中…" : "开始测试(openai)"}
</button>
{busy ? "测试中..." : "开始测试"}
</Button>
{testResult ? <div className="text-sm text-slate-700">OK</div> : null}
</div>
{testResult ? (
@@ -349,7 +723,7 @@ export default function SettingsTranslatePage() {
<div className="mt-1 whitespace-pre-wrap text-sm text-slate-800">{testResult}</div>
</div>
) : null}
</div>
</Section>
</div>
);
}
+9 -1
View File
@@ -1,4 +1,5 @@
import { useEffect, useState } from "react";
import { useConfirm } from "../components/feedbackContext";
import { fetchJson } from "../lib/http";
import { ORCHESTRATOR_URL } from "../lib/urls";
@@ -66,6 +67,7 @@ type YouTubeHomeScanRunResponse = {
};
export default function SettingsYouTubePage() {
const confirm = useConfirm();
const [settings, setSettings] = useState<YouTubeSettings | null>(null);
const [error, setError] = useState<string | null>(null);
const [busy, setBusy] = useState(false);
@@ -513,7 +515,13 @@ export default function SettingsYouTubePage() {
disabled={cookiesBusy || !settings?.cookies_set}
className="rounded border border-rose-300 px-3 py-2 text-sm text-rose-700 hover:bg-rose-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("确定清空已保存的 YouTube Cookies 吗?")) return;
const ok = await confirm({
title: "清空 YouTube Cookies",
message: "确定清空已保存的 YouTube Cookies 吗?",
confirmLabel: "清空",
tone: "danger",
});
if (!ok) return;
setCookiesBusy(true);
setError(null);
try {
+203 -85
View File
@@ -1,6 +1,8 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { Link, useParams } from "react-router-dom";
import StatusBadge from "../components/StatusBadge";
import { useConfirm, useToast } from "../components/feedbackContext";
import { Button } from "../components/ui";
import { fetchJson } from "../lib/http";
import { BILIBILI_PUBLISHER_URL, ORCHESTRATOR_URL, SUBTITLE_SERVICE_URL } from "../lib/urls";
import { Asset, PublishJob, SubtitleJob, Task } from "../lib/types";
@@ -66,6 +68,8 @@ type PublishReview = {
checked_at?: string | null;
};
type TaskDetailTab = "overview" | "media" | "subtitle" | "publish" | "logs";
function normalizeYouTubeSubtitleMode(value: unknown, legacyPrefer?: boolean | null): YouTubeSubtitleMode {
const mode = String(value ?? "").trim().toLowerCase();
if (mode === "off" || mode === "target" || mode === "auto_source") return mode;
@@ -75,6 +79,9 @@ function normalizeYouTubeSubtitleMode(value: unknown, legacyPrefer?: boolean | n
export default function TaskDetailPage() {
const { taskId } = useParams();
const toast = useToast();
const confirm = useConfirm();
const [activeTab, setActiveTab] = useState<TaskDetailTab>("overview");
const [task, setTask] = useState<Task | null>(null);
const [assets, setAssets] = useState<Asset[] | null>(null);
const [subtitleJobs, setSubtitleJobs] = useState<SubtitleJob[] | null>(null);
@@ -388,6 +395,12 @@ export default function TaskDetailPage() {
if (!Number.isFinite(tid) || tid <= 0) return "";
return biliTypeOptions.find((o) => o.id === tid)?.label ?? String(tid);
}, [biliTypeOptions, publishTypeid]);
const failedSubtitleJobs = useMemo(() => (subtitleJobs ?? []).filter((j) => j.status === "failed").length, [subtitleJobs]);
const runningSubtitleJobs = useMemo(
() => (subtitleJobs ?? []).filter((j) => j.status === "queued" || j.status === "running").length,
[subtitleJobs],
);
const failedPublishJobs = useMemo(() => (publishJobs ?? []).filter((j) => j.state === "failed").length, [publishJobs]);
async function loadBilibiliTypes() {
setBiliTypesBusy(true);
@@ -469,7 +482,7 @@ export default function TaskDetailPage() {
}),
});
await refresh();
alert(`Publish state=${resp.state} bvid=${resp.bvid ?? "-"}`);
toast({ kind: "success", title: "投稿任务已提交", message: `state=${resp.state} bvid=${resp.bvid ?? "-"}` });
} catch (e: unknown) {
try {
await refresh({ silent: true });
@@ -555,7 +568,7 @@ export default function TaskDetailPage() {
});
}
await refresh();
alert(`${opts.resume ? "已继续字幕任务" : "已提交字幕任务"}:${resp.job_id}`);
toast({ kind: "success", title: opts.resume ? "已继续字幕任务" : "已提交字幕任务", message: resp.job_id });
} catch (e: unknown) {
try {
await refresh({ silent: true });
@@ -691,17 +704,25 @@ export default function TaskDetailPage() {
if (!taskId) return null;
const tabs: Array<{ id: TaskDetailTab; label: string; badge?: string | number | null }> = [
{ id: "overview", label: "Overview" },
{ id: "media", label: "Media & Assets", badge: assets?.length ?? null },
{ id: "subtitle", label: "Subtitle / Render", badge: runningSubtitleJobs || failedSubtitleJobs || null },
{ id: "publish", label: "Publish", badge: publishJobs?.length ?? null },
{ id: "logs", label: "Logs", badge: logAssets.length || null },
];
return (
<div className="space-y-4">
<div className="rounded border bg-white p-4">
<div className="vr-section">
<div className="flex items-start justify-between gap-3">
<div>
<div className="text-lg font-semibold">Task Detail</div>
<div className="mt-1 font-mono text-xs text-slate-600">{taskId}</div>
</div>
<button onClick={() => refresh()} className="rounded border px-3 py-2 text-sm hover:bg-slate-50">
<Button onClick={() => refresh()}>
刷新
</button>
</Button>
</div>
{error ? <div className="mt-3 whitespace-pre-wrap break-words text-sm text-rose-700">{error}</div> : null}
@@ -729,8 +750,91 @@ export default function TaskDetailPage() {
<div className="mt-1 whitespace-pre-wrap break-words">{task.error_message}</div>
</div>
) : null}
<div className="mt-4 flex gap-1 overflow-x-auto border-b border-slate-200">
{tabs.map((tab) => {
const active = activeTab === tab.id;
return (
<button
key={tab.id}
type="button"
onClick={() => setActiveTab(tab.id)}
className={[
"mb-[-1px] inline-flex shrink-0 items-center gap-2 border-b-2 px-3 py-2 text-sm",
active
? "border-slate-900 text-slate-950"
: "border-transparent text-slate-600 hover:border-slate-300 hover:text-slate-950",
].join(" ")}
>
{tab.label}
{tab.badge ? <span className="rounded-md bg-slate-100 px-1.5 py-0.5 text-[11px] text-slate-600">{tab.badge}</span> : null}
</button>
);
})}
</div>
</div>
{activeTab === "overview" ? (
<div className="grid gap-4 lg:grid-cols-2">
<div className="vr-section">
<div className="text-sm font-semibold">Workflow</div>
<div className="mt-3 grid grid-cols-2 gap-2">
<button type="button" onClick={() => setActiveTab("media")} className="rounded-md border border-slate-200 p-3 text-left hover:bg-slate-50">
<div className="text-xs text-slate-500">Assets</div>
<div className="mt-1 text-xl font-semibold text-slate-950">{assets?.length ?? "-"}</div>
</button>
<button type="button" onClick={() => setActiveTab("subtitle")} className="rounded-md border border-slate-200 p-3 text-left hover:bg-slate-50">
<div className="text-xs text-slate-500">Subtitle Jobs</div>
<div className="mt-1 text-xl font-semibold text-slate-950">{subtitleJobs?.length ?? "-"}</div>
</button>
<button type="button" onClick={() => setActiveTab("publish")} className="rounded-md border border-slate-200 p-3 text-left hover:bg-slate-50">
<div className="text-xs text-slate-500">Publish Jobs</div>
<div className="mt-1 text-xl font-semibold text-slate-950">{publishJobs?.length ?? "-"}</div>
</button>
<button type="button" onClick={() => setActiveTab("logs")} className="rounded-md border border-slate-200 p-3 text-left hover:bg-slate-50">
<div className="text-xs text-slate-500">Logs</div>
<div className="mt-1 text-xl font-semibold text-slate-950">{logAssets.length}</div>
</button>
</div>
<div className="mt-3 flex flex-wrap gap-2">
<Button type="button" tone="primary" onClick={() => setActiveTab(rawAsset ? "subtitle" : "media")}>
{rawAsset || !isYouTubeTask ? "处理字幕" : "获取视频"}
</Button>
<Button type="button" onClick={() => setActiveTab("publish")}>
投稿设置
</Button>
<Button type="button" onClick={() => setActiveTab("logs")}>
查看日志
</Button>
</div>
</div>
<div className="vr-section">
<div className="text-sm font-semibold">Latest State</div>
<div className="mt-3 space-y-3 text-sm">
<div className="flex items-center justify-between gap-3">
<span className="text-slate-500">最终视频</span>
<span className="font-medium text-slate-950">{finalAssets.length}</span>
</div>
<div className="flex items-center justify-between gap-3">
<span className="text-slate-500">字幕运行 / 失败</span>
<span className="font-medium text-slate-950">{runningSubtitleJobs} / {failedSubtitleJobs}</span>
</div>
<div className="flex items-center justify-between gap-3">
<span className="text-slate-500">投稿失败</span>
<span className={failedPublishJobs ? "font-medium text-rose-700" : "font-medium text-slate-950"}>{failedPublishJobs}</span>
</div>
<div className="flex items-center justify-between gap-3">
<span className="text-slate-500">更新时间</span>
<span className="font-medium text-slate-950">{task ? new Date(task.updated_at).toLocaleString() : "-"}</span>
</div>
</div>
</div>
</div>
) : null}
{activeTab === "media" ? (
<>
<div className="rounded border bg-white p-4">
<div className="text-sm font-semibold">Upload / Raw Video</div>
<div className="mt-2 text-xs text-slate-500">已上传:{rawAsset ? rawAsset.storage_key : "无"}</div>
@@ -803,6 +907,94 @@ export default function TaskDetailPage() {
</div>
</div>
<div className="rounded border bg-white p-4">
<div className="text-sm font-semibold">Assets</div>
{!assets ? <div className="mt-2 text-sm text-slate-500">加载中…</div> : null}
{assets ? (
<div className="mt-2 overflow-auto">
<table className="min-w-full text-left text-sm">
<thead className="text-xs text-slate-500">
<tr>
<th className="py-2 pr-3">Kind</th>
<th className="py-2 pr-3">Key</th>
<th className="py-2 pr-3">Created</th>
<th className="py-2 pr-3">Actions</th>
</tr>
</thead>
<tbody>
{assets.map((a) => (
<tr key={a.id} className="border-t">
<td className="py-2 pr-3 text-xs">{a.kind}</td>
<td className="py-2 pr-3 font-mono text-xs">{a.storage_key}</td>
<td className="py-2 pr-3 text-xs text-slate-600">{new Date(a.created_at).toLocaleString()}</td>
<td className="py-2 pr-3">
<div className="flex flex-wrap items-center gap-2">
<a
className="rounded border px-2 py-1 text-xs hover:bg-slate-50"
href={`${ORCHESTRATOR_URL}/tasks/${taskId}/assets/${a.id}/download`}
target="_blank"
rel="noreferrer"
>
Download
</a>
{a.kind === "video_final" ? (
<button
disabled={busy}
className="rounded border border-rose-300 px-2 py-1 text-xs text-rose-700 hover:bg-rose-50 disabled:opacity-50"
onClick={async () => {
const ok = await confirm({
title: "删除最终视频资产",
message: "确定删除该最终视频资产?这会从存储中删除文件。",
confirmLabel: "删除",
tone: "danger",
});
if (!ok) return;
setBusy(true);
setError(null);
try {
await fetchJson(`${ORCHESTRATOR_URL}/tasks/${taskId}/assets/${a.id}`, { method: "DELETE" });
await refresh();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}}
>
Delete
</button>
) : null}
</div>
</td>
</tr>
))}
</tbody>
</table>
</div>
) : null}
{finalAssets.length > 0 ? (
<div className="mt-3 rounded border bg-slate-50 p-3 text-xs text-slate-700">
<div className="font-semibold text-slate-700">最终视频</div>
<div className="mt-2 flex flex-wrap gap-2">
{finalAssets.map((a) => (
<a
key={a.id}
className="rounded bg-slate-900 px-2 py-1 text-xs text-white hover:bg-slate-800"
href={`${ORCHESTRATOR_URL}/tasks/${taskId}/assets/${a.id}/download`}
target="_blank"
rel="noreferrer"
>
下载:{a.storage_key.split("/").slice(-1)[0]}
</a>
))}
</div>
</div>
) : null}
</div>
</>
) : null}
{activeTab === "subtitle" ? (
<div className="rounded border bg-white p-4">
<div className="text-sm font-semibold">Subtitle</div>
<div className="mt-3 grid gap-3 md:grid-cols-2">
@@ -1070,7 +1262,9 @@ export default function TaskDetailPage() {
) : null}
</div>
</div>
) : null}
{activeTab === "logs" ? (
<div className="rounded border bg-white p-4">
<div className="flex items-center justify-between gap-3">
<div className="text-sm font-semibold">Logs</div>
@@ -1100,86 +1294,9 @@ export default function TaskDetailPage() {
className="mt-2 h-72 w-full rounded border bg-slate-50 p-2 font-mono text-xs text-slate-800"
/>
</div>
) : null}
<div className="rounded border bg-white p-4">
<div className="text-sm font-semibold">Assets</div>
{!assets ? <div className="mt-2 text-sm text-slate-500">加载中…</div> : null}
{assets ? (
<div className="mt-2 overflow-auto">
<table className="min-w-full text-left text-sm">
<thead className="text-xs text-slate-500">
<tr>
<th className="py-2 pr-3">Kind</th>
<th className="py-2 pr-3">Key</th>
<th className="py-2 pr-3">Created</th>
<th className="py-2 pr-3">Actions</th>
</tr>
</thead>
<tbody>
{assets.map((a) => (
<tr key={a.id} className="border-t">
<td className="py-2 pr-3 text-xs">{a.kind}</td>
<td className="py-2 pr-3 font-mono text-xs">{a.storage_key}</td>
<td className="py-2 pr-3 text-xs text-slate-600">{new Date(a.created_at).toLocaleString()}</td>
<td className="py-2 pr-3">
<div className="flex flex-wrap items-center gap-2">
<a
className="rounded border px-2 py-1 text-xs hover:bg-slate-50"
href={`${ORCHESTRATOR_URL}/tasks/${taskId}/assets/${a.id}/download`}
target="_blank"
rel="noreferrer"
>
Download
</a>
{a.kind === "video_final" ? (
<button
disabled={busy}
className="rounded border border-rose-300 px-2 py-1 text-xs text-rose-700 hover:bg-rose-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("确定删除该最终视频资产?(会从存储中删除)")) return;
setBusy(true);
setError(null);
try {
await fetchJson(`${ORCHESTRATOR_URL}/tasks/${taskId}/assets/${a.id}`, { method: "DELETE" });
await refresh();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
setBusy(false);
}
}}
>
Delete
</button>
) : null}
</div>
</td>
</tr>
))}
</tbody>
</table>
</div>
) : null}
{finalAssets.length > 0 ? (
<div className="mt-3 rounded border bg-slate-50 p-3 text-xs text-slate-700">
<div className="font-semibold text-slate-700">最终视频</div>
<div className="mt-2 flex flex-wrap gap-2">
{finalAssets.map((a) => (
<a
key={a.id}
className="rounded bg-slate-900 px-2 py-1 text-xs text-white hover:bg-slate-800"
href={`${ORCHESTRATOR_URL}/tasks/${taskId}/assets/${a.id}/download`}
target="_blank"
rel="noreferrer"
>
下载:{a.storage_key.split("/").slice(-1)[0]}
</a>
))}
</div>
</div>
) : null}
</div>
{activeTab === "publish" ? (
<div className="rounded border bg-white p-4">
<div className="text-sm font-semibold">Publish(Bilibili)</div>
<div className="mt-2 text-xs text-slate-500">
@@ -1438,7 +1555,7 @@ export default function TaskDetailPage() {
const meta = JSON.parse(publishMetaText);
if (!meta || typeof meta !== "object" || Array.isArray(meta)) throw new Error("meta must be a JSON object");
await savePublishMetaFile(meta);
alert("已保存到 publish_meta.json");
toast({ kind: "success", title: "已保存", message: "publish_meta.json 已更新。" });
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
@@ -1536,6 +1653,7 @@ export default function TaskDetailPage() {
) : null}
</div>
</div>
) : null}
</div>
);
}
+53 -25
View File
@@ -1,6 +1,8 @@
import { useEffect, useMemo, useState } from "react";
import { Link, useSearchParams } from "react-router-dom";
import StatusBadge from "../components/StatusBadge";
import { useConfirm, useToast } from "../components/feedbackContext";
import { Button, PageHeader } from "../components/ui";
import { fetchJson } from "../lib/http";
import { ORCHESTRATOR_URL } from "../lib/urls";
import { Task, TaskStatus } from "../lib/types";
@@ -31,6 +33,8 @@ function formatRecentFailedResumeSummary(resp: RecentFailedResumeResponse): stri
}
export default function TasksPage() {
const confirm = useConfirm();
const toast = useToast();
const [searchParams, setSearchParams] = useSearchParams();
const [tasks, setTasks] = useState<Task[] | null>(null);
const [error, setError] = useState<string | null>(null);
@@ -76,25 +80,28 @@ export default function TasksPage() {
return (
<div className="space-y-4">
<div className="rounded border bg-white p-4">
<div className="flex items-center justify-between gap-3">
<div>
<div className="text-lg font-semibold">Tasks</div>
<div className="text-sm text-slate-600">任务列表与状态筛选</div>
</div>
<div className="flex flex-wrap items-center gap-2">
<button
<PageHeader
title="Tasks"
description="任务列表与状态筛选"
actions={
<>
<Button
disabled={bulkResumeBusy}
className="rounded border px-3 py-2 text-sm text-slate-700 hover:bg-slate-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("一键继续最近 24 小时内失败任务?会批量恢复字幕任务,并在渲染后自动投稿。")) return;
const ok = await confirm({
title: "继续最近失败任务",
message: "一键继续最近 24 小时内失败任务?会批量恢复字幕任务,并在渲染后自动投稿。",
confirmLabel: "继续",
tone: "warning",
});
if (!ok) return;
setBulkResumeBusy(true);
setError(null);
try {
const resp = await fetchJson<RecentFailedResumeResponse>(`${ORCHESTRATOR_URL}/tasks/actions/resume_failed_recent`, {
method: "POST",
});
alert(formatRecentFailedResumeSummary(resp));
toast({ kind: "info", title: "批量恢复已处理", message: formatRecentFailedResumeSummary(resp) });
await reloadTasks();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
@@ -104,13 +111,15 @@ export default function TasksPage() {
}}
>
一键继续24h失败任务
</button>
<Link to="/tasks/new" className="rounded bg-slate-900 px-3 py-2 text-sm text-white hover:bg-slate-800">
</Button>
<Link to="/tasks/new" className="rounded-md bg-slate-900 px-3 py-2 text-sm text-white hover:bg-slate-800">
新建任务
</Link>
</div>
</div>
</>
}
/>
<div className="vr-section">
<div className="mt-3 flex flex-wrap items-center gap-2">
<div className="text-xs text-slate-600">状态:</div>
{statusOptions.map((s) => {
@@ -125,7 +134,7 @@ export default function TasksPage() {
setSearchParams(sp);
}}
className={[
"rounded border px-2 py-1 text-xs",
"rounded-md border px-2 py-1 text-xs",
active ? "border-slate-900 bg-slate-900 text-white" : "bg-white text-slate-700 hover:bg-slate-50",
].join(" ")}
>
@@ -136,11 +145,11 @@ export default function TasksPage() {
</div>
</div>
<div className="rounded border bg-white p-4">
<div className="vr-section">
{error ? <div className="text-sm text-rose-700">{error}</div> : null}
{!tasks ? <div className="text-sm text-slate-500">加载中…</div> : null}
{tasks ? (
<div className="overflow-auto">
<div className="overflow-auto rounded-md border border-slate-200">
<table className="min-w-full text-left text-sm">
<thead className="text-xs text-slate-500">
<tr>
@@ -169,9 +178,12 @@ export default function TasksPage() {
<div className="text-xs text-slate-500">{t.source_type}</div>
<div className="max-w-[28rem] truncate text-xs text-slate-700">{t.source_url ?? "-"}</div>
{t.error_message ? (
<div className="mt-1 max-w-[28rem] truncate text-xs text-rose-700" title={t.error_message}>
<details className="mt-1 max-w-[28rem] text-xs text-rose-700">
<summary className="cursor-pointer truncate" title={t.error_message}>
{t.error_message}
</div>
</summary>
<div className="mt-1 whitespace-pre-wrap break-words rounded-md bg-rose-50 p-2">{t.error_message}</div>
</details>
) : null}
</td>
<td className="py-2 pr-3">
@@ -184,14 +196,19 @@ export default function TasksPage() {
disabled={actionBusyId === t.id}
className="rounded border px-2 py-1 text-xs hover:bg-slate-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("继续字幕任务?会尽量从已有产物处继续(resume=true)。")) return;
const ok = await confirm({
title: "继续字幕任务",
message: "会尽量从已有产物处继续(resume=true)。",
confirmLabel: "继续",
});
if (!ok) return;
setActionBusyId(t.id);
setError(null);
try {
const resp = await fetchJson<SubtitleActionResponse>(`${ORCHESTRATOR_URL}/tasks/${t.id}/actions/subtitle_resume`, {
method: "POST",
});
alert(`已提交继续任务:${resp.job_id}`);
toast({ kind: "success", title: "已提交继续任务", message: resp.job_id });
await reloadTasks();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
@@ -208,7 +225,12 @@ export default function TasksPage() {
disabled={actionBusyId === t.id}
className="rounded border px-2 py-1 text-xs hover:bg-slate-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("重试 YouTube 下载(会重新拉取视频/元信息/封面)?")) return;
const ok = await confirm({
title: "重试 YouTube 下载",
message: "会重新拉取视频、元信息和封面。",
confirmLabel: "重试",
});
if (!ok) return;
setActionBusyId(t.id);
setError(null);
try {
@@ -229,7 +251,13 @@ export default function TasksPage() {
disabled={actionBusyId === t.id}
className="rounded border px-2 py-1 text-xs hover:bg-slate-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("启动 YouTube 自动模式?会继续执行下载、字幕、压制和自动投稿。")) return;
const ok = await confirm({
title: "启动 YouTube 自动模式",
message: "会继续执行下载、字幕、压制和自动投稿。",
confirmLabel: "启动",
tone: "warning",
});
if (!ok) return;
setActionBusyId(t.id);
setError(null);
try {
@@ -237,7 +265,7 @@ export default function TasksPage() {
`${ORCHESTRATOR_URL}/tasks/${t.id}/actions/auto_youtube_start`,
{ method: "POST" },
);
alert(`已提交自动任务:${resp.pipeline_job_id}`);
toast({ kind: "success", title: "已提交自动任务", message: resp.pipeline_job_id });
await reloadTasks();
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
+51 -25
View File
@@ -1,6 +1,8 @@
import { useEffect, useMemo, useRef, useState } from "react";
import { Link } from "react-router-dom";
import StatusBadge from "../components/StatusBadge";
import { useConfirm, useToast } from "../components/feedbackContext";
import { Button, PageHeader } from "../components/ui";
import { fetchJson } from "../lib/http";
import { ORCHESTRATOR_URL } from "../lib/urls";
import { Asset, Task } from "../lib/types";
@@ -62,6 +64,8 @@ function formatWorkdirSummary(result: WorkdirMaintenance, mode: "scan" | "cleanu
}
export default function VideosPage() {
const confirm = useConfirm();
const toast = useToast();
const [items, setItems] = useState<ConvertedVideoItem[] | null>(null);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
@@ -110,19 +114,16 @@ export default function VideosPage() {
return (
<div className="space-y-4">
<div className="rounded border bg-white p-4">
<div className="flex items-center justify-between gap-3">
<div>
<div className="text-lg font-semibold">Videos</div>
<div className="text-sm text-slate-600">已经转换完成(存在 video_final)的任务列表</div>
</div>
<div className="flex items-center gap-2">
<button onClick={() => refresh()} className="rounded border px-3 py-2 text-sm hover:bg-slate-50">
<PageHeader
title="Videos"
description="已经转换完成(存在 video_final)的任务列表"
actions={
<>
<Button onClick={() => refresh()}>
刷新
</button>
<button
</Button>
<Button
disabled={busy}
className="rounded border px-3 py-2 text-sm hover:bg-slate-50 disabled:opacity-50"
onClick={async () => {
setBusy(true);
setError(null);
@@ -138,12 +139,18 @@ export default function VideosPage() {
}}
>
扫描临时目录
</button>
<button
</Button>
<Button
disabled={busy}
className="rounded border border-amber-300 px-3 py-2 text-sm text-amber-800 hover:bg-amber-50 disabled:opacity-50"
tone="warning"
onClick={async () => {
if (!confirm("确定清理可安全回收的临时目录吗?不会删除运行中的任务目录。")) return;
const ok = await confirm({
title: "清理临时目录",
message: "确定清理可安全回收的临时目录吗?不会删除运行中的任务目录。",
confirmLabel: "清理",
tone: "warning",
});
if (!ok) return;
setBusy(true);
setError(null);
setMaintenanceSummary(null);
@@ -160,7 +167,7 @@ export default function VideosPage() {
}}
>
清理临时目录
</button>
</Button>
<label className="flex items-center gap-2 rounded border px-3 py-2 text-sm text-slate-700 hover:bg-slate-50">
<input
ref={selectAllRef}
@@ -177,12 +184,18 @@ export default function VideosPage() {
/>
全选
</label>
<button
<Button
disabled={busy || selectedCount === 0}
className="rounded border border-rose-300 px-3 py-2 text-sm text-rose-700 hover:bg-rose-50 disabled:opacity-50"
tone="danger"
onClick={async () => {
if (!items || selectedCount === 0) return;
if (!confirm(`确定删除选中的 ${selectedCount} 个最终视频(video_final)吗?`)) return;
const ok = await confirm({
title: "删除最终视频",
message: `确定删除选中的 ${selectedCount} 个最终视频(video_final)吗?`,
confirmLabel: "删除",
tone: "danger",
});
if (!ok) return;
setBusy(true);
setError(null);
setDeleteSummary(null);
@@ -206,6 +219,11 @@ export default function VideosPage() {
results,
);
setDeleteSummary(formatBulkDeleteSummary(summary));
toast({
kind: summary.failureCount === 0 ? "success" : "warning",
title: "删除操作完成",
message: formatBulkDeleteSummary(summary),
});
setSelected(new Set(summary.failures.map((failure) => failure.assetId)));
await refresh();
} catch (e: unknown) {
@@ -216,12 +234,14 @@ export default function VideosPage() {
}}
>
删除选中{selectedCount ? ` (${selectedCount})` : ""}
</button>
<Link to="/tasks/new" className="rounded bg-slate-900 px-3 py-2 text-sm text-white hover:bg-slate-800">
</Button>
<Link to="/tasks/new" className="rounded-md bg-slate-900 px-3 py-2 text-sm text-white hover:bg-slate-800">
新建任务
</Link>
</div>
</div>
</>
}
/>
<div className="vr-section">
{error ? <div className="mt-3 text-sm text-rose-700">{error}</div> : null}
{deleteSummary ? <div className="mt-3 text-sm text-slate-700">{deleteSummary}</div> : null}
{maintenanceSummary ? <div className="mt-3 text-sm text-slate-700">{maintenanceSummary}</div> : null}
@@ -230,7 +250,7 @@ export default function VideosPage() {
<div className="rounded border bg-white p-4">
{!items ? <div className="text-sm text-slate-500">加载中…</div> : null}
{items ? (
<div className="overflow-auto">
<div className="overflow-auto rounded-md border border-slate-200">
<table className="min-w-full text-left text-sm">
<thead className="text-xs text-slate-500">
<tr>
@@ -310,7 +330,13 @@ export default function VideosPage() {
disabled={busy}
className="rounded border border-rose-300 px-2 py-1 text-xs text-rose-700 hover:bg-rose-50 disabled:opacity-50"
onClick={async () => {
if (!confirm("确定删除这个最终视频(video_final)吗?")) return;
const ok = await confirm({
title: "删除最终视频",
message: "确定删除这个最终视频(video_final)吗?",
confirmLabel: "删除",
tone: "danger",
});
if (!ok) return;
setBusy(true);
setError(null);
setDeleteSummary(null);
+31 -15
View File
@@ -1,4 +1,6 @@
import { useEffect, useState } from "react";
import { useConfirm, useToast } from "../components/feedbackContext";
import { Button, PageHeader } from "../components/ui";
import { fetchJson } from "../lib/http";
import { YOUTUBE_INGEST_URL } from "../lib/urls";
import { SourceLicense, YouTubeSource } from "../lib/types";
@@ -27,6 +29,8 @@ function currentValue<T>(source: YouTubeSource, draft: SourceDraft | undefined,
}
export default function YouTubeSourcesPage() {
const confirm = useConfirm();
const toast = useToast();
const [sources, setSources] = useState<YouTubeSource[] | null>(null);
const [drafts, setDrafts] = useState<Record<string, SourceDraft>>({});
const [busyMap, setBusyMap] = useState<Record<string, boolean>>({});
@@ -109,7 +113,11 @@ export default function YouTubeSourcesPage() {
if (failures.length > 0) {
setError(failures.slice(0, 3).join("\n"));
}
alert(`已处理 ${items.length} 个输入,成功 ${success} 个,失败 ${failures.length} 个。`);
toast({
kind: failures.length > 0 ? "warning" : "success",
title: "订阅源已处理",
message: `已处理 ${items.length} 个输入,成功 ${success} 个,失败 ${failures.length} 个。`,
});
} finally {
setAdding(false);
}
@@ -153,9 +161,11 @@ export default function YouTubeSourcesPage() {
}),
});
await refresh();
alert(
`discovered=${res.discovered_count} created=${res.created_task_ids.length} skipped=${res.skipped_duplicates} started_pipeline=${(res.started_pipeline_job_ids ?? []).length}`,
);
toast({
kind: "info",
title: "扫描完成",
message: `discovered=${res.discovered_count} created=${res.created_task_ids.length} skipped=${res.skipped_duplicates} started_pipeline=${(res.started_pipeline_job_ids ?? []).length}`,
});
} catch (e: unknown) {
setError(e instanceof Error ? e.message : String(e));
} finally {
@@ -164,7 +174,13 @@ export default function YouTubeSourcesPage() {
}
async function deleteSource(source: YouTubeSource) {
if (!window.confirm(`删除订阅源?\n${source.display_name || source.source_url || source.source_id}`)) return;
const ok = await confirm({
title: "删除订阅源",
message: source.display_name || source.source_url || source.source_id,
confirmLabel: "删除",
tone: "danger",
});
if (!ok) return;
setBusy(`delete:${source.id}`, true);
setError(null);
try {
@@ -186,16 +202,16 @@ export default function YouTubeSourcesPage() {
return (
<div className="space-y-4">
<div className="rounded border bg-white p-4">
<div className="flex items-center justify-between gap-4">
<div>
<div className="text-lg font-semibold">YouTube Sources</div>
<div className="text-sm text-slate-600">订阅 YouTube 频道 / 播放列表,后台按间隔自动扫描,新视频可直接进入自动模式。</div>
</div>
<button onClick={() => refresh()} className="rounded border px-3 py-2 text-sm hover:bg-slate-50">
<PageHeader
title="YouTube Sources"
description="订阅 YouTube 频道 / 播放列表,后台按间隔自动扫描,新视频可直接进入自动模式。"
actions={
<Button onClick={() => refresh()}>
刷新
</button>
</div>
</Button>
}
/>
<div className="vr-section">
{error ? <div className="mt-3 whitespace-pre-wrap text-sm text-rose-700">{error}</div> : null}
</div>
@@ -281,7 +297,7 @@ export default function YouTubeSourcesPage() {
{!sources ? <div className="mt-2 text-sm text-slate-500">加载中...</div> : null}
{sources && sources.length === 0 ? <div className="mt-2 text-sm text-slate-500">暂无订阅源。</div> : null}
{sources ? (
<div className="mt-3 overflow-auto">
<div className="mt-3 overflow-auto rounded-md border border-slate-200">
<table className="min-w-full text-left text-sm">
<thead className="text-xs text-slate-500">
<tr>
+244
View File
@@ -7,3 +7,247 @@ body {
height: 100%;
}
@layer base {
:focus-visible {
outline: 2px solid rgb(15 23 42);
outline-offset: 2px;
}
.dark :focus-visible {
outline-color: rgb(226 232 240);
}
}
@layer components {
.vr-section {
@apply rounded-md border border-slate-200 bg-white p-4 shadow-sm;
}
.vr-table-wrap {
@apply overflow-auto rounded-md border border-slate-200;
}
.vr-table {
@apply min-w-full border-collapse text-left text-sm;
}
.vr-table thead {
@apply sticky top-0 z-10 bg-slate-50 text-xs font-medium uppercase tracking-normal text-slate-500;
}
.vr-table th {
@apply border-b border-slate-200 px-3 py-2 align-bottom;
}
.vr-table td {
@apply border-b border-slate-100 px-3 py-2 align-top;
}
.vr-table tbody tr {
@apply hover:bg-slate-50/70;
}
.vr-table tbody tr:last-child td {
@apply border-b-0;
}
.vr-input {
@apply rounded-md border border-slate-300 px-3 py-2 text-sm outline-none focus:border-slate-500;
}
.vr-chip {
@apply inline-flex items-center rounded-md border border-slate-200 bg-slate-50 px-2 py-1 text-xs text-slate-700;
}
main table {
@apply min-w-full border-collapse text-left text-sm;
}
main thead {
@apply sticky top-0 z-10 bg-slate-50 text-xs font-medium uppercase tracking-normal text-slate-500;
}
main th {
@apply border-b border-slate-200 px-3 py-2 align-bottom;
}
main td {
@apply border-b border-slate-100 px-3 py-2 align-top;
}
main tbody tr {
@apply hover:bg-slate-50/70;
}
main tbody tr:last-child td {
@apply border-b-0;
}
}
@layer utilities {
.dark body {
background: rgb(2 6 23);
color: rgb(226 232 240);
}
.dark .bg-white {
background-color: rgb(15 23 42);
}
.dark .bg-slate-50 {
background-color: rgb(2 6 23);
}
.dark .bg-slate-100 {
background-color: rgb(30 41 59);
}
.dark .bg-slate-200 {
background-color: rgb(51 65 85);
}
.dark .border,
.dark .border-slate-100,
.dark .border-slate-200,
.dark .border-slate-300 {
border-color: rgb(51 65 85);
}
.dark .shadow-sm,
.dark .shadow-lg,
.dark .shadow-xl {
box-shadow: none;
}
.dark .text-slate-950,
.dark .text-slate-900,
.dark .text-slate-800,
.dark .text-slate-700 {
color: rgb(226 232 240);
}
.dark .text-slate-600,
.dark .text-slate-500,
.dark .text-slate-400 {
color: rgb(148 163 184);
}
.dark .text-sky-700,
.dark .text-sky-900,
.dark .text-sky-950 {
color: rgb(125 211 252);
}
.dark .text-emerald-700,
.dark .text-emerald-900,
.dark .text-emerald-950 {
color: rgb(110 231 183);
}
.dark .text-rose-700,
.dark .text-rose-900,
.dark .text-rose-950 {
color: rgb(253 164 175);
}
.dark .text-amber-700,
.dark .text-amber-800,
.dark .text-amber-900,
.dark .text-amber-950 {
color: rgb(252 211 77);
}
.dark .bg-sky-50,
.dark .bg-sky-100 {
background-color: rgb(8 47 73);
border-color: rgb(14 116 144);
}
.dark .bg-emerald-50,
.dark .bg-emerald-100 {
background-color: rgb(6 78 59);
border-color: rgb(5 150 105);
}
.dark .bg-rose-50,
.dark .bg-rose-100 {
background-color: rgb(76 5 25);
border-color: rgb(190 18 60);
}
.dark .bg-amber-50,
.dark .bg-amber-100 {
background-color: rgb(69 26 3);
border-color: rgb(180 83 9);
}
.dark .bg-indigo-100 {
background-color: rgb(49 46 129);
}
.dark .bg-fuchsia-100 {
background-color: rgb(112 26 117);
}
.dark .text-indigo-800,
.dark .text-fuchsia-800 {
color: rgb(221 214 254);
}
.dark input,
.dark textarea,
.dark select {
background-color: rgb(15 23 42);
border-color: rgb(71 85 105);
color: rgb(226 232 240);
}
.dark input::placeholder,
.dark textarea::placeholder {
color: rgb(100 116 139);
}
.dark button:hover,
.dark a:hover {
text-decoration-color: currentColor;
}
.dark .hover\:bg-slate-50:hover,
.dark .hover\:bg-slate-100:hover {
background-color: rgb(30 41 59);
}
.dark main thead,
.dark .vr-table thead {
background-color: rgb(15 23 42);
color: rgb(148 163 184);
}
.dark main th,
.dark .vr-table th {
border-color: rgb(51 65 85);
}
.dark main td,
.dark .vr-table td {
border-color: rgb(30 41 59);
}
.dark main tbody tr:hover,
.dark .vr-table tbody tr:hover {
background-color: rgb(15 23 42 / 0.65);
}
.dark .vr-section,
.dark .vr-table-wrap {
border-color: rgb(51 65 85);
background-color: rgb(15 23 42);
}
.dark .vr-input,
.dark .vr-chip {
border-color: rgb(71 85 105);
background-color: rgb(15 23 42);
color: rgb(226 232 240);
}
}
+1 -1
View File
@@ -1,9 +1,9 @@
/** @type {import('tailwindcss').Config} */
export default {
darkMode: "class",
content: ["./index.html", "./src/**/*.{ts,tsx}"],
theme: {
extend: {},
},
plugins: [],
};
+21
View File
@@ -53,3 +53,24 @@ def test_auto_migrate_cached_per_pid(monkeypatch) -> None:
("auto_migrate_engine", "engine-for-postgresql+psycopg://user:pass@db/app"),
("auto_migrate_engine", "engine-for-postgresql+psycopg://user:pass@db/app"),
]
def test_auto_migrate_force_bypasses_process_cache(monkeypatch) -> None:
auto_migrate_module._auto_migrate_cached.cache_clear()
calls: list[tuple[str, object]] = []
monkeypatch.setattr(auto_migrate_module, "get_engine", lambda url: f"engine-for-{url}")
monkeypatch.setattr(
auto_migrate_module,
"auto_migrate_engine",
lambda engine: calls.append(("auto_migrate_engine", engine)),
)
monkeypatch.setattr(auto_migrate_module.os, "getpid", lambda: 3001)
auto_migrate_module.auto_migrate("postgresql+psycopg://user:pass@db/app")
auto_migrate_module.auto_migrate("postgresql+psycopg://user:pass@db/app", force=True)
assert calls == [
("auto_migrate_engine", "engine-for-postgresql+psycopg://user:pass@db/app"),
("auto_migrate_engine", "engine-for-postgresql+psycopg://user:pass@db/app"),
]
+4
View File
@@ -21,6 +21,10 @@ class ModelDownloadsTests(unittest.TestCase):
def test_openvino_known_size_maps_to_official_repo(self) -> None:
self.assertEqual(resolve_model_repo_id("openvino", "large-v3"), "OpenVINO/whisper-large-v3-fp16-ov")
def test_embedding_known_alias_maps_to_repo(self) -> None:
self.assertEqual(resolve_model_repo_id("embedding", "bge-small-zh-v1.5"), "BAAI/bge-small-zh-v1.5")
self.assertEqual(default_model_dir_name("embedding", "bge-small-zh-v1.5"), "bge-small-zh-v1.5")
def test_download_model_snapshot_uses_repo_mapping_and_writes_dest(self) -> None:
fake_hf = types.ModuleType("huggingface_hub")
seen: dict[str, object] = {}
+32
View File
@@ -0,0 +1,32 @@
from __future__ import annotations
from videoroll.utils.resources import cpu_percent_between, memory_stats_from_meminfo
def test_cpu_percent_between_snapshots() -> None:
# total delta=100, idle delta=25, busy=75
assert cpu_percent_between((1000, 500), (1100, 525)) == 75.0
def test_cpu_percent_between_rejects_non_positive_delta() -> None:
assert cpu_percent_between((1000, 500), (1000, 500)) is None
def test_memory_stats_from_meminfo_uses_memavailable() -> None:
stats = memory_stats_from_meminfo(
"\n".join(
[
"MemTotal: 1000 kB",
"MemFree: 100 kB",
"MemAvailable: 250 kB",
"Buffers: 50 kB",
"Cached: 100 kB",
]
)
)
assert stats is not None
assert stats["total_bytes"] == 1000 * 1024
assert stats["available_bytes"] == 250 * 1024
assert stats["used_bytes"] == 750 * 1024
assert stats["percent"] == 75.0
+71
View File
@@ -263,6 +263,77 @@ class TranslateResumeTests(unittest.TestCase):
self.assertIn('"summary": "summary-start"', str(second_prompt))
self.assertIn('"idx": 3', str(second_prompt))
def test_rag_context_provider_is_included_in_prompt(self) -> None:
source_segments = [Segment(start=0.0, end=1.0, text="Rush B with an AWP")]
seen_requests: list[dict[str, object]] = []
responses = [
_FakeResponse(
{
"choices": [
{
"message": {
"content": json.dumps(
{
"updated_summary": "cs2",
"translations": [{"idx": 1, "text": "快冲 B 点,带着 AWP 狙击枪"}],
},
ensure_ascii=False,
)
}
}
]
}
)
]
class FakeClient:
def __init__(self, *_args: object, **_kwargs: object) -> None:
pass
def __enter__(self) -> "FakeClient":
return self
def __exit__(self, _exc_type: object, _exc: object, _tb: object) -> None:
return None
def post(self, _url: str, *, headers: dict[str, str], json: dict[str, object]) -> _FakeResponse:
del headers
seen_requests.append(json)
return responses.pop(0)
def rag_context_provider(batch: list[Segment], start_idx: int, summary: str) -> dict[str, object]:
self.assertEqual([seg.text for seg in batch], ["Rush B with an AWP"])
self.assertEqual(start_idx, 0)
self.assertEqual(summary, "")
return {
"term_cards": [
{
"term": "AWP",
"translation": "AWP 狙击枪",
"domain": "CS2",
"description": "Counter-Strike 系列中的狙击枪。",
}
]
}
with patch.object(processing, "create_openai_http_client", lambda _timeout: FakeClient()):
translated, summary = processing.translate_segments_openai_with_summary(
source_segments,
target_lang="zh",
style="自然",
api_key="test-key",
base_url="https://example.invalid/v1",
model="fake-model",
batch_size=1,
rag_context_provider=rag_context_provider,
)
self.assertEqual([seg.text for seg in translated], ["快冲 B 点,带着 AWP 狙击枪"])
self.assertEqual(summary, "cs2")
prompt = str(seen_requests[0]["messages"][1]["content"])
self.assertIn("rag_context", prompt)
self.assertIn("AWP 狙击枪", prompt)
if __name__ == "__main__":
unittest.main()
+587
View File
@@ -0,0 +1,587 @@
from __future__ import annotations
from videoroll.ai.client import OpenAIChatConfig
from videoroll.apps.subtitle_service.rag import (
_fallback_search_queries,
_search_url_with_params,
build_knowledge_embedding_text,
embedding_model_key,
fetch_search_evidence,
normalize_term,
rag_settings_from_translate_settings,
rebuild_knowledge_embeddings,
search_knowledge,
should_research_term,
)
from videoroll.apps.subtitle_service.embeddings import (
assert_embedding_dimensions,
embed_text,
embedding_settings_from_translate_settings,
safe_embedding_model_name,
)
def test_normalize_term_collapses_case_and_separators() -> None:
assert normalize_term(" Rush-B ") == "rush b"
assert normalize_term("Minecraft_Forge") == "minecraft forge"
def test_rag_settings_from_translate_settings_clamps_values() -> None:
cfg = rag_settings_from_translate_settings(
{
"rag_enabled": True,
"rag_top_k": 99,
"rag_min_score": -1,
"rag_embedding_provider": "local",
"rag_embedding_model": "embed-small",
"rag_embedding_dimensions": 99999,
"rag_embedding_model_dir": "/models/embeddings",
"rag_embedding_device": "cpu",
"rag_auto_discover_terms": True,
"rag_auto_learn_terms": True,
"rag_search_enabled": True,
"rag_search_url": "https://search.example",
"rag_domain": "CS2",
}
)
assert cfg.enabled is True
assert cfg.top_k == 30
assert cfg.min_score == 0.0
assert cfg.embedding_provider == "local"
assert cfg.embedding_model == "embed-small"
assert cfg.embedding_dimensions == 4096
assert cfg.embedding_model_dir == "/models/embeddings"
assert cfg.embedding_device == "cpu"
assert cfg.auto_discover_terms is True
assert cfg.auto_learn_terms is True
assert cfg.search_enabled is True
assert cfg.search_url == "https://search.example"
assert cfg.domain == "CS2"
def test_build_knowledge_embedding_text_includes_term_fields() -> None:
text = build_knowledge_embedding_text(
item_type="term",
term="AWP",
translation="AWP 狙击枪",
domain="CS2",
aliases=["Magnum Sniper Rifle"],
description="Counter-Strike 系列中的狙击枪。",
)
assert "term: AWP" in text
assert "translation: AWP 狙击枪" in text
assert "domain: CS2" in text
assert "Magnum Sniper Rifle" in text
def test_should_research_term_skips_logic_variables() -> None:
decision = should_research_term("Q", domain="技术/逻辑", context="truth table for P and Q")
assert decision["should_research"] is False
assert decision["should_persist"] is False
assert decision["category"] == "local_variable"
def test_should_research_term_keeps_common_terms_context_only() -> None:
decision = should_research_term("truth table", domain="logic")
assert decision["should_research"] is False
assert decision["should_persist"] is False
assert decision["category"] == "context_only"
assert decision["translation"] == "真值表"
def test_fallback_search_queries_include_definition_and_target_language() -> None:
queries = _fallback_search_queries("Hopper Minecart", domain="Minecraft", target_lang="zh")
assert queries[0] == "Minecraft Hopper Minecart"
assert any("definition" in query for query in queries)
assert any("中文" in query for query in queries)
def test_embedding_settings_from_translate_settings_supports_local_provider() -> None:
cfg = embedding_settings_from_translate_settings(
{
"rag_embedding_provider": "local",
"rag_embedding_model": "BAAI--bge-small-zh-v1.5",
"rag_embedding_dimensions": 512,
"rag_embedding_model_dir": "/models/embeddings",
"rag_embedding_device": "cpu",
"openai_api_key": "unused",
"openai_base_url": "https://example.invalid/v1",
"openai_timeout_seconds": 30,
}
)
assert cfg.provider == "local"
assert cfg.model == "BAAI--bge-small-zh-v1.5"
assert cfg.dimensions == 512
def test_local_embedding_openvino_device_uses_openvino_backend(monkeypatch) -> None:
from videoroll.apps.subtitle_service import embeddings as embeddings_module
calls: list[tuple[str, str, str]] = []
def fake_embed(path, text, *, device):
calls.append((str(path), text, device))
return [0.1, 0.2]
monkeypatch.setattr(embeddings_module, "_embed_text_openvino", fake_embed)
cfg = embedding_settings_from_translate_settings(
{
"rag_embedding_provider": "local",
"rag_embedding_model": "embed-small",
"rag_embedding_dimensions": 2,
"rag_embedding_model_dir": "/models/embeddings",
"rag_embedding_device": "openvino:GPU",
}
)
assert embed_text("hello", settings=cfg) == [0.1, 0.2]
assert calls == [("/models/embeddings/embed-small", "hello", "GPU")]
def test_safe_embedding_model_name_matches_repo_style() -> None:
assert safe_embedding_model_name("BAAI/bge-small-zh-v1.5") == "BAAI--bge-small-zh-v1.5"
def test_assert_embedding_dimensions() -> None:
assert_embedding_dimensions([0.1, 0.2], 2)
try:
assert_embedding_dimensions([0.1], 2)
except RuntimeError as e:
assert "dimension mismatch" in str(e)
else:
raise AssertionError("expected dimension mismatch")
class _Result:
def __init__(self, rows: list[object]) -> None:
self._rows = rows
def all(self) -> list[object]:
return self._rows
class _Row:
def __init__(self, **kwargs: object) -> None:
self._mapping = kwargs
class _NestedTransaction:
def __enter__(self) -> "_NestedTransaction":
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> bool:
return False
class _FakeKnowledgeDb:
def __init__(self, rows: list[object]) -> None:
self.rows = rows
self.calls: list[tuple[str, dict[str, object]]] = []
def execute(self, stmt: object, params: dict[str, object] | None = None) -> _Result:
sql = str(stmt)
self.calls.append((sql, dict(params or {})))
if sql.strip().startswith("SELECT"):
return _Result(self.rows)
return _Result([])
def begin_nested(self) -> _NestedTransaction:
return _NestedTransaction()
def test_search_knowledge_filters_by_embedding_model() -> None:
db = _FakeKnowledgeDb([])
search_knowledge(
db, # type: ignore[arg-type]
query_embedding=[0.1, 0.2],
target_lang="zh",
embedding_model="local:embed-small",
)
sql, params = db.calls[0]
assert "embedding_model = :embedding_model" in sql
assert params["embedding_model"] == "local:embed-small"
def test_search_url_accepts_base_url_and_legacy_search_url() -> None:
assert (
_search_url_with_params("https://search.example", query="truth table", json_format=True)
== "https://search.example/search?q=truth+table&format=json"
)
assert (
_search_url_with_params("https://search.example/search?q=", query="truth table", json_format=True)
== "https://search.example/search?q=truth+table&format=json"
)
assert (
_search_url_with_params("https://search.example/searxng", query="truth table", json_format=False)
== "https://search.example/searxng/search?q=truth+table"
)
def test_fetch_search_evidence_reads_result_pages(monkeypatch) -> None:
from videoroll.apps.subtitle_service import rag as rag_module
class _Response:
def __init__(self, text: str, *, url: str, content_type: str = "text/html") -> None:
self.text = text
self.url = url
self.headers = {"content-type": content_type}
def raise_for_status(self) -> None:
return None
def json(self) -> object:
raise ValueError("not json")
class _Client:
def __init__(self, *args: object, **kwargs: object) -> None:
self.calls: list[str] = []
def __enter__(self) -> "_Client":
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> bool:
return False
def get(self, url: str) -> _Response:
self.calls.append(url)
if "search" in url:
return _Response(
"""
<html><body>
<article class="result">
<h3><a href="https://example.com/wiki/awp">AWP - Wiki</a></h3>
<p class="content">Counter-Strike sniper rifle.</p>
</article>
</body></html>
""",
url=url,
)
return _Response(
"""
<html>
<head><title>AWP</title></head>
<body><main>In Counter-Strike, the AWP is a powerful sniper rifle.</main></body>
</html>
""",
url=url,
)
monkeypatch.setattr(rag_module.httpx, "Client", _Client)
monkeypatch.setattr(rag_module, "_is_fetchable_url", lambda _url: True)
evidence = fetch_search_evidence("AWP", domain="CS2", search_url="https://search.example/search?q=")
assert evidence[0]["title"] == "AWP - Wiki"
assert evidence[0]["url"] == "https://example.com/wiki/awp"
assert "powerful sniper rifle" in evidence[0]["content"]
def test_fetch_search_evidence_skips_pages_when_llm_says_summary_is_enough(monkeypatch) -> None:
from videoroll.apps.subtitle_service import rag as rag_module
calls: list[str] = []
class _Response:
def __init__(self, text: str, *, url: str, content_type: str = "text/html") -> None:
self.text = text
self.url = url
self.headers = {"content-type": content_type}
def raise_for_status(self) -> None:
return None
def json(self) -> object:
raise ValueError("not json")
class _Client:
def __init__(self, *args: object, **kwargs: object) -> None:
return None
def __enter__(self) -> "_Client":
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> bool:
return False
def get(self, url: str) -> _Response:
calls.append(url)
if "search" in url:
return _Response(
"""
<html><body>
<article class="result">
<h3><a href="https://example.com/wiki/hopper">Minecraft Hopper</a></h3>
<p>Hopper is a block that captures item entities.</p>
</article>
</body></html>
""",
url=url,
)
return _Response("<html><body>should not fetch</body></html>", url=url)
monkeypatch.setattr(rag_module.httpx, "Client", _Client)
monkeypatch.setattr(
rag_module,
"request_openai_json_object",
lambda **_kwargs: {
"summary_sufficient": True,
"fetch_urls": [],
"reason": "snippet is enough",
"confidence": 0.92,
},
)
evidence = fetch_search_evidence(
"Hopper",
domain="Minecraft",
search_url="https://search.example/search",
context="This build uses Hopper Minecart.",
target_lang="zh",
config=OpenAIChatConfig(api_key="x", base_url="https://example.invalid/v1", model="demo"),
)
assert len(calls) == 1
assert "format=json" in calls[0]
assert evidence[0]["fetch_skipped"] == "not_selected_by_agent"
assert "captures item entities" in evidence[0]["snippet"]
def test_fetch_search_evidence_filters_searxng_internal_about(monkeypatch) -> None:
from videoroll.apps.subtitle_service import rag as rag_module
calls: list[str] = []
class _Response:
def __init__(self, text: str, *, url: str, content_type: str = "text/html") -> None:
self.text = text
self.url = url
self.headers = {"content-type": content_type}
def raise_for_status(self) -> None:
return None
def json(self) -> object:
raise ValueError("not json")
class _Client:
def __init__(self, *args: object, **kwargs: object) -> None:
return None
def __enter__(self) -> "_Client":
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> bool:
return False
def get(self, url: str) -> _Response:
calls.append(url)
return _Response(
"""
<html><body>
<article class="result">
<h3><a href="https://search.example/info/en/about">About</a></h3>
<p>About SearXNG preferences search syntax.</p>
</article>
</body></html>
""",
url=url,
)
monkeypatch.setattr(rag_module.httpx, "Client", _Client)
evidence = fetch_search_evidence("truth table", domain="logic", search_url="https://search.example/search")
assert evidence == []
assert len(calls) == 2
def test_fetch_search_evidence_does_not_fallback_when_llm_selects_no_urls(monkeypatch) -> None:
from videoroll.apps.subtitle_service import rag as rag_module
calls: list[str] = []
class _Response:
def __init__(self, text: str, *, url: str, content_type: str = "text/html") -> None:
self.text = text
self.url = url
self.headers = {"content-type": content_type}
def raise_for_status(self) -> None:
return None
def json(self) -> object:
raise ValueError("not json")
class _Client:
def __init__(self, *args: object, **kwargs: object) -> None:
return None
def __enter__(self) -> "_Client":
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> bool:
return False
def get(self, url: str) -> _Response:
calls.append(url)
if "search" in url:
return _Response(
"""
<html><body>
<article class="result">
<h3><a href="https://example.com/wiki/truth-table">Truth table</a></h3>
<p>Read more...</p>
</article>
</body></html>
""",
url=url,
)
return _Response("<html><body>should not fetch</body></html>", url=url)
monkeypatch.setattr(rag_module.httpx, "Client", _Client)
monkeypatch.setattr(
rag_module,
"request_openai_json_object",
lambda **_kwargs: {
"summary_sufficient": False,
"fetch_urls": [],
"reason": "no useful URL selected",
"confidence": 0.0,
},
)
evidence = fetch_search_evidence(
"truth table",
domain="logic",
search_url="https://search.example/search",
context="come up with a truth table",
target_lang="zh",
config=OpenAIChatConfig(api_key="x", base_url="https://example.invalid/v1", model="demo"),
)
assert len(calls) == 1
assert evidence[0]["url"] == "https://example.com/wiki/truth-table"
assert evidence[0]["fetch_skipped"] == "not_selected_by_agent"
def test_fetch_search_evidence_merges_multiple_queries(monkeypatch) -> None:
from videoroll.apps.subtitle_service import rag as rag_module
class _Response:
def __init__(self, text: str = "", *, url: str, content_type: str = "application/json", data: object | None = None) -> None:
self.text = text
self.url = url
self.headers = {"content-type": content_type}
self._data = data
def raise_for_status(self) -> None:
return None
def json(self) -> object:
if self._data is None:
raise ValueError("not json")
return self._data
class _Client:
def __init__(self, *args: object, **kwargs: object) -> None:
return None
def __enter__(self) -> "_Client":
return self
def __exit__(self, exc_type: object, exc: object, tb: object) -> bool:
return False
def get(self, url: str) -> _Response:
if "search" in url and "awp+cs2" in url:
return _Response(
url=url,
data={
"results": [
{"title": "AWP Wiki", "url": "https://example.com/wiki/awp", "content": "Counter-Strike sniper rifle."},
]
},
)
if "search" in url:
return _Response(
url=url,
data={
"results": [
{"title": "AWP duplicate", "url": "https://example.com/wiki/awp", "content": "Duplicate result."},
{"title": "Valve AWP", "url": "https://developer.valvesoftware.com/wiki/AWP", "content": "AWP weapon reference."},
]
},
)
return _Response(
"""
<html>
<head><title>AWP</title></head>
<body><main>AWP is a sniper rifle.</main></body>
</html>
""",
url=url,
content_type="text/html",
)
monkeypatch.setattr(rag_module.httpx, "Client", _Client)
monkeypatch.setattr(rag_module, "_is_fetchable_url", lambda _url: True)
evidence = fetch_search_evidence(
"AWP",
domain="CS2",
search_url="https://search.example/search",
queries=["awp cs2", "awp sniper rifle"],
)
assert [item["url"] for item in evidence] == [
"https://example.com/wiki/awp",
"https://developer.valvesoftware.com/wiki/AWP",
]
def test_rebuild_knowledge_embeddings_updates_current_model(monkeypatch) -> None:
from videoroll.apps.subtitle_service import rag as rag_module
monkeypatch.setattr(rag_module, "embed_text", lambda _text, *, settings: [0.1, 0.2])
rag_cfg = rag_settings_from_translate_settings(
{
"rag_embedding_provider": "local",
"rag_embedding_model": "embed-small",
"rag_embedding_dimensions": 2,
}
)
emb_cfg = embedding_settings_from_translate_settings(
{
"rag_embedding_provider": "local",
"rag_embedding_model": "embed-small",
"rag_embedding_dimensions": 2,
}
)
db = _FakeKnowledgeDb(
[
_Row(
id="00000000-0000-0000-0000-000000000001",
item_type="term",
term="AWP",
translation="AWP 狙击枪",
target_lang="zh",
domain="CS2",
aliases=["Magnum Sniper Rifle"],
title="",
content="",
description="Counter-Strike weapon",
)
]
)
result = rebuild_knowledge_embeddings(db, rag_settings=rag_cfg, embedding_settings=emb_cfg) # type: ignore[arg-type]
update_params = db.calls[-1][1]
assert embedding_model_key(rag_cfg) == "local:embed-small"
assert result["updated"] == 1
assert result["failed"] == 0
assert update_params["embedding_model"] == "local:embed-small"
assert update_params["embedding"] == "[0.1,0.2]"