diff --git a/.gitignore b/.gitignore index 763f56d..82d6c06 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ videoroll-prod-bundle-*.tar videoroll-prod-bundle-*.tar.sha256 videoroll-deploy-*/ videoroll-prod-deploy-*/ +docker_exports/ diff --git a/Dockerfile b/Dockerfile index df83e56..47253a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 +ENV PIP_DEFAULT_TIMEOUT=300 +ENV PIP_RETRIES=20 ENV VIRTUAL_ENV=/opt/venv ENV PATH="/opt/venv/bin:${PATH}" @@ -31,11 +33,13 @@ COPY pyproject.toml README.md ./ ARG INSTALL_ASR=0 ARG YTDLP_VERSION=latest +ARG TORCH_CPU_INDEX_URL=https://download.pytorch.org/whl/cpu # Install dependencies in a cache-friendly layer so editing source code doesn't # force re-downloading everything on every docker build. RUN INSTALL_ASR="$INSTALL_ASR" python -c "import os, tomllib; from pathlib import Path; data=tomllib.loads(Path('pyproject.toml').read_text('utf-8')); deps=list(data.get('project', {}).get('dependencies', []) or []); opt=data.get('project', {}).get('optional-dependencies', {}) or {}; deps += list(opt.get('asr', []) or []) if os.getenv('INSTALL_ASR','0')=='1' else []; Path('/tmp/requirements.txt').write_text('\\n'.join(deps) + '\\n', encoding='utf-8')" \ && pip install --no-cache-dir -U pip \ + && pip install --no-cache-dir --index-url "$TORCH_CPU_INDEX_URL" "torch>=2.3,<3" \ && pip install --no-cache-dir -r /tmp/requirements.txt COPY src/videoroll ./src/videoroll diff --git a/README.md b/README.md index e653a9d..1555741 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ - 若未配置 `YOUTUBE_COOKIE_FILE`，已保存到数据库的 YouTube cookies 会在下载/元信息提取时写入临时文件供 `yt-dlp` 使用。 - YouTube 触发风控时，接口会返回更明确的 cookies / proxy 提示。 - Bilibili 投稿支持在 Web UI 中保存 Cookie、测试登录、维护默认投稿模板。 +- Web UI 支持浅色 / 暗色模式切换，Dashboard 增加 CPU、内存和 Intel GPU 资源监控。 +- 字幕翻译支持真正的 RAG 术语知识库：可使用 PostgreSQL + pgvector 检索术语，支持本地 embedding 模型、手动下载模型、重建向量，以及 Agent 自动发现/搜索/验证/入库术语。 +- RAG Agent 支持 SearXNG 搜索、网页正文读取、独立 verifier、保守入库策略，并可在 Dashboard 查看每个 Agent 的工具调用 trace。 ## 架构概览 @@ -165,7 +168,7 @@ docker compose -f docker-compose.yml --env-file .env up --build -d - `data/minio` MinIO 数据目录 - `data/models` - ASR 模型目录（可存放 faster-whisper 或 OpenVINO Whisper 导出模型） + ASR / embedding 模型目录（可存放 faster-whisper、OpenVINO Whisper 导出模型、本地 embedding 模型） - `data/secrets` 本地密钥和机密目录 - `data/secrets/fernet.key` @@ -263,6 +266,7 @@ YouTube 风控注意事项： - `Settings -> ASR` - `Settings -> Translate` - `Settings -> Auto` +- `Dashboard` 说明： @@ -274,6 +278,7 @@ YouTube 风控注意事项： - 编码参数 - 翻译目标语言与 provider - 是否自动投稿 Bilibili +- `Dashboard` 会显示任务状态、资源监控、RAG Agent 当前运行状态和历史 trace。 如果只是想做轻量演示，可把： @@ -298,6 +303,71 @@ SUBTITLE_OPENVINO_MODEL=/models/whisper/whisper-large-v3-ov - 也支持手工准备模型：可以在宿主机先用 `optimum-cli export openvino --model openai/whisper-large-v3 ` 生成，再挂载/上传到 `data/models`。 - 下载或上传后，在 `Settings -> ASR` 中把默认引擎切到 `openvino`，并配置 `device / num_beams / max_new_tokens`。 +### 翻译 RAG / Agent + +相关页面： + +- `Settings -> Translate` +- `Dashboard -> RAG Agent` + +RAG 用于解决视频字幕翻译中的上下文和术语问题，例如游戏、动漫、技术讲解中的专有名词、缩写、黑话和社区固定译法。 + +当前流程： + +```text +字幕片段 + -> 术语发现 + -> 查询 PostgreSQL/pgvector 知识库 + -> 未命中时生成多条搜索 query + -> 调用 SearXNG 搜索 + -> 必要时读取网页正文 + -> LLM 总结候选术语 + -> verifier 检查来源和上下文 + -> 写入知识库或标记为 context_only / skipped + -> 翻译时注入 rag_context +``` + +关键行为： + +- 单字母变量（如逻辑课里的 `P/Q/R`）默认视为局部变量，不搜索、不写入长期知识库。 +- 常见基础术语（如 `truth table`）可以作为本次翻译提示，但默认不自动污染长期知识库。 +- 自动正式入库要求 verifier 通过、上下文一致、有有效外部来源，并达到较高置信度；否则进入待审核或跳过。 +- Dashboard 中可以点击 RAG Agent 查看每步工具调用、耗时、模型、错误类型和 JSON 输入输出。 + +PostgreSQL 需要启用 pgvector： + +```sql +CREATE EXTENSION IF NOT EXISTS vector; +``` + +如果应用账号不是数据库超级用户，请用数据库管理员账号先在目标库执行上面的 SQL。 + +SearXNG 搜索地址建议填写 base URL，例如： + +```text +https://search.example.com +``` + +系统会自动请求 `/search?q=...&format=json`，不需要在配置里手写 `?q=`。 + +Embedding 支持两种方式： + +- OpenAI 兼容 embedding：填写 provider 支持的 embedding 模型名。 +- 本地 embedding：在 `Settings -> Translate` 中选择本地 provider，可手动下载模型到 `data/models`，并选择 CPU / OpenVINO GPU 等设备。 + +更换 embedding 模型后，已有知识库向量不会静默重建；需要在 `Settings -> Translate` 中点击重建按钮，避免模型切换时旧向量和新向量混用。 + +### Dashboard 资源监控 + +Dashboard 会定时显示： + +- CPU 使用率和 load average +- 内存使用率 +- 启用自动模式并配置 Intel GPU 时的 Intel GPU 状态 +- 正在运行和最近完成的 RAG Agent + +Intel GPU 使用率依赖宿主机和容器能读取对应设备/驱动信息；如果 busy 不可读，页面会显示不可读原因，但不影响其他流程。 + ### Bilibili 配置 相关页面： diff --git a/deploy_prod/docker-compose.intel.yml b/deploy_prod/docker-compose.intel.yml deleted file mode 100644 index 0400b34..0000000 --- a/deploy_prod/docker-compose.intel.yml +++ /dev/null @@ -1,6 +0,0 @@ -services: - app: - devices: - - "${INTEL_GPU_RENDER_DEVICE:-/dev/dri/renderD128}:${INTEL_GPU_RENDER_DEVICE:-/dev/dri/renderD128}" - environment: - INTEL_GPU_RENDER_DEVICE: ${INTEL_GPU_RENDER_DEVICE:-/dev/dri/renderD128} diff --git a/deploy_prod/docker-compose.yml b/deploy_prod/docker-compose.yml new file mode 100644 index 0000000..2b41c66 --- /dev/null +++ b/deploy_prod/docker-compose.yml @@ -0,0 +1,89 @@ +# This file is intended to be placed at the repository root before deployment. +services: + redis: + image: redis:7 + restart: unless-stopped + + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY_ID:-videoroll} + MINIO_ROOT_PASSWORD: ${S3_SECRET_ACCESS_KEY:-videorollsecret} + ports: + - "${PUBLISH_ADDR:-127.0.0.1}:${MINIO_CONSOLE_PORT:-9001}:9001" + volumes: + - ./data/minio:/data + restart: unless-stopped + + minio-init: + image: minio/mc:latest + depends_on: + - minio + entrypoint: /bin/sh + command: > + -c " + until mc alias set local http://minio:9000 ${S3_ACCESS_KEY_ID:-videoroll} ${S3_SECRET_ACCESS_KEY:-videorollsecret}; do + echo 'waiting for minio...'; + sleep 1; + done && + mc mb -p local/${S3_BUCKET:-videoroll} || true + " + restart: "no" + + app: + build: + context: . + dockerfile: Dockerfile + args: + INSTALL_ASR: ${INSTALL_ASR:-1} + YTDLP_VERSION: ${YTDLP_VERSION:-latest} + image: videoroll:prod + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL:-redis://redis:6379/0} + S3_ENDPOINT_URL: ${S3_ENDPOINT_URL:-http://minio:9000} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-videoroll} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-videorollsecret} + S3_BUCKET: ${S3_BUCKET:-videoroll} + S3_REGION_NAME: ${S3_REGION_NAME:-us-east-1} + S3_USE_SSL: ${S3_USE_SSL:-false} + UVICORN_ROOT_PATH: ${UVICORN_ROOT_PATH:-/api} + SUBTITLE_SERVICE_URL: ${SUBTITLE_SERVICE_URL:-http://localhost:8000/subtitle-service} + YOUTUBE_INGEST_URL: ${YOUTUBE_INGEST_URL:-http://localhost:8000/youtube-ingest} + BILIBILI_PUBLISHER_URL: ${BILIBILI_PUBLISHER_URL:-http://localhost:8000/bilibili-publisher} + BILIBILI_PUBLISH_MODE: ${BILIBILI_PUBLISH_MODE:-web} + volumes: + - ./data/models:/models + - ./data/secrets:/secrets + - ./data/work:/work + # Expose the whole Intel DRM device tree so both VAAPI and OpenVINO GPU + # runtimes can discover the render device and initialize compute contexts. + devices: + - "/dev/dri:/dev/dri" + extra_hosts: + - host.docker.internal:host-gateway + depends_on: + redis: + condition: service_started + minio: + condition: service_started + minio-init: + condition: service_completed_successfully + restart: unless-stopped + + web: + build: + context: ./src/web + args: + VITE_ORCHESTRATOR_URL: ${VITE_ORCHESTRATOR_URL:-} + VITE_SUBTITLE_SERVICE_URL: ${VITE_SUBTITLE_SERVICE_URL:-} + VITE_YOUTUBE_INGEST_URL: ${VITE_YOUTUBE_INGEST_URL:-} + VITE_BILIBILI_PUBLISHER_URL: ${VITE_BILIBILI_PUBLISHER_URL:-} + image: videoroll-web:prod + ports: + - "${PUBLISH_ADDR:-127.0.0.1}:${WEB_PORT:-3000}:80" + depends_on: + app: + condition: service_started + restart: unless-stopped diff --git a/pyproject.toml b/pyproject.toml index ed9cd19..09ee66b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,8 @@ dependencies = [ "defusedxml>=0.7.1", "yt-dlp[default]>=2026.2.4", "deno>=2.6.9", + "sentence-transformers>=3.0.0", + "optimum-intel[openvino]>=1.20.0", ] [project.optional-dependencies] diff --git a/src/videoroll/ai/client.py b/src/videoroll/ai/client.py index 430c8ec..b3aa6f8 100644 --- a/src/videoroll/ai/client.py +++ b/src/videoroll/ai/client.py @@ -9,6 +9,7 @@ from typing import Any, Mapping import httpx from videoroll.utils.openai_compat import build_openai_chat_completions_url +from videoroll.utils.openai_compat import build_openai_embeddings_url _RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504} @@ -203,3 +204,93 @@ def request_openai_json_object( format_retries=format_retries, network_retries=network_retries, ) + + +def request_openai_embedding( + *, + config: OpenAIChatConfig, + text: str, + client: httpx.Client | None = None, + network_retries: int = 3, +) -> list[float]: + if not config.api_key: + raise RuntimeError("OpenAI API key is not set") + + source = str(text or "").strip() + if not source: + raise ValueError("embedding text is empty") + + url = build_openai_embeddings_url(config.base_url) + headers = {"Authorization": f"Bearer {config.api_key}"} + req = {"model": config.model, "input": source} + attempts_network = max(1, int(network_retries)) + last_err: Exception | None = None + + def _with_client(c: httpx.Client) -> list[float]: + nonlocal last_err + for net_attempt in range(attempts_network): + try: + resp = c.post(url, headers=headers, json=req) + try: + resp.raise_for_status() + except httpx.HTTPStatusError as e: + status = resp.status_code + if status in _RETRYABLE_STATUS_CODES and net_attempt < attempts_network - 1: + retry_after = (resp.headers.get("retry-after") or "").strip() + if retry_after: + try: + time.sleep(min(30.0, float(retry_after))) + except Exception: + _sleep_backoff(net_attempt) + else: + _sleep_backoff(net_attempt) + continue + + ct = (resp.headers.get("content-type") or "").split(";")[0].strip() + snippet = _resp_snippet(resp) + raise RuntimeError( + f"OpenAI embedding request failed (status={resp.status_code}, content-type={ct}, url={url}). {snippet}" + ) from e + + try: + resp_json = resp.json() + except Exception as e: + ct = (resp.headers.get("content-type") or "").split(";")[0].strip() + hint = " (check openai_base_url; most providers require it to end with /v1)" if "text/html" in ct else "" + raise RuntimeError( + f"OpenAI embedding endpoint did not return JSON (status={resp.status_code}, content-type={ct}, url={url}){hint}." + ) from e + + try: + raw = resp_json["data"][0]["embedding"] + except Exception as e: + raise RuntimeError(f"unexpected OpenAI embedding response shape: {resp_json}") from e + + if not isinstance(raw, list) or not raw: + raise RuntimeError("OpenAI embedding output is empty") + return [float(x) for x in raw] + except httpx.TimeoutException as e: + last_err = e + if net_attempt < attempts_network - 1: + _sleep_backoff(net_attempt) + continue + break + except httpx.TransportError as e: + last_err = e + if net_attempt < attempts_network - 1: + _sleep_backoff(net_attempt) + continue + break + except Exception as e: + last_err = e + break + + if last_err is not None: + raise last_err + raise RuntimeError("OpenAI embedding request failed") + + if client is not None: + return _with_client(client) + + with create_openai_http_client(config.timeout_seconds) as owned_client: + return _with_client(owned_client) diff --git a/src/videoroll/apps/orchestrator_api/main.py b/src/videoroll/apps/orchestrator_api/main.py index 6f8ea09..4a6133d 100644 --- a/src/videoroll/apps/orchestrator_api/main.py +++ b/src/videoroll/apps/orchestrator_api/main.py @@ -52,7 +52,9 @@ from videoroll.storage.s3 import S3Store from videoroll.utils.auto_youtube import encode_auto_youtube_created_by from videoroll.utils.hashing import sha256_file from videoroll.utils.httpx_proxy import HTTPX_PROXY_KWARG_UNSUPPORTED, format_httpx_proxy_error +from videoroll.utils.intel_gpu import sample_intel_gpu_usage from videoroll.utils.internal_api_token import internal_api_token +from videoroll.utils.resources import process_cpu_summary, read_cgroup_memory_stats, read_memory_stats from videoroll.utils.workdir_maintenance import ( WorkdirJobState, cleanup_reclaimable_dirs, @@ -85,6 +87,10 @@ from videoroll.apps.orchestrator_api.schemas import ( StorageRetentionSettingsUpdate, SubtitleActionRequest, SubtitleJobSummary, + SystemCPURead, + SystemIntelGPURead, + SystemMemoryRead, + SystemResourcesRead, TaskCreate, TaskPublishReviewRead, TaskRead, @@ -1633,6 +1639,74 @@ def health() -> dict[str, str]: return {"status": "ok"} +def _memory_read(data: dict[str, Any] | None) -> SystemMemoryRead: + data = data or {} + return SystemMemoryRead( + total_bytes=int(data.get("total_bytes") or 0), + used_bytes=int(data.get("used_bytes") or 0), + available_bytes=int(data.get("available_bytes") or 0), + percent=float(data["percent"]) if data.get("percent") is not None else None, + ) + + +@app.get("/system/resources", response_model=SystemResourcesRead) +def system_resources(db: Session = Depends(get_db)) -> SystemResourcesRead: + cpu_data = process_cpu_summary() + mem = read_memory_stats() + cgroup_mem = read_cgroup_memory_stats() + + intel_gpu: SystemIntelGPURead | None = None + try: + profile = get_auto_profile(db) + intel_enabled = bool(profile.get("use_intel_gpu")) + except Exception: + intel_enabled = False + + if intel_enabled: + subtitle_settings = get_subtitle_settings() + render_device = str(subtitle_settings.intel_gpu_render_device or "").strip() or "/dev/dri/renderD128" + try: + gpu_info = sample_intel_gpu_usage(render_device) + except Exception as e: + gpu_info = { + "enabled": True, + "checked": True, + "available": False, + "render_device": render_device, + "detail": str(e), + } + intel_gpu = SystemIntelGPURead( + enabled=True, + checked=bool(gpu_info.get("checked", True)), + available=bool(gpu_info.get("available", False)), + render_device=str(gpu_info.get("render_device") or render_device), + model_name=gpu_info.get("model_name"), + driver=gpu_info.get("driver"), + pci_slot=gpu_info.get("pci_slot"), + pci_id=gpu_info.get("pci_id"), + usage_supported=bool(gpu_info.get("usage_supported", False)), + usage_percent=float(gpu_info["usage_percent"]) if gpu_info.get("usage_percent") is not None else None, + engines=[ + {"name": str(item.get("name") or ""), "percent": float(item["percent"]) if item.get("percent") is not None else None} + for item in list(gpu_info.get("engines") or []) + if isinstance(item, dict) + ], + detail=str(gpu_info.get("detail") or ""), + ) + + return SystemResourcesRead( + sampled_at=datetime.now(tz=timezone.utc).isoformat(), + cpu=SystemCPURead( + percent=float(cpu_data["percent"]) if cpu_data.get("percent") is not None else None, + cores=int(cpu_data.get("cores") or 0), + load_average=cpu_data.get("load_average"), + ), + memory=_memory_read(mem), + cgroup_memory=_memory_read(cgroup_mem) if cgroup_mem else None, + intel_gpu=intel_gpu, + ) + + def _secure_cookie(request: Request) -> bool: proto = str(request.headers.get("x-forwarded-proto") or request.url.scheme or "").lower() return proto == "https" diff --git a/src/videoroll/apps/orchestrator_api/schemas.py b/src/videoroll/apps/orchestrator_api/schemas.py index 54e1209..7a2e40f 100644 --- a/src/videoroll/apps/orchestrator_api/schemas.py +++ b/src/videoroll/apps/orchestrator_api/schemas.py @@ -168,6 +168,47 @@ class RemotePublishResponse(BaseModel): response: Optional[dict[str, Any]] = None +class SystemCPURead(BaseModel): + percent: Optional[float] = None + cores: int = 0 + load_average: Optional[list[float]] = None + + +class SystemMemoryRead(BaseModel): + total_bytes: int = 0 + used_bytes: int = 0 + available_bytes: int = 0 + percent: Optional[float] = None + + +class SystemIntelGPUEngineRead(BaseModel): + name: str + percent: Optional[float] = None + + +class SystemIntelGPURead(BaseModel): + enabled: bool = False + checked: bool = False + available: bool = False + render_device: str = "" + model_name: Optional[str] = None + driver: Optional[str] = None + pci_slot: Optional[str] = None + pci_id: Optional[str] = None + usage_supported: bool = False + usage_percent: Optional[float] = None + engines: list[SystemIntelGPUEngineRead] = Field(default_factory=list) + detail: str = "" + + +class SystemResourcesRead(BaseModel): + sampled_at: str + cpu: SystemCPURead + memory: SystemMemoryRead + cgroup_memory: Optional[SystemMemoryRead] = None + intel_gpu: Optional[SystemIntelGPURead] = None + + class SubtitleJobSummary(BaseModel): id: uuid.UUID task_id: uuid.UUID diff --git a/src/videoroll/apps/subtitle_service/embeddings.py b/src/videoroll/apps/subtitle_service/embeddings.py new file mode 100644 index 0000000..6672e46 --- /dev/null +++ b/src/videoroll/apps/subtitle_service/embeddings.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import shutil +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from videoroll.ai.client import OpenAIChatConfig, request_openai_embedding +from videoroll.apps.subtitle_service.model_downloads import download_model_snapshot + + +_SAFE_MODEL_NAME_REPLACEMENTS = {"/": "--", "\\": "--"} +_LOCAL_MODEL_CACHE: dict[tuple[str, str, str], Any] = {} +_LOCAL_MODEL_CACHE_LOCK = threading.Lock() + + +@dataclass(frozen=True) +class EmbeddingSettings: + provider: str + model: str + dimensions: int + model_dir: str + device: str + openai_config: OpenAIChatConfig + + +def normalize_embedding_provider(provider: str | None) -> str: + out = str(provider or "").strip().lower() or "openai" + if out not in {"openai", "local"}: + raise ValueError(f"unsupported embedding provider: {out}") + return out + + +def safe_embedding_model_name(model: str) -> str: + out = str(model or "").strip() + if not out: + raise ValueError("model is required") + for old, new in _SAFE_MODEL_NAME_REPLACEMENTS.items(): + out = out.replace(old, new) + out = "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in out) + return out[:96] or "embedding-model" + + +def embedding_model_path(model_dir: str | Path, model: str) -> Path: + p = Path(model_dir) / safe_embedding_model_name(model) + return p + + +def list_local_embedding_models(model_dir: str | Path) -> list[dict[str, Any]]: + root = Path(model_dir) + if not root.exists(): + return [] + out: list[dict[str, Any]] = [] + for p in sorted(root.iterdir(), key=lambda x: x.name.lower()): + if not p.is_dir(): + continue + size = 0 + for item in p.rglob("*"): + if item.is_file(): + try: + size += item.stat().st_size + except OSError: + pass + out.append({"name": p.name, "path": str(p), "size_bytes": size}) + return out + + +def download_local_embedding_model( + *, + model: str, + model_dir: str | Path, + name: str | None = None, + revision: str | None = None, + force: bool = False, + proxy: str | None = None, +) -> Path: + model_name = str(model or "").strip() + if not model_name: + raise ValueError("model is required") + dest_name = safe_embedding_model_name(name or model_name) + return download_model_snapshot( + engine="embedding", + model=model_name, + model_dir=Path(model_dir), + name=dest_name, + revision=revision, + force=force, + proxy=proxy, + ) + + +def delete_local_embedding_model(*, model_dir: str | Path, name: str) -> None: + path = Path(model_dir) / safe_embedding_model_name(name) + if path.exists(): + shutil.rmtree(path) + + +def _load_sentence_transformer(path: Path, *, device: str) -> Any: + try: + from sentence_transformers import SentenceTransformer # type: ignore + except Exception as e: + raise RuntimeError("sentence-transformers is not installed. Rebuild the app image with embedding dependencies.") from e + if not path.exists(): + raise RuntimeError(f"local embedding model is not downloaded: {path}") + device_arg = str(device or "cpu").strip() or "cpu" + cache_key = ("sentence-transformers", str(path), device_arg) + with _LOCAL_MODEL_CACHE_LOCK: + model = _LOCAL_MODEL_CACHE.get(cache_key) + if model is None: + model = SentenceTransformer(str(path), device=device_arg) + _LOCAL_MODEL_CACHE[cache_key] = model + return model + + +def _parse_local_device(device: str) -> tuple[str, str]: + value = str(device or "cpu").strip() or "cpu" + lowered = value.lower() + if lowered.startswith("openvino"): + _, sep, raw_device = value.partition(":") + ov_device = raw_device.strip() if sep else "CPU" + return "openvino", ov_device or "CPU" + return "sentence-transformers", value + + +def _openvino_export_dir(path: Path) -> Path: + if (path / "openvino_model.xml").exists(): + return path + return path / "openvino" + + +def _load_openvino_feature_extractor(path: Path, *, device: str) -> tuple[Any, Any]: + try: + from optimum.intel.openvino import OVModelForFeatureExtraction # type: ignore + from transformers import AutoTokenizer # type: ignore + except Exception as e: + raise RuntimeError("OpenVINO embedding requires optimum-intel[openvino]. Rebuild the app image with embedding dependencies.") from e + if not path.exists(): + raise RuntimeError(f"local embedding model is not downloaded: {path}") + + device_arg = str(device or "CPU").strip() or "CPU" + ov_dir = _openvino_export_dir(path) + cache_key = ("openvino-feature-extraction", str(ov_dir), device_arg) + with _LOCAL_MODEL_CACHE_LOCK: + cached = _LOCAL_MODEL_CACHE.get(cache_key) + if cached is not None: + return cached + + if (ov_dir / "openvino_model.xml").exists(): + model = OVModelForFeatureExtraction.from_pretrained(str(ov_dir), device=device_arg) + tokenizer = AutoTokenizer.from_pretrained(str(path)) + else: + ov_dir.mkdir(parents=True, exist_ok=True) + model = OVModelForFeatureExtraction.from_pretrained(str(path), export=True, device=device_arg) + tokenizer = AutoTokenizer.from_pretrained(str(path)) + try: + model.save_pretrained(str(ov_dir)) + tokenizer.save_pretrained(str(ov_dir)) + except Exception: + pass + + cached = (model, tokenizer) + _LOCAL_MODEL_CACHE[cache_key] = cached + return cached + + +def _mean_pool_embedding(model_output: Any, attention_mask: Any) -> list[float]: + try: + import numpy as np # type: ignore + import torch # type: ignore + except Exception as e: + raise RuntimeError("OpenVINO embedding pooling requires numpy and torch.") from e + + if isinstance(model_output, dict): + token_embeddings = model_output.get("last_hidden_state") + else: + token_embeddings = getattr(model_output, "last_hidden_state", None) + if token_embeddings is None and isinstance(model_output, (list, tuple)) and model_output: + token_embeddings = model_output[0] + if token_embeddings is None: + raise RuntimeError("OpenVINO embedding output missing last_hidden_state") + + if hasattr(token_embeddings, "detach"): + token_embeddings_np = token_embeddings.detach().cpu().numpy() + else: + token_embeddings_np = np.asarray(token_embeddings) + + if hasattr(attention_mask, "detach"): + mask_np = attention_mask.detach().cpu().numpy() + else: + mask_np = np.asarray(attention_mask) + + mask_np = np.expand_dims(mask_np, axis=-1).astype(np.float32) + summed = (token_embeddings_np * mask_np).sum(axis=1) + counts = np.clip(mask_np.sum(axis=1), 1e-9, None) + vector = summed / counts + tensor = torch.tensor(vector, dtype=torch.float32) + tensor = torch.nn.functional.normalize(tensor, p=2, dim=1) + return _normalize_vector(tensor.detach().cpu().numpy()) + + +def _embed_text_openvino(path: Path, text: str, *, device: str) -> list[float]: + model, tokenizer = _load_openvino_feature_extractor(path, device=device) + inputs = tokenizer(text, padding=True, truncation=True, return_tensors="pt") + outputs = model(**inputs) + return _mean_pool_embedding(outputs, inputs["attention_mask"]) + + +def _normalize_vector(values: Any) -> list[float]: + try: + import numpy as np # type: ignore + + if isinstance(values, np.ndarray): + raw = values.tolist() + else: + raw = values + except Exception: + raw = values + + if isinstance(raw, list) and raw and isinstance(raw[0], list): + raw = raw[0] + if not isinstance(raw, list) or not raw: + raise RuntimeError("embedding output is empty") + return [float(x) for x in raw] + + +def embed_text(text: str, *, settings: EmbeddingSettings) -> list[float]: + provider = normalize_embedding_provider(settings.provider) + source = str(text or "").strip() + if not source: + raise ValueError("embedding text is empty") + + if provider == "openai": + return request_openai_embedding(config=settings.openai_config, text=source) + + model_path = embedding_model_path(settings.model_dir, settings.model) + backend, device = _parse_local_device(settings.device) + if backend == "openvino": + return _embed_text_openvino(model_path, source, device=device) + + model = _load_sentence_transformer(model_path, device=device) + try: + encoded = model.encode(source, normalize_embeddings=True) + except TypeError: + encoded = model.encode(source) + return _normalize_vector(encoded) + + +def assert_embedding_dimensions(vector: list[float], expected: int) -> None: + if expected <= 0: + return + if len(vector) != expected: + raise RuntimeError(f"embedding dimension mismatch: expected {expected}, got {len(vector)}") + + +def embedding_settings_from_translate_settings(settings: dict[str, Any]) -> EmbeddingSettings: + provider = normalize_embedding_provider(str(settings.get("rag_embedding_provider") or "openai")) + return EmbeddingSettings( + provider=provider, + model=str(settings.get("rag_embedding_model") or "text-embedding-3-small").strip() or "text-embedding-3-small", + dimensions=max(1, min(4096, int(settings.get("rag_embedding_dimensions") or 1536))), + model_dir=str(settings.get("rag_embedding_model_dir") or "/models/embeddings").strip() or "/models/embeddings", + device=str(settings.get("rag_embedding_device") or "cpu").strip() or "cpu", + openai_config=OpenAIChatConfig( + api_key=str(settings.get("openai_api_key") or "").strip() or None, + base_url=str(settings.get("openai_base_url") or "").strip(), + model=str(settings.get("rag_embedding_model") or "text-embedding-3-small").strip() or "text-embedding-3-small", + temperature=0.0, + timeout_seconds=float(settings.get("openai_timeout_seconds") or 60.0), + ), + ) diff --git a/src/videoroll/apps/subtitle_service/main.py b/src/videoroll/apps/subtitle_service/main.py index c57b2d9..851e757 100644 --- a/src/videoroll/apps/subtitle_service/main.py +++ b/src/videoroll/apps/subtitle_service/main.py @@ -16,6 +16,7 @@ import httpx from fastapi import Depends, FastAPI, File, HTTPException, UploadFile from fastapi.middleware.cors import CORSMiddleware from sqlalchemy.orm import Session +from sqlalchemy.exc import ProgrammingError from videoroll.ai.client import openai_chat_config_from_settings from videoroll.ai.service import translate_text_openai @@ -42,6 +43,17 @@ from videoroll.apps.subtitle_service.schemas import ( WhisperSettingsRead, ModelDownloadProxyTestRequest, ModelDownloadProxyTestResponse, + KnowledgeItemRead, + KnowledgeEmbeddingRebuildRequest, + KnowledgeEmbeddingRebuildResponse, + KnowledgeItemUpsertRequest, + KnowledgeItemUpsertResponse, + AgentRunRead, + EmbeddingModelDownloadRequest, + EmbeddingModelListRequest, + EmbeddingModelInfo, + EmbeddingTestRequest, + EmbeddingTestResponse, TaskQueueItemRead, TaskQueueRead, TaskQueueSettingsRead, @@ -55,6 +67,22 @@ from videoroll.apps.subtitle_service.model_downloads import ( normalize_model_download_engine, ) from videoroll.apps.subtitle_service.render_queue_store import get_task_queue_settings, update_task_queue_settings +from videoroll.apps.subtitle_service.embeddings import ( + assert_embedding_dimensions, + delete_local_embedding_model, + download_local_embedding_model, + embedding_settings_from_translate_settings, + embed_text, + list_local_embedding_models, +) +from videoroll.apps.subtitle_service.rag import ( + build_knowledge_embedding_text, + list_agent_runs, + rebuild_knowledge_embeddings, + list_knowledge_items, + rag_settings_from_translate_settings, + upsert_knowledge_item, +) from videoroll.apps.subtitle_service.translate_settings_store import get_translate_settings, update_translate_settings from videoroll.apps.subtitle_service.worker_concurrency import sync_subtitle_worker_concurrency_for_task_queue_settings from videoroll.apps.subtitle_service.worker import TASK_QUEUE_LOCK_OWNER, celery_app @@ -78,6 +106,59 @@ def _models_dir(settings: SubtitleServiceSettings) -> Path: return Path(settings.whisper_model_dir) +def _embedding_models_dir(settings: SubtitleServiceSettings) -> Path: + return Path(settings.rag_embedding_model_dir) + + +def _embedding_models_dir_from_translate_settings(cfg: dict[str, Any], settings: SubtitleServiceSettings) -> Path: + path = str(cfg.get("rag_embedding_model_dir") or "").strip() + if path: + return Path(path) + return _embedding_models_dir(settings) + + +def _is_missing_knowledge_table_error(exc: Exception) -> bool: + orig = getattr(exc, "orig", None) + sqlstate = getattr(orig, "sqlstate", None) or getattr(orig, "pgcode", None) + if sqlstate == "42P01": # PostgreSQL undefined_table + return True + msg = str(exc).lower() + return ( + "does not exist" in msg + and any( + table in msg + for table in [ + "translation_knowledge_items", + "translation_term_evidence", + "translation_term_matches", + "translation_agent_runs", + ] + ) + ) + + +def _ensure_rag_schema(settings: SubtitleServiceSettings) -> None: + try: + auto_migrate(settings.database_url, force=True) + except Exception as e: + raise HTTPException( + status_code=503, + detail=( + "RAG knowledge tables are not ready. Ensure PostgreSQL has pgvector installed " + f"and restart the app. Migration error: {e}" + ), + ) from e + + +def _handle_knowledge_db_error(exc: Exception, settings: SubtitleServiceSettings) -> None: + if _is_missing_knowledge_table_error(exc): + _ensure_rag_schema(settings) + return + if isinstance(exc, ProgrammingError): + raise HTTPException(status_code=503, detail=f"RAG database schema is not ready: {exc}") from exc + raise HTTPException(status_code=500, detail=f"knowledge database error: {exc}") from exc + + _SAFE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") @@ -254,6 +335,19 @@ def get_translate_settings_view( openai_model=cfg["openai_model"], openai_temperature=cfg["openai_temperature"], openai_timeout_seconds=cfg["openai_timeout_seconds"], + rag_enabled=cfg["rag_enabled"], + rag_top_k=cfg["rag_top_k"], + rag_min_score=cfg["rag_min_score"], + rag_embedding_provider=cfg["rag_embedding_provider"], + rag_embedding_model=cfg["rag_embedding_model"], + rag_embedding_dimensions=cfg["rag_embedding_dimensions"], + rag_embedding_model_dir=cfg["rag_embedding_model_dir"], + rag_embedding_device=cfg["rag_embedding_device"], + rag_auto_discover_terms=cfg["rag_auto_discover_terms"], + rag_auto_learn_terms=cfg["rag_auto_learn_terms"], + rag_search_enabled=cfg["rag_search_enabled"], + rag_search_url=cfg["rag_search_url"], + rag_domain=cfg["rag_domain"], ) @@ -276,6 +370,273 @@ def put_translate_settings_view( openai_model=cfg["openai_model"], openai_temperature=cfg["openai_temperature"], openai_timeout_seconds=cfg["openai_timeout_seconds"], + rag_enabled=cfg["rag_enabled"], + rag_top_k=cfg["rag_top_k"], + rag_min_score=cfg["rag_min_score"], + rag_embedding_provider=cfg["rag_embedding_provider"], + rag_embedding_model=cfg["rag_embedding_model"], + rag_embedding_dimensions=cfg["rag_embedding_dimensions"], + rag_embedding_model_dir=cfg["rag_embedding_model_dir"], + rag_embedding_device=cfg["rag_embedding_device"], + rag_auto_discover_terms=cfg["rag_auto_discover_terms"], + rag_auto_learn_terms=cfg["rag_auto_learn_terms"], + rag_search_enabled=cfg["rag_search_enabled"], + rag_search_url=cfg["rag_search_url"], + rag_domain=cfg["rag_domain"], + ) + + +@app.get("/subtitle/knowledge/items", response_model=list[KnowledgeItemRead]) +def list_knowledge_items_view( + item_type: str | None = None, + status: str | None = None, + limit: int = 100, + settings: SubtitleServiceSettings = Depends(get_settings), + db: Session = Depends(get_db), +) -> list[KnowledgeItemRead]: + try: + rows = list_knowledge_items(db, item_type=item_type, status=status, limit=limit) + except Exception as e: + db.rollback() + if not _is_missing_knowledge_table_error(e): + _handle_knowledge_db_error(e, settings) + _ensure_rag_schema(settings) + try: + rows = list_knowledge_items(db, item_type=item_type, status=status, limit=limit) + except Exception as retry_error: + db.rollback() + raise HTTPException(status_code=503, detail=f"RAG knowledge tables are not ready: {retry_error}") from retry_error + return [KnowledgeItemRead(**row) for row in rows] + + +@app.post("/subtitle/knowledge/items", response_model=KnowledgeItemUpsertResponse) +def upsert_knowledge_item_view( + payload: KnowledgeItemUpsertRequest, + settings: SubtitleServiceSettings = Depends(get_settings), + db: Session = Depends(get_db), +) -> KnowledgeItemUpsertResponse: + cfg = get_translate_settings(db, settings) + rag_cfg = rag_settings_from_translate_settings(cfg) + + if payload.item_type == "term" and not payload.term.strip(): + raise HTTPException(status_code=400, detail="term is required for term items") + if payload.item_type == "term" and not payload.translation.strip(): + raise HTTPException(status_code=400, detail="translation is required for term items") + if payload.item_type == "document" and not (payload.title.strip() or payload.content.strip()): + raise HTTPException(status_code=400, detail="title or content is required for document items") + + embedding_text = build_knowledge_embedding_text( + item_type=payload.item_type, + term=payload.term, + translation=payload.translation, + domain=payload.domain, + aliases=payload.aliases, + title=payload.title, + content=payload.content, + description=payload.description, + ) + + embedding: list[float] | None = None + if embedding_text.strip(): + try: + embedding = embed_text(embedding_text, settings=embedding_settings_from_translate_settings(cfg)) + assert_embedding_dimensions(embedding, rag_cfg.embedding_dimensions) + except Exception as e: + if payload.item_type == "document": + raise HTTPException(status_code=502, detail=f"embedding failed: {e}") from e + embedding = None + + def _save_item() -> str: + item_id = upsert_knowledge_item( + db, + item_type=payload.item_type, + target_lang=payload.target_lang, + term=payload.term, + translation=payload.translation, + domain=payload.domain, + aliases=payload.aliases, + title=payload.title, + content=payload.content, + description=payload.description, + sources=payload.sources, + confidence=payload.confidence, + status=payload.status, + created_by=payload.created_by, + embedding=embedding, + embedding_model=f"{rag_cfg.embedding_provider}:{rag_cfg.embedding_model}" if embedding else "", + ) + db.commit() + return item_id + + try: + item_id = _save_item() + except Exception as e: + db.rollback() + if not _is_missing_knowledge_table_error(e): + raise HTTPException(status_code=500, detail=f"knowledge item save failed: {e}") from e + _ensure_rag_schema(settings) + try: + item_id = _save_item() + except Exception as retry_error: + db.rollback() + raise HTTPException(status_code=503, detail=f"knowledge item save failed after schema migration: {retry_error}") from retry_error + + return KnowledgeItemUpsertResponse(id=uuid.UUID(item_id)) + + +@app.post("/subtitle/knowledge/rebuild-embeddings", response_model=KnowledgeEmbeddingRebuildResponse) +def rebuild_knowledge_embeddings_view( + payload: KnowledgeEmbeddingRebuildRequest, + settings: SubtitleServiceSettings = Depends(get_settings), + db: Session = Depends(get_db), +) -> KnowledgeEmbeddingRebuildResponse: + cfg = get_translate_settings(db, settings) + rag_cfg = rag_settings_from_translate_settings(cfg) + emb_cfg = embedding_settings_from_translate_settings(cfg) + def _rebuild() -> dict[str, Any]: + result = rebuild_knowledge_embeddings( + db, + rag_settings=rag_cfg, + embedding_settings=emb_cfg, + item_type=payload.item_type, + status=payload.status, + limit=payload.limit, + ) + db.commit() + return result + + try: + result = _rebuild() + except Exception as e: + db.rollback() + if not _is_missing_knowledge_table_error(e): + raise HTTPException(status_code=502, detail=f"knowledge embedding rebuild failed: {e}") from e + _ensure_rag_schema(settings) + try: + result = _rebuild() + except Exception as retry_error: + db.rollback() + raise HTTPException(status_code=503, detail=f"knowledge embedding rebuild failed after schema migration: {retry_error}") from retry_error + return KnowledgeEmbeddingRebuildResponse(**result) + + +@app.get("/subtitle/agents/runs", response_model=list[AgentRunRead]) +def list_agent_runs_view( + status: str | None = None, + limit: int = 30, + settings: SubtitleServiceSettings = Depends(get_settings), + db: Session = Depends(get_db), +) -> list[AgentRunRead]: + try: + rows = list_agent_runs(db, status=status, limit=limit) + except Exception as e: + db.rollback() + if not _is_missing_knowledge_table_error(e): + _handle_knowledge_db_error(e, settings) + _ensure_rag_schema(settings) + try: + rows = list_agent_runs(db, status=status, limit=limit) + except Exception as retry_error: + db.rollback() + raise HTTPException(status_code=503, detail=f"RAG agent tables are not ready: {retry_error}") from retry_error + return [AgentRunRead(**row) for row in rows] + + +@app.get("/subtitle/embedding/models", response_model=list[EmbeddingModelInfo]) +def list_embedding_models( + settings: SubtitleServiceSettings = Depends(get_settings), + db: Session = Depends(get_db), +) -> list[EmbeddingModelInfo]: + cfg = get_translate_settings(db, settings) + return [EmbeddingModelInfo(**item) for item in list_local_embedding_models(_embedding_models_dir_from_translate_settings(cfg, settings))] + + +@app.post("/subtitle/embedding/models/list", response_model=list[EmbeddingModelInfo]) +def list_embedding_models_for_dir( + payload: EmbeddingModelListRequest, + settings: SubtitleServiceSettings = Depends(get_settings), + db: Session = Depends(get_db), +) -> list[EmbeddingModelInfo]: + cfg = get_translate_settings(db, settings) + if payload.model_dir is not None: + cfg["rag_embedding_model_dir"] = payload.model_dir + return [EmbeddingModelInfo(**item) for item in list_local_embedding_models(_embedding_models_dir_from_translate_settings(cfg, settings))] + + +@app.post("/subtitle/embedding/models/download", response_model=EmbeddingModelInfo) +def download_embedding_model( + payload: EmbeddingModelDownloadRequest, + settings: SubtitleServiceSettings = Depends(get_settings), + db: Session = Depends(get_db), +) -> EmbeddingModelInfo: + asr_cfg = get_asr_settings(db, settings) + cfg = get_translate_settings(db, settings) + if payload.model_dir is not None: + cfg["rag_embedding_model_dir"] = payload.model_dir + proxy = str(asr_cfg.get("model_download_proxy") or "").strip() or None + try: + dest = download_local_embedding_model( + model=payload.model, + model_dir=_embedding_models_dir_from_translate_settings(cfg, settings), + name=payload.name, + revision=payload.revision, + force=payload.force, + proxy=proxy, + ) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) from e + size = sum(p.stat().st_size for p in dest.rglob("*") if p.is_file()) + return EmbeddingModelInfo(name=dest.name, path=str(dest), size_bytes=size) + + +@app.delete("/subtitle/embedding/models/{name}") +def delete_embedding_model( + name: str, + settings: SubtitleServiceSettings = Depends(get_settings), + db: Session = Depends(get_db), +) -> dict[str, bool]: + model_name = _validate_model_name(name) + cfg = get_translate_settings(db, settings) + delete_local_embedding_model(model_dir=_embedding_models_dir_from_translate_settings(cfg, settings), name=model_name) + return {"deleted": True} + + +@app.post("/subtitle/embedding/test", response_model=EmbeddingTestResponse) +def test_embedding( + payload: EmbeddingTestRequest, + settings: SubtitleServiceSettings = Depends(get_settings), + db: Session = Depends(get_db), +) -> EmbeddingTestResponse: + text = str(payload.text or "").strip() + if not text: + raise HTTPException(status_code=400, detail="text is required") + if len(text) > 2000: + raise HTTPException(status_code=400, detail="text too long (max 2000 chars)") + + cfg = get_translate_settings(db, settings) + if payload.provider is not None: + cfg["rag_embedding_provider"] = payload.provider + if payload.model is not None: + cfg["rag_embedding_model"] = payload.model + if payload.model_dir is not None: + cfg["rag_embedding_model_dir"] = payload.model_dir + if payload.dimensions is not None: + cfg["rag_embedding_dimensions"] = payload.dimensions + if payload.device is not None: + cfg["rag_embedding_device"] = payload.device + + emb_settings = embedding_settings_from_translate_settings(cfg) + try: + vector = embed_text(text, settings=emb_settings) + except Exception as e: + raise HTTPException(status_code=502, detail=str(e)) from e + expected = int(cfg.get("rag_embedding_dimensions") or len(vector)) + return EmbeddingTestResponse( + provider=emb_settings.provider, + model=emb_settings.model, + dimensions=len(vector), + expected_dimensions=expected, + ok=len(vector) == expected, ) diff --git a/src/videoroll/apps/subtitle_service/model_downloads.py b/src/videoroll/apps/subtitle_service/model_downloads.py index efbf5ad..275e7c3 100644 --- a/src/videoroll/apps/subtitle_service/model_downloads.py +++ b/src/videoroll/apps/subtitle_service/model_downloads.py @@ -24,7 +24,12 @@ OPENVINO_WHISPER_SIZE_TO_REPO: dict[str, str] = { "large-v3": "OpenVINO/whisper-large-v3-fp16-ov", } -_SUPPORTED_MODEL_DOWNLOAD_ENGINES = {"faster-whisper", "openvino"} +EMBEDDING_MODEL_TO_REPO: dict[str, str] = { + "bge-small-zh-v1.5": "BAAI/bge-small-zh-v1.5", + "bge-m3": "BAAI/bge-m3", +} + +_SUPPORTED_MODEL_DOWNLOAD_ENGINES = {"faster-whisper", "openvino", "embedding"} def normalize_model_download_engine(engine: str | None) -> str: @@ -35,6 +40,8 @@ def normalize_model_download_engine(engine: str | None) -> str: def _engine_size_to_repo(engine: str) -> dict[str, str]: + if engine == "embedding": + return EMBEDDING_MODEL_TO_REPO if engine == "openvino": return OPENVINO_WHISPER_SIZE_TO_REPO return FASTER_WHISPER_SIZE_TO_REPO @@ -60,6 +67,9 @@ def default_model_dir_name(engine: str, model: str) -> str: if engine_n == "openvino" and model_s in OPENVINO_WHISPER_SIZE_TO_REPO: return OPENVINO_WHISPER_SIZE_TO_REPO[model_s].split("/", 1)[1] + if engine_n == "embedding" and model_s in EMBEDDING_MODEL_TO_REPO: + return EMBEDDING_MODEL_TO_REPO[model_s].split("/", 1)[1] + return model_s.replace("/", "--") diff --git a/src/videoroll/apps/subtitle_service/processing.py b/src/videoroll/apps/subtitle_service/processing.py index 9dc6a0d..e5f4aaa 100644 --- a/src/videoroll/apps/subtitle_service/processing.py +++ b/src/videoroll/apps/subtitle_service/processing.py @@ -686,6 +686,7 @@ def translate_segments_openai_with_summary( batch_size: int = 50, enable_summary: bool = True, glossary: dict[str, str] | None = None, + rag_context_provider: Callable[[list[Segment], int, str], dict[str, Any] | None] | None = None, resume_from: Iterable[Segment] | None = None, initial_summary: str = "", on_batch_done: Callable[[list[Segment], str, int], None] | None = None, @@ -726,6 +727,10 @@ def translate_segments_openai_with_summary( payload_in["summary"] = summary if glossary: payload_in["glossary"] = glossary + if rag_context_provider is not None: + rag_context = rag_context_provider(batch, start_idx, summary) + if rag_context: + payload_in["rag_context"] = rag_context user_prompt = ( "你将收到一批字幕 block。请按 block 为单位翻译。\n" @@ -733,6 +738,8 @@ def translate_segments_openai_with_summary( "- 保留每个 block 的 idx 不变；不得增删 block，不得改变顺序；\n" "- 只翻译 text 字段；同一 block 内多行先合并理解再翻译；\n" "- 术语、人名保持一致；数字/单位尽量保留原格式；\n" + "- 如果输入包含 rag_context，请优先参考其中的 term_cards/knowledge_cards 来理解专有名词、梗、作品设定和技术背景；\n" + "- term_cards 中的 translation 是推荐译法，除非明显不符合当前上下文，否则保持一致；\n" "- 输出必须是 JSON 对象，且必须包含 translations 数组；不要输出任何解释。\n" f"- 目标语言：{tgt}\n" f"- 风格：{tone}\n\n" @@ -859,6 +866,7 @@ def translate_segments_openai( batch_size: int = 50, enable_summary: bool = True, glossary: dict[str, str] | None = None, + rag_context_provider: Callable[[list[Segment], int, str], dict[str, Any] | None] | None = None, ) -> list[Segment]: out, _summary = translate_segments_openai_with_summary( segments, @@ -872,6 +880,7 @@ def translate_segments_openai( batch_size=batch_size, enable_summary=enable_summary, glossary=glossary, + rag_context_provider=rag_context_provider, ) return out def _srt_ts(seconds: float) -> str: diff --git a/src/videoroll/apps/subtitle_service/rag.py b/src/videoroll/apps/subtitle_service/rag.py new file mode 100644 index 0000000..047e63f --- /dev/null +++ b/src/videoroll/apps/subtitle_service/rag.py @@ -0,0 +1,2216 @@ +from __future__ import annotations + +import hashlib +import html +import ipaddress +import json +import re +import socket +import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Iterable +from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse + +import httpx +from sqlalchemy import text +from sqlalchemy.orm import Session + +from videoroll.ai.client import OpenAIChatConfig, request_openai_json_object +from videoroll.apps.subtitle_service.embeddings import EmbeddingSettings, assert_embedding_dimensions, embed_text +from videoroll.apps.subtitle_service.processing import Segment + + +_TERM_SPLIT_RE = re.compile(r"[\s\-_]+") +_CANDIDATE_RE = re.compile(r"\b[A-Za-z][A-Za-z0-9][A-Za-z0-9'._:+#/-]*(?:\s+[A-Za-z0-9][A-Za-z0-9'._:+#/-]*){0,3}\b") +_SCRIPT_STYLE_RE = re.compile(r"<(script|style|noscript)\b[^>]*>.*?", re.IGNORECASE | re.DOTALL) +_TAG_RE = re.compile(r"<[^>]+>") +_ARTICLE_RE = re.compile(r"<(?:article|div)\b[^>]*class=[\"'][^\"']*\bresult\b[^\"']*[\"'][^>]*>.*?", re.IGNORECASE | re.DOTALL) +_ANCHOR_RE = re.compile(r"]*href=[\"']([^\"']+)[\"'][^>]*>(.*?)", re.IGNORECASE | re.DOTALL) +_SINGLE_LETTER_RE = re.compile(r"^[A-Za-z]$") +_MATH_LOGIC_DOMAIN_RE = re.compile(r"(logic|math|数学|逻辑|命题|proposition|propositional)", re.IGNORECASE) +_AUTO_APPROVE_CONFIDENCE_THRESHOLD = 0.9 +_COMMON_CONTEXT_TRANSLATIONS: dict[str, dict[str, str]] = { + "truth table": { + "translation": "真值表", + "domain": "技术/逻辑", + "description": "命题逻辑中列出命题变量所有取值组合及表达式真假结果的表格。", + }, + "propositional logic": { + "translation": "命题逻辑", + "domain": "技术/逻辑", + "description": "研究命题及命题连接词推理关系的逻辑分支。", + }, +} + + +@dataclass(frozen=True) +class RagSettings: + enabled: bool = False + top_k: int = 8 + min_score: float = 0.68 + embedding_provider: str = "openai" + embedding_model: str = "text-embedding-3-small" + embedding_dimensions: int = 1536 + embedding_model_dir: str = "/models/embeddings" + embedding_device: str = "cpu" + auto_discover_terms: bool = False + auto_learn_terms: bool = False + search_enabled: bool = False + search_url: str = "" + domain: str = "" + + +@dataclass(frozen=True) +class RagHit: + id: str + item_type: str + term: str + translation: str + target_lang: str + domain: str + aliases: list[str] + title: str + content: str + description: str + sources: list[dict[str, Any]] + confidence: float + status: str + score: float + + +@dataclass(frozen=True) +class RagContext: + term_cards: list[dict[str, Any]] + knowledge_cards: list[dict[str, Any]] + hits: list[RagHit] + + +@dataclass(frozen=True) +class ToolSpec: + tool_name: str + input_schema: dict[str, Any] + output_schema: dict[str, Any] + timeout_seconds: float = 20.0 + retry_count: int = 0 + cost: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class ToolResult: + spec: ToolSpec + input: dict[str, Any] + output: dict[str, Any] + ok: bool + duration_ms: int + error_type: str = "" + error: str = "" + + def to_step(self, *, action: str) -> dict[str, Any]: + step: dict[str, Any] = { + "kind": "tool", + "action": action, + "tool": self.spec.tool_name, + "tool_name": self.spec.tool_name, + "input_schema": self.spec.input_schema, + "output_schema": self.spec.output_schema, + "input": self.input, + "output": self.output, + "ok": self.ok, + "duration_ms": self.duration_ms, + "timeout_seconds": self.spec.timeout_seconds, + "retry_count": self.spec.retry_count, + "cost": self.spec.cost or {}, + } + if self.error_type: + step["error_type"] = self.error_type + if self.error: + step["error"] = self.error + return step + + +_SEARCH_TOOL_SPEC = ToolSpec( + tool_name="search", + input_schema={"type": "object", "required": ["query"], "properties": {"query": {"type": "string"}, "url": {"type": "string"}}}, + output_schema={"type": "object", "properties": {"count": {"type": "integer"}, "results": {"type": "array"}}}, + timeout_seconds=20.0, + retry_count=0, + cost={"network_requests": 1}, +) + +_FETCH_TOOL_SPEC = ToolSpec( + tool_name="fetch", + input_schema={"type": "object", "required": ["url"], "properties": {"url": {"type": "string"}, "title": {"type": "string"}}}, + output_schema={"type": "object", "properties": {"chars": {"type": "integer"}, "excerpt": {"type": "string"}}}, + timeout_seconds=20.0, + retry_count=0, + cost={"network_requests": 1}, +) + + +def rag_settings_from_translate_settings(settings: dict[str, Any]) -> RagSettings: + return RagSettings( + enabled=bool(settings.get("rag_enabled")), + top_k=max(0, min(30, int(settings.get("rag_top_k") or 8))), + min_score=max(0.0, min(1.0, float(settings.get("rag_min_score") or 0.68))), + embedding_provider=str(settings.get("rag_embedding_provider") or "openai").strip().lower() or "openai", + embedding_model=str(settings.get("rag_embedding_model") or "text-embedding-3-small").strip() or "text-embedding-3-small", + embedding_dimensions=max(1, min(4096, int(settings.get("rag_embedding_dimensions") or 1536))), + embedding_model_dir=str(settings.get("rag_embedding_model_dir") or "/models/embeddings").strip() or "/models/embeddings", + embedding_device=str(settings.get("rag_embedding_device") or "cpu").strip() or "cpu", + auto_discover_terms=bool(settings.get("rag_auto_discover_terms")), + auto_learn_terms=bool(settings.get("rag_auto_learn_terms")), + search_enabled=bool(settings.get("rag_search_enabled")), + search_url=str(settings.get("rag_search_url") or "").strip(), + domain=str(settings.get("rag_domain") or "").strip(), + ) + + +def normalize_term(term: str) -> str: + s = str(term or "").strip().lower() + s = _TERM_SPLIT_RE.sub(" ", s) + return s + + +def build_knowledge_embedding_text( + *, + item_type: str, + term: str = "", + translation: str = "", + domain: str = "", + aliases: Iterable[str] | None = None, + title: str = "", + content: str = "", + description: str = "", +) -> str: + parts = [ + f"type: {item_type}", + f"domain: {domain}".strip(), + f"term: {term}".strip(), + f"translation: {translation}".strip(), + f"aliases: {', '.join([a for a in aliases or [] if a])}".strip(), + f"title: {title}".strip(), + f"description: {description}".strip(), + f"content: {content}".strip(), + ] + return "\n".join([p for p in parts if p and not p.endswith(":")]).strip() + + +def _hash_text(text_value: str) -> str: + return hashlib.sha256(str(text_value or "").encode("utf-8")).hexdigest() + + +def _vector_literal(values: list[float]) -> str: + if not values: + raise ValueError("embedding vector is empty") + return "[" + ",".join(f"{float(v):.8g}" for v in values) + "]" + + +def embedding_model_key(rag_settings: RagSettings) -> str: + provider = str(rag_settings.embedding_provider or "openai").strip().lower() or "openai" + model = str(rag_settings.embedding_model or "").strip() + return f"{provider}:{model}" if model else provider + + +def _json_list(value: Any) -> list[Any]: + if isinstance(value, list): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + return parsed if isinstance(parsed, list) else [] + except Exception: + return [] + return [] + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _duration_ms(start: float) -> int: + return max(0, int((time.perf_counter() - start) * 1000)) + + +def _strip_html(value: str) -> str: + text_value = _SCRIPT_STYLE_RE.sub(" ", str(value or "")) + text_value = re.sub(r"", "\n", text_value, flags=re.IGNORECASE) + text_value = _TAG_RE.sub(" ", text_value) + text_value = html.unescape(text_value) + text_value = re.sub(r"[ \t\r\f\v]+", " ", text_value) + text_value = re.sub(r"\n\s+", "\n", text_value) + return text_value.strip() + + +def _collapse_text(value: str, *, limit: int = 12000) -> str: + text_value = html.unescape(str(value or "")) + text_value = re.sub(r"[ \t\r\f\v]+", " ", text_value) + text_value = re.sub(r"\n{3,}", "\n\n", text_value) + text_value = text_value.strip() + return text_value[:limit] + + +def _search_endpoint_from_base(search_base_url: str) -> str: + raw = str(search_base_url or "").strip() + if not raw: + return "" + if "://" not in raw: + raw = f"https://{raw}" + parsed = urlparse(raw) + params = [(k, v) for k, v in parse_qsl(parsed.query, keep_blank_values=True) if k not in {"q", "format"}] + path = parsed.path.rstrip("/") + if not path: + path = "/search" + elif path.endswith("/search"): + path = path + else: + path = f"{path}/search" + return urlunparse(parsed._replace(path=path, query=urlencode(params))) + + +def _search_url_with_params(search_base_url: str, *, query: str, json_format: bool) -> str: + endpoint = _search_endpoint_from_base(search_base_url) + parsed = urlparse(endpoint) + params = [(k, v) for k, v in parse_qsl(parsed.query, keep_blank_values=True) if k not in {"q", "format"}] + params.append(("q", query)) + if json_format: + params.append(("format", "json")) + return urlunparse(parsed._replace(query=urlencode(params))) + + +def _normalize_result_url(raw_url: str, *, base_url: str) -> str: + url = html.unescape(str(raw_url or "").strip()) + if not url: + return "" + parsed = urlparse(url) + if parsed.scheme in {"http", "https"}: + return url + if url.startswith("//"): + return f"https:{url}" + return urljoin(base_url, url) + + +def _is_search_engine_internal_url(url: str, *, search_url: str) -> bool: + parsed = urlparse(str(url or "").strip()) + search_parsed = urlparse(_search_endpoint_from_base(search_url)) + if not parsed.netloc or not search_parsed.netloc: + return False + if parsed.netloc.lower() != search_parsed.netloc.lower(): + return False + path = parsed.path.rstrip("/").lower() + search_path = search_parsed.path.rstrip("/").lower() + search_root = search_path[: -len("/search")] if search_path.endswith("/search") else "" + if path in {"", "/"}: + return True + if search_root and path == search_root: + return True + return ( + path == search_path + or path.startswith(f"{search_root}/info") + or path.startswith(f"{search_root}/preferences") + or path.startswith(f"{search_root}/stats") + or path.startswith(f"{search_root}/config") + or path.startswith(f"{search_root}/about") + ) + + +def _is_private_host(hostname: str) -> bool: + host = str(hostname or "").strip().strip("[]") + if not host: + return True + if host.lower() in {"localhost", "localhost.localdomain"}: + return True + try: + ip = ipaddress.ip_address(host) + return bool(ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved) + except ValueError: + pass + try: + for info in socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP): + addr = info[4][0] + ip = ipaddress.ip_address(addr) + if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved: + return True + except Exception: + return False + return False + + +def _is_fetchable_url(url: str) -> bool: + parsed = urlparse(str(url or "").strip()) + if parsed.scheme not in {"http", "https"}: + return False + if _is_private_host(parsed.hostname or ""): + return False + return True + + +def _parse_search_json(data: Any) -> list[dict[str, Any]]: + raw_results = data.get("results") if isinstance(data, dict) else None + if raw_results is None and isinstance(data, list): + raw_results = data + if not isinstance(raw_results, list): + return [] + + out: list[dict[str, Any]] = [] + for item in raw_results[:10]: + if not isinstance(item, dict): + continue + title = str(item.get("title") or item.get("name") or "").strip() + url = str(item.get("url") or item.get("link") or "").strip() + snippet = str(item.get("snippet") or item.get("content") or item.get("description") or "").strip() + if not title and not snippet: + continue + out.append({"title": _strip_html(title), "url": url, "snippet": _strip_html(snippet)[:800]}) + return out + + +def _filter_search_results(results: list[dict[str, Any]], *, search_url: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for item in results: + url = str(item.get("url") or "").strip() + title = str(item.get("title") or "").strip() + snippet = str(item.get("snippet") or "").strip() + if not url: + continue + if _is_search_engine_internal_url(url, search_url=search_url): + continue + if title.lower() in {"about", "preferences", "search syntax"} and "searxng" in snippet.lower(): + continue + if url in seen: + continue + seen.add(url) + out.append({"title": title, "url": url, "snippet": snippet[:800]}) + if len(out) >= 8: + break + return out + + +def _parse_search_html(html_value: str, *, base_url: str) -> list[dict[str, Any]]: + source = str(html_value or "") + chunks = _ARTICLE_RE.findall(source) + if not chunks: + chunks = re.findall(r"]*>.*?", source, flags=re.IGNORECASE | re.DOTALL) + if not chunks: + chunks = source.split("= 8: + break + return out + + +def _extract_page_text(html_value: str) -> str: + source = _SCRIPT_STYLE_RE.sub(" ", str(html_value or "")) + title_match = re.search(r"]*>(.*?)", source, flags=re.IGNORECASE | re.DOTALL) + title = _strip_html(title_match.group(1)) if title_match else "" + main_match = re.search(r"]*>(.*?)", source, flags=re.IGNORECASE | re.DOTALL) + article_match = re.search(r"]*>(.*?)", source, flags=re.IGNORECASE | re.DOTALL) + body_match = re.search(r"]*>(.*?)", source, flags=re.IGNORECASE | re.DOTALL) + body = (article_match or main_match or body_match) + body_text = _strip_html(body.group(1) if body else source) + joined = "\n\n".join([x for x in [title, body_text] if x]) + return _collapse_text(joined, limit=12000) + + +def _start_agent_run( + db: Session, + *, + term: str, + domain: str, + target_lang: str, + query: str, + task_id: str | None = None, + subtitle_job_id: str | None = None, +) -> str: + run_id = str(uuid.uuid4()) + db.execute( + text( + """ + INSERT INTO translation_agent_runs ( + id, agent_type, status, term, normalized_term, domain, target_lang, + task_id, subtitle_job_id, query, steps, result, started_at, updated_at + ) + VALUES ( + CAST(:id AS uuid), 'rag_term_research', 'running', :term, :normalized_term, + :domain, :target_lang, CAST(:task_id AS uuid), CAST(:subtitle_job_id AS uuid), + :query, '[]'::jsonb, '{}'::jsonb, now(), now() + ) + """ + ), + { + "id": run_id, + "term": str(term or "").strip(), + "normalized_term": normalize_term(term), + "domain": str(domain or "").strip(), + "target_lang": str(target_lang or "zh").strip() or "zh", + "task_id": task_id, + "subtitle_job_id": subtitle_job_id, + "query": str(query or "").strip(), + }, + ) + db.commit() + return run_id + + +def _append_agent_step(db: Session, run_id: str | None, step: dict[str, Any]) -> None: + if not run_id: + return + clean_step = dict(step) + clean_step.setdefault("at", _utc_now().isoformat()) + try: + db.execute( + text( + """ + UPDATE translation_agent_runs + SET steps = steps || CAST(:step AS jsonb), + updated_at = now() + WHERE id = CAST(:id AS uuid) + """ + ), + {"id": run_id, "step": json.dumps([clean_step], ensure_ascii=False)}, + ) + db.commit() + except Exception: + db.rollback() + + +def _append_tool_result(db: Session | None, run_id: str | None, result: ToolResult, *, action: str) -> None: + if db is None: + return + _append_agent_step(db, run_id, result.to_step(action=action)) + + +def _append_llm_step( + db: Session | None, + run_id: str | None, + *, + action: str, + config: OpenAIChatConfig | None = None, + input_value: dict[str, Any] | None = None, + output_value: dict[str, Any] | None = None, + duration_ms: int | None = None, + error: str = "", + error_type: str = "", +) -> None: + if db is None: + return + step: dict[str, Any] = {"kind": "llm", "action": action} + if config is not None: + step["model"] = config.model + step["tokens"] = None + if input_value is not None: + step["input"] = input_value + if output_value is not None: + step["output"] = output_value + if duration_ms is not None: + step["duration_ms"] = duration_ms + if error_type: + step["error_type"] = error_type + if error: + step["error"] = error[:1000] + _append_agent_step(db, run_id, step) + + +def _finish_agent_run( + db: Session, + run_id: str | None, + *, + status: str, + result: dict[str, Any] | None = None, + error: str = "", + knowledge_item_id: str | None = None, +) -> None: + if not run_id: + return + try: + db.execute( + text( + """ + UPDATE translation_agent_runs + SET status = :status, + result = CAST(:result AS jsonb), + error = :error, + knowledge_item_id = CAST(:knowledge_item_id AS uuid), + finished_at = now(), + updated_at = now() + WHERE id = CAST(:id AS uuid) + """ + ), + { + "id": run_id, + "status": str(status or "succeeded").strip() or "succeeded", + "result": json.dumps(result or {}, ensure_ascii=False), + "error": str(error or "")[:4000], + "knowledge_item_id": knowledge_item_id, + }, + ) + db.commit() + except Exception: + db.rollback() + + +def _row_to_hit(row: Any) -> RagHit: + mapping = getattr(row, "_mapping", row) + sources_raw = _json_list(mapping.get("sources")) + sources = [x for x in sources_raw if isinstance(x, dict)] + aliases = [str(x) for x in _json_list(mapping.get("aliases")) if str(x or "").strip()] + return RagHit( + id=str(mapping.get("id") or ""), + item_type=str(mapping.get("item_type") or ""), + term=str(mapping.get("term") or ""), + translation=str(mapping.get("translation") or ""), + target_lang=str(mapping.get("target_lang") or ""), + domain=str(mapping.get("domain") or ""), + aliases=aliases, + title=str(mapping.get("title") or ""), + content=str(mapping.get("content") or ""), + description=str(mapping.get("description") or ""), + sources=sources, + confidence=float(mapping.get("confidence") or 0.0), + status=str(mapping.get("status") or ""), + score=float(mapping.get("score") or 0.0), + ) + + +def _term_candidates_from_text(text_value: str, *, limit: int = 24) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for match in _CANDIDATE_RE.finditer(text_value or ""): + raw = match.group(0).strip(" .,:;!?()[]{}\"'") + if len(raw) < 2: + continue + if raw.lower() in {"the", "and", "you", "that", "this", "with", "have", "for"}: + continue + norm = normalize_term(raw) + if not norm or norm in seen: + continue + seen.add(norm) + out.append(raw) + if len(out) >= limit: + break + return out + + +def should_research_term(term: str, *, domain: str = "", context: str = "") -> dict[str, Any]: + clean = str(term or "").strip() + norm = normalize_term(clean) + domain_text = str(domain or "").strip() + if not clean or not norm: + return { + "should_research": False, + "should_persist": False, + "category": "empty", + "reason": "empty term", + } + if _SINGLE_LETTER_RE.fullmatch(clean): + category = "local_variable" if _MATH_LOGIC_DOMAIN_RE.search(domain_text) or _MATH_LOGIC_DOMAIN_RE.search(context or "") else "single_letter" + return { + "should_research": False, + "should_persist": False, + "category": category, + "reason": "single-letter symbols are treated as local variables unless manually added to the knowledge base", + } + context_term = _COMMON_CONTEXT_TRANSLATIONS.get(norm) + if context_term: + return { + "should_research": False, + "should_persist": False, + "category": "context_only", + "reason": "common foundational term; provide a temporary translation hint without long-term auto-learning", + "translation": context_term["translation"], + "domain": context_term.get("domain") or domain_text, + "description": context_term.get("description") or "", + } + if len(norm) <= 2 and clean.isalpha() and not clean.isupper(): + return { + "should_research": False, + "should_persist": False, + "category": "short_token", + "reason": "short lowercase token is unlikely to be a reusable translation term", + } + if norm in {"the", "and", "or", "to", "of", "in", "this", "that", "you", "we", "they", "it"}: + return { + "should_research": False, + "should_persist": False, + "category": "common_word", + "reason": "common function word", + } + return { + "should_research": True, + "should_persist": True, + "category": "research", + "reason": "term may need external context", + } + + +def _context_only_term_card(term: str, decision: dict[str, Any], *, target_lang: str, domain: str) -> dict[str, Any]: + return { + "term": str(term or "").strip(), + "translation": str(decision.get("translation") or "").strip(), + "domain": str(decision.get("domain") or domain or "").strip(), + "aliases": [], + "description": str(decision.get("description") or decision.get("reason") or "").strip(), + "confidence": 0.9, + "score": 0.9, + "sources": [], + "target_lang": target_lang, + "status": "context_only", + } + + +def discover_terms_openai( + text_value: str, + *, + target_lang: str, + domain_hint: str, + config: OpenAIChatConfig, + client: httpx.Client | None = None, +) -> list[dict[str, Any]]: + source = str(text_value or "").strip() + if not source: + return [] + data = request_openai_json_object( + config=config, + system_prompt="You identify translation terms in subtitles. Return ONLY valid JSON.", + user_prompt=( + "请从下面字幕片段中识别会影响翻译质量的术语/专有名词/梗/黑话。\n" + "要求：\n" + "- 只输出 JSON 对象；\n" + "- terms 最多 20 个；\n" + "- term 保持原文；domain 如果能判断就给出游戏/动漫/技术领域；\n" + "- reason 用中文简短说明为什么它是术语。\n" + f"- 目标语言：{target_lang or 'zh'}\n" + f"- 领域提示：{domain_hint or '未知'}\n\n" + f"字幕片段：\n{source[:8000]}\n\n" + '输出 JSON：{"terms":[{"term":"","domain":"","reason":""}]}' + ), + client=client, + ) + terms = data.get("terms") + if not isinstance(terms, list): + return [] + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for item in terms: + if not isinstance(item, dict): + continue + term = str(item.get("term") or "").strip() + norm = normalize_term(term) + if not term or norm in seen: + continue + seen.add(norm) + out.append( + { + "term": term, + "domain": str(item.get("domain") or domain_hint or "").strip(), + "reason": str(item.get("reason") or "").strip(), + } + ) + return out[:20] + + +def _dedupe_search_queries(queries: Iterable[str], *, limit: int = 3) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for query in queries: + clean = re.sub(r"\s+", " ", str(query or "").strip()) + if not clean: + continue + norm = clean.lower() + if norm in seen: + continue + seen.add(norm) + out.append(clean[:240]) + if len(out) >= max(1, min(5, int(limit))): + break + return out + + +def _fallback_search_queries(term: str, *, domain: str = "", target_lang: str = "zh", limit: int = 3) -> list[str]: + clean_term = str(term or "").strip() + clean_domain = str(domain or "").strip() + if not clean_term: + return [] + queries = [ + " ".join([p for p in [clean_domain, clean_term] if p]), + " ".join([p for p in [clean_term, clean_domain, "definition"] if p]), + ] + if str(target_lang or "").lower().startswith("zh"): + queries.append(" ".join([p for p in [clean_term, clean_domain, "中文 术语"] if p])) + return _dedupe_search_queries(queries, limit=limit) + + +def generate_search_queries_openai( + *, + term: str, + context: str, + target_lang: str, + domain_hint: str, + config: OpenAIChatConfig, + client: httpx.Client | None = None, + max_queries: int = 3, +) -> list[str]: + clean_term = str(term or "").strip() + if not clean_term: + return [] + safe_max = max(1, min(5, int(max_queries))) + data = request_openai_json_object( + config=config, + system_prompt="You generate web search queries for a translation research agent. Return ONLY valid JSON.", + user_prompt=( + "请为字幕翻译术语研究生成高质量搜索 query。\n" + "要求：\n" + f"- queries 输出 2 到 {safe_max} 条；\n" + "- query 要能帮助判断术语在当前上下文里的含义；\n" + "- 优先加入领域关键词、definition/wiki/documentation 等有助于找到权威解释的词；\n" + "- 不要生成站内搜索语法，不要编造 URL。\n\n" + f"术语：{clean_term}\n" + f"目标语言：{target_lang or 'zh'}\n" + f"领域提示：{domain_hint or '未知'}\n\n" + f"字幕上下文：\n{context[:2500]}\n\n" + '输出 JSON：{"queries":[""]}' + ), + client=client, + ) + raw_queries = data.get("queries") + if not isinstance(raw_queries, list): + return [] + queries = _dedupe_search_queries([str(x or "") for x in raw_queries], limit=safe_max) + return queries or _fallback_search_queries(clean_term, domain=domain_hint, target_lang=target_lang, limit=safe_max) + + +def decide_fetch_urls_openai( + *, + term: str, + context: str, + target_lang: str, + domain_hint: str, + search_results: list[dict[str, Any]], + config: OpenAIChatConfig, + client: httpx.Client | None = None, + max_pages: int = 4, +) -> dict[str, Any]: + compact_results: list[dict[str, Any]] = [] + for idx, item in enumerate(search_results[:8]): + compact_results.append( + { + "index": idx, + "title": str(item.get("title") or "")[:240], + "snippet": str(item.get("snippet") or "")[:700], + "url": str(item.get("url") or "")[:1000], + } + ) + safe_max_pages = max(1, min(6, int(max_pages))) + data = request_openai_json_object( + config=config, + system_prompt="You decide whether a translation research agent should fetch pages. Return ONLY valid JSON.", + user_prompt=( + "你是字幕翻译术语研究 Agent。请根据字幕上下文和搜索结果，决定是否需要打开网页正文。\n" + "要求：\n" + "- 如果搜索标题和摘要已经足够判断术语含义，就不要打开网页；\n" + "- 如果摘要过短、来源不清、存在歧义，选择最有价值的 URL 打开；\n" + f"- 最多选择 {safe_max_pages} 个 URL；优先官方 Wiki、官方文档、百科、项目文档；\n" + "- 只从给定 search_results 里选择 URL，不要编造 URL；\n" + "- confidence 表示仅凭当前搜索摘要判断术语含义的把握。\n\n" + f"术语：{term}\n" + f"目标语言：{target_lang or 'zh'}\n" + f"领域提示：{domain_hint or '未知'}\n\n" + f"字幕上下文：\n{context[:2500]}\n\n" + f"search_results JSON：\n{json.dumps(compact_results, ensure_ascii=False)}\n\n" + '输出 JSON：{"summary_sufficient":true,"fetch_urls":[],"reason":"","confidence":0.0}' + ), + client=client, + ) + allowed = {str(item.get("url") or "").strip() for item in search_results if str(item.get("url") or "").strip()} + urls: list[str] = [] + raw_urls = data.get("fetch_urls") + if isinstance(raw_urls, list): + for value in raw_urls: + url = str(value or "").strip() + if url and url in allowed and url not in urls: + urls.append(url) + raw_indexes = data.get("fetch_indexes") + if isinstance(raw_indexes, list): + for value in raw_indexes: + try: + idx = int(value) + except Exception: + continue + if 0 <= idx < len(search_results): + url = str(search_results[idx].get("url") or "").strip() + if url and url in allowed and url not in urls: + urls.append(url) + try: + confidence = float(data.get("confidence") or 0.0) + except Exception: + confidence = 0.0 + raw_sufficient = data.get("summary_sufficient") + if isinstance(raw_sufficient, str): + summary_sufficient = raw_sufficient.strip().lower() in {"1", "true", "yes", "y"} + else: + summary_sufficient = bool(raw_sufficient) + return { + "summary_sufficient": summary_sufficient, + "fetch_urls": urls[: max(1, min(6, int(max_pages)))], + "reason": str(data.get("reason") or "").strip()[:1000], + "confidence": max(0.0, min(1.0, confidence)), + } + + +def fetch_search_evidence( + term: str, + *, + domain: str, + search_url: str, + queries: list[str] | None = None, + context: str = "", + target_lang: str = "zh", + config: OpenAIChatConfig | None = None, + timeout_seconds: float = 20.0, + max_pages: int = 4, + db: Session | None = None, + agent_run_id: str | None = None, +) -> list[dict[str, Any]]: + endpoint = str(search_url or "").strip() + if not endpoint: + return [] + if queries is None: + search_queries = _dedupe_search_queries([" ".join([p for p in [str(domain or "").strip(), str(term or "").strip()] if p])], limit=1) + else: + search_queries = _dedupe_search_queries(queries, limit=3) + if not search_queries: + return [] + + search_results: list[dict[str, Any]] = [] + seen_result_urls: set[str] = set() + try: + with httpx.Client( + timeout=timeout_seconds, + follow_redirects=True, + headers={"User-Agent": "VideoRoll-RAG-Agent/1.0"}, + ) as client: + for query in search_queries: + json_url = _search_url_with_params(endpoint, query=query, json_format=True) + html_url = _search_url_with_params(endpoint, query=query, json_format=False) + started = time.perf_counter() + filtered_results: list[dict[str, Any]] = [] + try: + resp = client.get(json_url) + resp.raise_for_status() + content_type = resp.headers.get("content-type", "") + if "json" in content_type.lower(): + filtered_results = _parse_search_json(resp.json()) + else: + try: + filtered_results = _parse_search_json(resp.json()) + except Exception: + filtered_results = _parse_search_html(resp.text, base_url=str(resp.url)) + if not filtered_results: + resp = client.get(html_url) + resp.raise_for_status() + filtered_results = _parse_search_html(resp.text, base_url=str(resp.url)) + filtered_results = _filter_search_results(filtered_results, search_url=endpoint) + compact_results = [ + {"title": r.get("title", ""), "url": r.get("url", ""), "snippet": str(r.get("snippet") or "")[:240]} + for r in filtered_results[:8] + ] + step = ToolResult( + spec=_SEARCH_TOOL_SPEC, + input={"query": query, "url": json_url}, + output={"count": len(filtered_results), "results": compact_results}, + ok=True, + duration_ms=_duration_ms(started), + ).to_step(action="search") + step.update({"query": query, "url": json_url, "count": len(filtered_results), "results": compact_results}) + if db is not None: + _append_agent_step(db, agent_run_id, step) + except Exception as e: + step = ToolResult( + spec=_SEARCH_TOOL_SPEC, + input={"query": query, "url": json_url}, + output={"count": 0, "results": []}, + ok=False, + duration_ms=_duration_ms(started), + error_type=type(e).__name__, + error=str(e)[:300], + ).to_step(action="search_failed") + step.update({"query": query, "url": json_url}) + if db is not None: + _append_agent_step(db, agent_run_id, step) + continue + for item in filtered_results: + url = str(item.get("url") or "").strip() + if not url or url in seen_result_urls: + continue + seen_result_urls.add(url) + search_results.append(item) + if len(search_results) >= 8: + break + if len(search_results) >= 8: + break + + if not search_results: + if db is not None: + _append_agent_step( + db, + agent_run_id, + { + "kind": "tool", + "action": "search_no_valid_results", + "tool": "search", + "reason": "No usable external search results after filtering search engine internal pages.", + }, + ) + return [] + fetch_decision = { + "summary_sufficient": False, + "fetch_urls": [], + "reason": "LLM fetch decision was not available; using compatibility fallback.", + "confidence": 0.0, + } + decision_failed = False + if config is not None and search_results: + started = time.perf_counter() + try: + decision_input = { + "term": term, + "domain": domain, + "target_lang": target_lang, + "context_excerpt": str(context or "")[:500], + "results": [ + {"title": r.get("title", ""), "url": r.get("url", ""), "snippet": r.get("snippet", "")[:240]} + for r in search_results[:8] + ], + } + fetch_decision = decide_fetch_urls_openai( + term=term, + context=context, + target_lang=target_lang, + domain_hint=domain, + search_results=search_results, + config=config, + max_pages=max_pages, + ) + _append_llm_step( + db, + agent_run_id, + action="decide_fetch", + config=config, + input_value=decision_input, + output_value=fetch_decision, + duration_ms=_duration_ms(started), + ) + except Exception as e: + decision_failed = True + _append_llm_step( + db, + agent_run_id, + action="decide_fetch_failed", + config=config, + duration_ms=_duration_ms(started), + error=str(e)[:300], + error_type=type(e).__name__, + ) + if config is None: + fetch_decision["fetch_urls"] = [ + str(r.get("url") or "").strip() + for r in search_results[: max(1, min(6, int(max_pages)))] + if str(r.get("url") or "").strip() + ] + elif decision_failed and not fetch_decision.get("fetch_urls"): + fetch_decision["fetch_urls"] = [ + str(r.get("url") or "").strip() + for r in search_results[:1] + if str(r.get("url") or "").strip() + ] + if db is not None: + _append_agent_step( + db, + agent_run_id, + { + "kind": "policy", + "action": "fetch_fallback", + "reason": "LLM fetch decision failed; falling back to the first filtered external result.", + "urls": fetch_decision["fetch_urls"], + }, + ) + selected_urls = {str(url or "").strip() for url in fetch_decision.get("fetch_urls") or [] if str(url or "").strip()} + out: list[dict[str, Any]] = [] + fetched_count = 0 + for item in search_results[:8]: + title = str(item.get("title") or "").strip() + url = str(item.get("url") or "").strip() + snippet = str(item.get("snippet") or "").strip() + page: dict[str, Any] = {"title": title, "url": url, "snippet": snippet[:800]} + if url and url in selected_urls and _is_fetchable_url(url) and fetched_count < max(1, min(6, int(max_pages))): + started = time.perf_counter() + try: + page_resp = client.get(url) + page_resp.raise_for_status() + ctype = page_resp.headers.get("content-type", "").lower() + body = page_resp.text[:500_000] + if "html" in ctype or " dict[str, Any] | None: + clean_term = str(term or "").strip() + if not clean_term: + return None + data = request_openai_json_object( + config=config, + system_prompt="You build a verified translation glossary. Return ONLY valid JSON.", + user_prompt=( + "请根据字幕上下文和检索资料，判断术语最贴切的中文译法。\n" + "要求：\n" + "- 不确定时 confidence 低于 0.7；\n" + "- 检索资料可能包含搜索结果摘要、url、以及打开网页后抽取的 content；优先使用 content 和可靠来源；\n" + "- 不要把无关网页、广告、导航文字当成术语依据；\n" + "- sources 只保留实际支持判断的来源；\n" + "- description 用中文说明语境；\n" + "- translation 是字幕翻译可直接使用的译法。\n\n" + f"术语：{clean_term}\n" + f"目标语言：{target_lang or 'zh'}\n" + f"领域提示：{domain_hint or '未知'}\n\n" + f"字幕上下文：\n{context[:3000]}\n\n" + f"检索资料 JSON：\n{json.dumps(evidence, ensure_ascii=False)[:6000]}\n\n" + '输出 JSON：{"term":"","translation":"","domain":"","aliases":[],"description":"","sources":[],"confidence":0.0}' + ), + client=client, + ) + translation = str(data.get("translation") or "").strip() + if not translation: + return None + aliases = data.get("aliases") + sources = data.get("sources") + try: + confidence = float(data.get("confidence") or 0.0) + except Exception: + confidence = 0.0 + return { + "term": str(data.get("term") or clean_term).strip(), + "translation": translation, + "domain": str(data.get("domain") or domain_hint or "").strip(), + "aliases": aliases if isinstance(aliases, list) else [], + "description": str(data.get("description") or "").strip(), + "sources": sources if isinstance(sources, list) else [], + "confidence": max(0.0, min(1.0, confidence)), + } + + +def _json_bool(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "y"} + return bool(value) + + +def _valid_external_evidence(evidence: list[dict[str, Any]], *, search_url: str = "") -> list[dict[str, str]]: + out: list[dict[str, str]] = [] + for item in evidence: + if not isinstance(item, dict): + continue + url = str(item.get("url") or "").strip() + title = str(item.get("title") or "").strip() + snippet = str(item.get("snippet") or "").strip() + content = str(item.get("content") or "").strip() + if not url or item.get("fetch_error"): + continue + if search_url and _is_search_engine_internal_url(url, search_url=search_url): + continue + if not snippet and not content: + continue + snippet_norm = re.sub(r"[\W_]+", " ", snippet.lower()).strip() + if not content and (len(snippet) < 24 or snippet_norm in {"read more", "read more read more"}): + continue + out.append({"title": title, "url": url}) + return out + + +def verify_glossary_entry_openai( + *, + term: str, + context: str, + target_lang: str, + domain_hint: str, + evidence: list[dict[str, Any]], + candidate: dict[str, Any], + config: OpenAIChatConfig, + client: httpx.Client | None = None, +) -> dict[str, Any]: + clean_term = str(term or "").strip() + compact_evidence = [ + { + "title": str(item.get("title") or "")[:240], + "url": str(item.get("url") or "")[:1000], + "snippet": str(item.get("snippet") or "")[:700], + "content": str(item.get("content") or "")[:1200], + "fetch_error": str(item.get("fetch_error") or "")[:300], + } + for item in evidence[:8] + if isinstance(item, dict) + ] + compact_candidate = { + "term": str(candidate.get("term") or clean_term), + "translation": str(candidate.get("translation") or ""), + "domain": str(candidate.get("domain") or domain_hint or ""), + "aliases": candidate.get("aliases") if isinstance(candidate.get("aliases"), list) else [], + "description": str(candidate.get("description") or ""), + "sources": candidate.get("sources") if isinstance(candidate.get("sources"), list) else [], + "confidence": float(candidate.get("confidence") or 0.0), + } + data = request_openai_json_object( + config=config, + system_prompt="You verify a translation glossary candidate for a RAG knowledge base. Return ONLY valid JSON.", + user_prompt=( + "请作为独立 verifier，判断候选术语条目是否应该写入长期翻译知识库。\n" + "要求：\n" + "- 必须检查检索资料是否真正支持候选译法和描述；\n" + "- 必须检查候选解释是否符合字幕上下文；\n" + "- 搜索引擎 About/Preferences/导航页、广告页、无正文摘要不能作为有效来源；\n" + "- 单字母变量、局部变量、一次性占位符不应该写入长期知识库；\n" + "- 如果没有可靠外部来源，但译法只适合当前字幕，failure_category 使用 context_only，should_write=false；\n" + "- should_auto_approve 只有在来源明确、上下文一致、置信度很高时才为 true。\n\n" + f"术语：{clean_term}\n" + f"目标语言：{target_lang or 'zh'}\n" + f"领域提示：{domain_hint or '未知'}\n\n" + f"字幕上下文：\n{context[:3000]}\n\n" + f"候选条目 JSON：\n{json.dumps(compact_candidate, ensure_ascii=False)}\n\n" + f"检索资料 JSON：\n{json.dumps(compact_evidence, ensure_ascii=False)[:7000]}\n\n" + '输出 JSON：{"supported":true,"context_consistent":true,"should_write":true,' + '"should_auto_approve":false,"confidence":0.0,"reason":"","failure_category":""}' + ), + client=client, + ) + try: + confidence = float(data.get("confidence") or 0.0) + except Exception: + confidence = 0.0 + return { + "supported": _json_bool(data.get("supported")), + "context_consistent": _json_bool(data.get("context_consistent")), + "should_write": _json_bool(data.get("should_write")), + "should_auto_approve": _json_bool(data.get("should_auto_approve")), + "confidence": max(0.0, min(1.0, confidence)), + "reason": str(data.get("reason") or "").strip()[:1200], + "failure_category": str(data.get("failure_category") or "").strip()[:120], + } + + +def upsert_knowledge_item( + db: Session, + *, + item_type: str, + target_lang: str, + term: str = "", + translation: str = "", + domain: str = "", + aliases: list[str] | None = None, + title: str = "", + content: str = "", + description: str = "", + sources: list[dict[str, Any]] | None = None, + confidence: float = 0.0, + status: str = "approved", + created_by: str = "manual", + embedding: list[float] | None = None, + embedding_model: str = "", +) -> str: + item_id = str(uuid.uuid4()) + clean_item_type = str(item_type or "document").strip() or "document" + clean_target_lang = str(target_lang or "zh").strip() or "zh" + clean_term = str(term or "").strip() + clean_domain = str(domain or "").strip() + norm = normalize_term(clean_term) + alias_list = [str(x).strip() for x in aliases or [] if str(x or "").strip()] + source_list = [x for x in sources or [] if isinstance(x, dict)] + emb_literal = _vector_literal(embedding) if embedding else None + embedding_text = build_knowledge_embedding_text( + item_type=clean_item_type, + term=clean_term, + translation=translation, + domain=clean_domain, + aliases=alias_list, + title=title, + content=content, + description=description, + ) + embedding_hash = _hash_text(embedding_text) if embedding else "" + + if clean_item_type == "term" and norm: + existing = db.execute( + text( + """ + SELECT id FROM translation_knowledge_items + WHERE item_type = 'term' + AND target_lang = :target_lang + AND domain = :domain + AND normalized_term = :normalized_term + LIMIT 1 + """ + ), + {"target_lang": clean_target_lang, "domain": clean_domain, "normalized_term": norm}, + ).first() + if existing: + item_id = str(existing[0]) + db.execute( + text( + """ + UPDATE translation_knowledge_items + SET term = :term, + translation = :translation, + aliases = CAST(:aliases AS jsonb), + title = :title, + content = :content, + description = :description, + sources = CAST(:sources AS jsonb), + confidence = :confidence, + status = :status, + created_by = :created_by, + embedding = COALESCE(CAST(:embedding AS vector), embedding), + embedding_model = CASE WHEN :embedding_model <> '' THEN :embedding_model ELSE embedding_model END, + embedding_text_hash = CASE WHEN :embedding_hash <> '' THEN :embedding_hash ELSE embedding_text_hash END, + last_verified_at = :last_verified_at, + updated_at = now() + WHERE id = CAST(:id AS uuid) + """ + ), + { + "id": item_id, + "term": clean_term, + "translation": str(translation or "").strip(), + "aliases": json.dumps(alias_list, ensure_ascii=False), + "title": str(title or "").strip(), + "content": str(content or "").strip(), + "description": str(description or "").strip(), + "sources": json.dumps(source_list, ensure_ascii=False), + "confidence": max(0.0, min(1.0, float(confidence or 0.0))), + "status": str(status or "approved").strip() or "approved", + "created_by": str(created_by or "manual").strip() or "manual", + "embedding": emb_literal, + "embedding_model": str(embedding_model or "").strip(), + "embedding_hash": embedding_hash, + "last_verified_at": datetime.now(timezone.utc), + }, + ) + return item_id + + db.execute( + text( + """ + INSERT INTO translation_knowledge_items ( + id, item_type, term, normalized_term, translation, target_lang, domain, + aliases, title, content, description, sources, confidence, status, + created_by, embedding, embedding_model, embedding_text_hash, last_verified_at + ) + VALUES ( + CAST(:id AS uuid), :item_type, :term, :normalized_term, :translation, :target_lang, :domain, + CAST(:aliases AS jsonb), :title, :content, :description, CAST(:sources AS jsonb), + :confidence, :status, :created_by, CAST(:embedding AS vector), :embedding_model, + :embedding_hash, :last_verified_at + ) + """ + ), + { + "id": item_id, + "item_type": clean_item_type, + "term": clean_term, + "normalized_term": norm, + "translation": str(translation or "").strip(), + "target_lang": clean_target_lang, + "domain": clean_domain, + "aliases": json.dumps(alias_list, ensure_ascii=False), + "title": str(title or "").strip(), + "content": str(content or "").strip(), + "description": str(description or "").strip(), + "sources": json.dumps(source_list, ensure_ascii=False), + "confidence": max(0.0, min(1.0, float(confidence or 0.0))), + "status": str(status or "approved").strip() or "approved", + "created_by": str(created_by or "manual").strip() or "manual", + "embedding": emb_literal, + "embedding_model": str(embedding_model or "").strip(), + "embedding_hash": embedding_hash, + "last_verified_at": datetime.now(timezone.utc), + }, + ) + return item_id + + +def search_knowledge( + db: Session, + *, + query_embedding: list[float], + target_lang: str, + domain: str = "", + embedding_model: str = "", + top_k: int = 8, + min_score: float = 0.68, +) -> list[RagHit]: + if not query_embedding or top_k <= 0: + return [] + params = { + "embedding": _vector_literal(query_embedding), + "dimensions": len(query_embedding), + "target_lang": str(target_lang or "zh").strip() or "zh", + "domain": str(domain or "").strip(), + "embedding_model": str(embedding_model or "").strip(), + "limit": max(1, min(30, int(top_k))), + "min_score": max(0.0, min(1.0, float(min_score))), + } + rows = db.execute( + text( + """ + SELECT id, item_type, term, translation, target_lang, domain, aliases, title, + content, description, sources, confidence, status, + GREATEST(0, 1 - (embedding <=> CAST(:embedding AS vector))) AS score + FROM translation_knowledge_items + WHERE target_lang = :target_lang + AND status IN ('approved', 'auto_approved') + AND embedding IS NOT NULL + AND vector_dims(embedding) = :dimensions + AND (:embedding_model = '' OR embedding_model = :embedding_model) + AND (:domain = '' OR domain = '' OR domain = :domain) + AND GREATEST(0, 1 - (embedding <=> CAST(:embedding AS vector))) >= :min_score + ORDER BY embedding <=> CAST(:embedding AS vector) + LIMIT :limit + """ + ), + params, + ).all() + return [_row_to_hit(row) for row in rows] + + +def exact_term_hits( + db: Session, + *, + text_value: str, + target_lang: str, + domain: str = "", + limit: int = 20, +) -> list[RagHit]: + candidates = _term_candidates_from_text(text_value, limit=limit) + norms = [normalize_term(x) for x in candidates] + if not norms: + return [] + rows = db.execute( + text( + """ + SELECT id, item_type, term, translation, target_lang, domain, aliases, title, + content, description, sources, confidence, status, 1.0 AS score + FROM translation_knowledge_items + WHERE item_type = 'term' + AND target_lang = :target_lang + AND status IN ('approved', 'auto_approved') + AND normalized_term = ANY(:norms) + AND (:domain = '' OR domain = '' OR domain = :domain) + LIMIT :limit + """ + ), + { + "target_lang": str(target_lang or "zh").strip() or "zh", + "domain": str(domain or "").strip(), + "norms": norms, + "limit": max(1, min(50, int(limit))), + }, + ).all() + return [_row_to_hit(row) for row in rows] + + +def build_rag_context( + db: Session, + *, + segments: list[Segment], + target_lang: str, + rag_settings: RagSettings, + embedding_settings: EmbeddingSettings, + chat_config: OpenAIChatConfig, + task_id: str | None = None, + subtitle_job_id: str | None = None, +) -> RagContext: + if not rag_settings.enabled: + return RagContext(term_cards=[], knowledge_cards=[], hits=[]) + + text_value = "\n".join([str(s.text or "").strip() for s in segments if str(s.text or "").strip()]) + if not text_value: + return RagContext(term_cards=[], knowledge_cards=[], hits=[]) + + hits: list[RagHit] = [] + seen_ids: set[str] = set() + for hit in exact_term_hits( + db, + text_value=text_value, + target_lang=target_lang, + domain=rag_settings.domain, + limit=max(rag_settings.top_k * 2, 10), + ): + if hit.id not in seen_ids: + seen_ids.add(hit.id) + hits.append(hit) + + try: + query_embedding = embed_text(text_value[:8000], settings=embedding_settings) + assert_embedding_dimensions(query_embedding, rag_settings.embedding_dimensions) + for hit in search_knowledge( + db, + query_embedding=query_embedding, + target_lang=target_lang, + domain=rag_settings.domain, + embedding_model=embedding_model_key(rag_settings), + top_k=rag_settings.top_k, + min_score=rag_settings.min_score, + ): + if hit.id not in seen_ids: + seen_ids.add(hit.id) + hits.append(hit) + except Exception: + query_embedding = [] + + context_only_cards: list[dict[str, Any]] = [] + + if rag_settings.auto_discover_terms: + try: + discovered = discover_terms_openai( + text_value, + target_lang=target_lang, + domain_hint=rag_settings.domain, + config=chat_config, + ) + except Exception: + discovered = [] + known_norms = {normalize_term(h.term) for h in hits if h.term} + for item in discovered: + term = str(item.get("term") or "").strip() + norm = normalize_term(term) + if not term or norm in known_norms: + continue + if not rag_settings.auto_learn_terms: + continue + domain_hint = str(item.get("domain") or rag_settings.domain or "").strip() + query = " ".join([p for p in [domain_hint, term] if p]).strip() + agent_run_id: str | None = None + try: + agent_run_id = _start_agent_run( + db, + term=term, + domain=domain_hint, + target_lang=target_lang, + query=query, + task_id=task_id, + subtitle_job_id=subtitle_job_id, + ) + _append_llm_step( + db, + agent_run_id, + action="term_discovered", + config=chat_config, + output_value={ + "term": term, + "domain": domain_hint, + "reason": str(item.get("reason") or "").strip(), + }, + ) + except Exception: + db.rollback() + agent_run_id = None + + research_policy = should_research_term(term, domain=domain_hint, context=text_value) + _append_agent_step( + db, + agent_run_id, + { + "kind": "policy", + "action": "term_research_policy", + "term": term, + "domain": domain_hint, + "decision": research_policy, + }, + ) + if not research_policy.get("should_research"): + if research_policy.get("category") == "context_only" and research_policy.get("translation"): + context_only_cards.append(_context_only_term_card(term, research_policy, target_lang=target_lang, domain=domain_hint)) + _finish_agent_run( + db, + agent_run_id, + status="skipped", + result={ + "term": term, + "domain": domain_hint, + "knowledge_status": str(research_policy.get("category") or "skipped"), + "failure_category": str(research_policy.get("category") or "skipped"), + "reason": str(research_policy.get("reason") or ""), + }, + ) + known_norms.add(norm) + continue + + search_queries: list[str] = [] + if rag_settings.search_enabled: + started = time.perf_counter() + try: + search_queries = generate_search_queries_openai( + term=term, + context=text_value, + target_lang=target_lang, + domain_hint=domain_hint, + config=chat_config, + max_queries=3, + ) + _append_llm_step( + db, + agent_run_id, + action="generate_search_queries", + config=chat_config, + input_value={"term": term, "domain": domain_hint, "context_excerpt": text_value[:500]}, + output_value={"queries": search_queries}, + duration_ms=_duration_ms(started), + ) + except Exception as e: + search_queries = _fallback_search_queries(term, domain=domain_hint, target_lang=target_lang, limit=3) + _append_llm_step( + db, + agent_run_id, + action="generate_search_queries_failed", + config=chat_config, + duration_ms=_duration_ms(started), + error=str(e)[:300], + error_type=type(e).__name__, + ) + _append_agent_step( + db, + agent_run_id, + { + "kind": "policy", + "action": "search_query_fallback", + "queries": search_queries, + "reason": "LLM query generation failed; using deterministic fallback queries.", + }, + ) + evidence = [] + if rag_settings.search_enabled: + evidence = fetch_search_evidence( + term, + domain=domain_hint, + search_url=rag_settings.search_url, + queries=search_queries, + context=text_value, + target_lang=target_lang, + config=chat_config, + db=db, + agent_run_id=agent_run_id, + ) + if not evidence: + _finish_agent_run( + db, + agent_run_id, + status="failed", + result={ + "term": term, + "domain": domain_hint, + "search_queries": search_queries, + "failure_category": "no_search_evidence", + }, + error="no search evidence", + ) + continue + try: + summarize_started = time.perf_counter() + explained = explain_term_from_evidence_openai( + term=term, + context=text_value, + target_lang=target_lang, + domain_hint=domain_hint, + evidence=evidence, + config=chat_config, + ) + _append_llm_step( + db, + agent_run_id, + action="summarize_evidence", + config=chat_config, + input_value={"term": term, "domain": domain_hint, "evidence_count": len(evidence)}, + output_value={ + "term": str((explained or {}).get("term") or term), + "translation": str((explained or {}).get("translation") or ""), + "confidence": float((explained or {}).get("confidence") or 0.0), + }, + duration_ms=_duration_ms(summarize_started), + ) + except Exception as e: + _append_llm_step( + db, + agent_run_id, + action="summarize_failed", + config=chat_config, + error=str(e)[:300], + error_type=type(e).__name__, + ) + explained = None + if not explained: + _finish_agent_run( + db, + agent_run_id, + status="failed", + result={"term": term, "domain": domain_hint, "evidence_count": len(evidence), "failure_category": "summarize_failed"}, + error="LLM did not produce a usable glossary entry", + ) + continue + + valid_sources = _valid_external_evidence(evidence, search_url=rag_settings.search_url) + try: + verify_started = time.perf_counter() + verification = verify_glossary_entry_openai( + term=term, + context=text_value, + target_lang=target_lang, + domain_hint=domain_hint, + evidence=evidence, + candidate=explained, + config=chat_config, + ) + _append_llm_step( + db, + agent_run_id, + action="verify_glossary_entry", + config=chat_config, + input_value={ + "term": term, + "candidate_confidence": float(explained.get("confidence") or 0.0), + "valid_source_count": len(valid_sources), + }, + output_value=verification, + duration_ms=_duration_ms(verify_started), + ) + except Exception as e: + verification = { + "supported": False, + "context_consistent": False, + "should_write": False, + "should_auto_approve": False, + "confidence": 0.0, + "reason": "verifier failed", + "failure_category": "verify_failed", + } + _append_llm_step( + db, + agent_run_id, + action="verify_failed", + config=chat_config, + error=str(e)[:300], + error_type=type(e).__name__, + ) + + candidate_confidence = float(explained.get("confidence") or 0.0) + verifier_confidence = float(verification.get("confidence") or 0.0) + final_confidence = min(candidate_confidence, verifier_confidence) if verifier_confidence > 0 else candidate_confidence + should_write = ( + bool(verification.get("supported")) + and bool(verification.get("context_consistent")) + and bool(verification.get("should_write")) + and bool(valid_sources) + ) + if not should_write: + failure_category = str(verification.get("failure_category") or "") + if not valid_sources: + failure_category = failure_category or "no_valid_external_source" + _finish_agent_run( + db, + agent_run_id, + status="skipped", + result={ + "term": str(explained.get("term") or term), + "translation": str(explained.get("translation") or ""), + "domain": str(explained.get("domain") or domain_hint or ""), + "confidence": final_confidence, + "knowledge_status": "context_only" if failure_category == "context_only" else "not_written", + "failure_category": failure_category or "verify_rejected", + "verification": verification, + "valid_source_count": len(valid_sources), + "search_queries": search_queries, + }, + error=str(verification.get("reason") or "verifier rejected glossary entry")[:4000], + ) + known_norms.add(norm) + continue + + status = ( + "auto_approved" + if bool(verification.get("should_auto_approve")) and final_confidence >= _AUTO_APPROVE_CONFIDENCE_THRESHOLD + else "pending" + ) + source_list = [x for x in explained.get("sources") or [] if isinstance(x, dict)] or valid_sources + try: + _append_agent_step( + db, + agent_run_id, + { + "kind": "policy", + "action": "knowledge_write_decision", + "term": str(explained.get("term") or term), + "translation": str(explained.get("translation") or ""), + "confidence": final_confidence, + "status": status, + "auto_threshold": _AUTO_APPROVE_CONFIDENCE_THRESHOLD, + "valid_source_count": len(valid_sources), + }, + ) + emb_text = build_knowledge_embedding_text( + item_type="term", + term=str(explained.get("term") or term), + translation=str(explained.get("translation") or ""), + domain=str(explained.get("domain") or ""), + aliases=[str(x) for x in explained.get("aliases") or []], + description=str(explained.get("description") or ""), + ) + emb = embed_text(emb_text, settings=embedding_settings) + assert_embedding_dimensions(emb, rag_settings.embedding_dimensions) + learned_id = upsert_knowledge_item( + db, + item_type="term", + target_lang=target_lang, + term=str(explained.get("term") or term), + translation=str(explained.get("translation") or ""), + domain=str(explained.get("domain") or rag_settings.domain or ""), + aliases=[str(x) for x in explained.get("aliases") or []], + description=str(explained.get("description") or ""), + sources=source_list, + confidence=final_confidence, + status=status, + created_by="agent", + embedding=emb, + embedding_model=embedding_model_key(rag_settings), + ) + db.commit() + _finish_agent_run( + db, + agent_run_id, + status="succeeded", + result={ + "term": str(explained.get("term") or term), + "translation": str(explained.get("translation") or ""), + "domain": str(explained.get("domain") or rag_settings.domain or ""), + "confidence": final_confidence, + "knowledge_status": status, + "verification": verification, + "valid_source_count": len(valid_sources), + "search_queries": search_queries, + "sources": source_list[:5], + }, + knowledge_item_id=learned_id, + ) + known_norms.add(norm) + if status == "auto_approved": + hits.append( + RagHit( + id=learned_id, + item_type="term", + term=str(explained.get("term") or term), + translation=str(explained.get("translation") or ""), + target_lang=target_lang, + domain=str(explained.get("domain") or rag_settings.domain or ""), + aliases=[str(x) for x in explained.get("aliases") or []], + title="", + content="", + description=str(explained.get("description") or ""), + sources=source_list, + confidence=final_confidence, + status=status, + score=final_confidence, + ) + ) + except Exception as e: + db.rollback() + _finish_agent_run( + db, + agent_run_id, + status="failed", + result={ + "term": str(explained.get("term") or term), + "translation": str(explained.get("translation") or ""), + "confidence": final_confidence, + "failure_category": "knowledge_save_failed", + }, + error=f"knowledge save failed: {e}", + ) + + term_cards: list[dict[str, Any]] = list(context_only_cards) + knowledge_cards: list[dict[str, Any]] = [] + for hit in hits[: rag_settings.top_k]: + if hit.item_type == "term": + term_cards.append( + { + "term": hit.term, + "translation": hit.translation, + "domain": hit.domain, + "aliases": hit.aliases, + "description": hit.description, + "confidence": hit.confidence, + "score": round(hit.score, 4), + "sources": hit.sources[:3], + } + ) + else: + knowledge_cards.append( + { + "title": hit.title or hit.term, + "domain": hit.domain, + "content": (hit.content or hit.description)[:1200], + "score": round(hit.score, 4), + "sources": hit.sources[:3], + } + ) + + if task_id or subtitle_job_id: + _record_matches( + db, + hits=hits[: rag_settings.top_k], + task_id=task_id, + subtitle_job_id=subtitle_job_id, + context=text_value[:2000], + ) + + return RagContext(term_cards=term_cards, knowledge_cards=knowledge_cards, hits=hits) + + +def _record_matches( + db: Session, + *, + hits: list[RagHit], + task_id: str | None, + subtitle_job_id: str | None, + context: str, +) -> None: + if not hits: + return + for hit in hits: + try: + db.execute( + text( + """ + INSERT INTO translation_term_matches ( + id, task_id, subtitle_job_id, knowledge_item_id, term, + normalized_term, raw_context, decision + ) + VALUES ( + CAST(:id AS uuid), + CAST(:task_id AS uuid), + CAST(:subtitle_job_id AS uuid), + CAST(:knowledge_item_id AS uuid), + :term, + :normalized_term, + :raw_context, + :decision + ) + """ + ), + { + "id": str(uuid.uuid4()), + "task_id": str(task_id) if task_id else None, + "subtitle_job_id": str(subtitle_job_id) if subtitle_job_id else None, + "knowledge_item_id": hit.id, + "term": hit.term or hit.title, + "normalized_term": normalize_term(hit.term or hit.title), + "raw_context": context, + "decision": f"score={hit.score:.4f}", + }, + ) + db.execute( + text("UPDATE translation_knowledge_items SET usage_count = usage_count + 1, updated_at = now() WHERE id = CAST(:id AS uuid)"), + {"id": hit.id}, + ) + except Exception: + continue + + +def list_knowledge_items(db: Session, *, item_type: str | None = None, status: str | None = None, limit: int = 100) -> list[dict[str, Any]]: + clauses = ["1=1"] + params: dict[str, Any] = {"limit": max(1, min(500, int(limit)))} + if item_type: + clauses.append("item_type = :item_type") + params["item_type"] = item_type + if status: + clauses.append("status = :status") + params["status"] = status + rows = db.execute( + text( + f""" + SELECT id, item_type, term, translation, target_lang, domain, aliases, title, + content, description, sources, confidence, status, created_by, + usage_count, embedding_model, last_verified_at, created_at, updated_at + FROM translation_knowledge_items + WHERE {' AND '.join(clauses)} + ORDER BY updated_at DESC, created_at DESC + LIMIT :limit + """ + ), + params, + ).all() + out: list[dict[str, Any]] = [] + for row in rows: + m = row._mapping + out.append( + { + "id": str(m["id"]), + "item_type": m["item_type"], + "term": m["term"], + "translation": m["translation"], + "target_lang": m["target_lang"], + "domain": m["domain"], + "aliases": _json_list(m["aliases"]), + "title": m["title"], + "content": m["content"], + "description": m["description"], + "sources": _json_list(m["sources"]), + "confidence": float(m["confidence"] or 0.0), + "status": m["status"], + "created_by": m["created_by"], + "usage_count": int(m["usage_count"] or 0), + "embedding_model": m["embedding_model"], + "last_verified_at": m["last_verified_at"], + "created_at": m["created_at"], + "updated_at": m["updated_at"], + } + ) + return out + + +def list_agent_runs(db: Session, *, status: str | None = None, limit: int = 50) -> list[dict[str, Any]]: + clauses = ["1=1"] + params: dict[str, Any] = {"limit": max(1, min(100, int(limit)))} + if status: + clauses.append("status = :status") + params["status"] = str(status).strip() + rows = db.execute( + text( + f""" + SELECT id, agent_type, status, term, domain, target_lang, task_id, subtitle_job_id, + query, steps, result, error, knowledge_item_id, + started_at, finished_at, created_at, updated_at + FROM translation_agent_runs + WHERE {' AND '.join(clauses)} + ORDER BY updated_at DESC, started_at DESC + LIMIT :limit + """ + ), + params, + ).all() + out: list[dict[str, Any]] = [] + for row in rows: + m = row._mapping + result = m["result"] + steps = m["steps"] + if isinstance(result, str): + try: + result = json.loads(result) + except Exception: + result = {} + if isinstance(steps, str): + try: + steps = json.loads(steps) + except Exception: + steps = [] + out.append( + { + "id": str(m["id"]), + "agent_type": m["agent_type"], + "status": m["status"], + "term": m["term"], + "domain": m["domain"], + "target_lang": m["target_lang"], + "task_id": str(m["task_id"]) if m["task_id"] else None, + "subtitle_job_id": str(m["subtitle_job_id"]) if m["subtitle_job_id"] else None, + "query": m["query"], + "steps": steps if isinstance(steps, list) else [], + "result": result if isinstance(result, dict) else {}, + "error": m["error"], + "knowledge_item_id": str(m["knowledge_item_id"]) if m["knowledge_item_id"] else None, + "started_at": m["started_at"], + "finished_at": m["finished_at"], + "created_at": m["created_at"], + "updated_at": m["updated_at"], + } + ) + return out + + +def rebuild_knowledge_embeddings( + db: Session, + *, + rag_settings: RagSettings, + embedding_settings: EmbeddingSettings, + item_type: str | None = None, + status: str | None = None, + limit: int = 1000, +) -> dict[str, Any]: + clauses = ["1=1"] + params: dict[str, Any] = {"limit": max(1, min(10000, int(limit)))} + if item_type: + clauses.append("item_type = :item_type") + params["item_type"] = str(item_type).strip() + if status: + clauses.append("status = :status") + params["status"] = str(status).strip() + + rows = db.execute( + text( + f""" + SELECT id, item_type, term, translation, target_lang, domain, aliases, title, + content, description + FROM translation_knowledge_items + WHERE {' AND '.join(clauses)} + ORDER BY updated_at DESC, created_at DESC + LIMIT :limit + """ + ), + params, + ).all() + + model_key = embedding_model_key(rag_settings) + updated = 0 + failed = 0 + skipped = 0 + errors: list[dict[str, str]] = [] + for row in rows: + m = row._mapping + embedding_text = build_knowledge_embedding_text( + item_type=str(m["item_type"] or ""), + term=str(m["term"] or ""), + translation=str(m["translation"] or ""), + domain=str(m["domain"] or ""), + aliases=[str(x) for x in _json_list(m["aliases"]) if str(x or "").strip()], + title=str(m["title"] or ""), + content=str(m["content"] or ""), + description=str(m["description"] or ""), + ) + if not embedding_text.strip(): + skipped += 1 + continue + try: + embedding = embed_text(embedding_text, settings=embedding_settings) + assert_embedding_dimensions(embedding, rag_settings.embedding_dimensions) + with db.begin_nested(): + db.execute( + text( + """ + UPDATE translation_knowledge_items + SET embedding = CAST(:embedding AS vector), + embedding_model = :embedding_model, + embedding_text_hash = :embedding_text_hash, + last_verified_at = :last_verified_at, + updated_at = now() + WHERE id = CAST(:id AS uuid) + """ + ), + { + "id": str(m["id"]), + "embedding": _vector_literal(embedding), + "embedding_model": model_key, + "embedding_text_hash": _hash_text(embedding_text), + "last_verified_at": datetime.now(timezone.utc), + }, + ) + updated += 1 + except Exception as e: + failed += 1 + if len(errors) < 20: + errors.append({"id": str(m["id"]), "error": str(e)}) + return { + "total": len(rows), + "updated": updated, + "failed": failed, + "skipped": skipped, + "embedding_model": model_key, + "dimensions": rag_settings.embedding_dimensions, + "errors": errors, + } diff --git a/src/videoroll/apps/subtitle_service/schemas.py b/src/videoroll/apps/subtitle_service/schemas.py index b8bd9c3..f026700 100644 --- a/src/videoroll/apps/subtitle_service/schemas.py +++ b/src/videoroll/apps/subtitle_service/schemas.py @@ -202,6 +202,20 @@ class TranslateSettingsRead(BaseModel): openai_temperature: float openai_timeout_seconds: float + rag_enabled: bool = False + rag_top_k: int = 8 + rag_min_score: float = 0.68 + rag_embedding_provider: str = "openai" + rag_embedding_model: str = "text-embedding-3-small" + rag_embedding_dimensions: int = 1536 + rag_embedding_model_dir: str = "/models/embeddings" + rag_embedding_device: str = "cpu" + rag_auto_discover_terms: bool = False + rag_auto_learn_terms: bool = False + rag_search_enabled: bool = False + rag_search_url: str = "" + rag_domain: str = "" + class TranslateSettingsUpdate(BaseModel): default_provider: Optional[str] = None @@ -218,6 +232,20 @@ class TranslateSettingsUpdate(BaseModel): openai_temperature: Optional[float] = None openai_timeout_seconds: Optional[float] = None + rag_enabled: Optional[bool] = None + rag_top_k: Optional[int] = Field(default=None, ge=0, le=30) + rag_min_score: Optional[float] = Field(default=None, ge=0.0, le=1.0) + rag_embedding_provider: Optional[str] = None + rag_embedding_model: Optional[str] = None + rag_embedding_dimensions: Optional[int] = Field(default=None, ge=1, le=4096) + rag_embedding_model_dir: Optional[str] = None + rag_embedding_device: Optional[str] = None + rag_auto_discover_terms: Optional[bool] = None + rag_auto_learn_terms: Optional[bool] = None + rag_search_enabled: Optional[bool] = None + rag_search_url: Optional[str] = None + rag_domain: Optional[str] = None + class TranslateTestRequest(BaseModel): text: str = "Hello world." @@ -229,6 +257,119 @@ class TranslateTestResponse(BaseModel): translated_text: str +class KnowledgeItemRead(BaseModel): + id: uuid.UUID + item_type: str + term: str = "" + translation: str = "" + target_lang: str = "zh" + domain: str = "" + aliases: list[Any] = Field(default_factory=list) + title: str = "" + content: str = "" + description: str = "" + sources: list[Any] = Field(default_factory=list) + confidence: float = 0.0 + status: str = "approved" + created_by: str = "manual" + usage_count: int = 0 + embedding_model: str = "" + last_verified_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + +class KnowledgeItemUpsertRequest(BaseModel): + item_type: Literal["term", "document"] = "term" + target_lang: str = "zh" + term: str = "" + translation: str = "" + domain: str = "" + aliases: list[str] = Field(default_factory=list) + title: str = "" + content: str = "" + description: str = "" + sources: list[dict[str, Any]] = Field(default_factory=list) + confidence: float = Field(default=1.0, ge=0.0, le=1.0) + status: str = "approved" + created_by: str = "manual" + + +class KnowledgeItemUpsertResponse(BaseModel): + id: uuid.UUID + + +class KnowledgeEmbeddingRebuildRequest(BaseModel): + item_type: Optional[Literal["term", "document"]] = None + status: Optional[str] = None + limit: int = Field(default=1000, ge=1, le=10000) + + +class KnowledgeEmbeddingRebuildResponse(BaseModel): + total: int + updated: int + failed: int + skipped: int + embedding_model: str + dimensions: int + errors: list[dict[str, str]] = Field(default_factory=list) + + +class AgentRunRead(BaseModel): + id: uuid.UUID + agent_type: str + status: str + term: str = "" + domain: str = "" + target_lang: str = "zh" + task_id: Optional[uuid.UUID] = None + subtitle_job_id: Optional[uuid.UUID] = None + query: str = "" + steps: list[Any] = Field(default_factory=list) + result: dict[str, Any] = Field(default_factory=dict) + error: str = "" + knowledge_item_id: Optional[uuid.UUID] = None + started_at: datetime + finished_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + +class EmbeddingModelInfo(BaseModel): + name: str + path: str + size_bytes: Optional[int] = None + + +class EmbeddingModelListRequest(BaseModel): + model_dir: Optional[str] = None + + +class EmbeddingModelDownloadRequest(BaseModel): + model: str = "BAAI/bge-small-zh-v1.5" + name: Optional[str] = None + model_dir: Optional[str] = None + revision: Optional[str] = None + force: bool = False + + +class EmbeddingTestRequest(BaseModel): + text: str = "hello world" + provider: Optional[str] = None + model: Optional[str] = None + model_dir: Optional[str] = None + dimensions: Optional[int] = None + device: Optional[str] = None + + +class EmbeddingTestResponse(BaseModel): + provider: str + model: str + dimensions: int + expected_dimensions: int + ok: bool + + class WhisperModelInfo(BaseModel): name: str path: str diff --git a/src/videoroll/apps/subtitle_service/translate_settings_store.py b/src/videoroll/apps/subtitle_service/translate_settings_store.py index db435de..1132a48 100644 --- a/src/videoroll/apps/subtitle_service/translate_settings_store.py +++ b/src/videoroll/apps/subtitle_service/translate_settings_store.py @@ -56,6 +56,21 @@ def get_translate_settings(db: Session, defaults: SubtitleServiceSettings) -> di "openai_model": str(openai.get("model") or defaults.openai_model), "openai_temperature": float(openai.get("temperature") or defaults.openai_temperature), "openai_timeout_seconds": float(openai.get("timeout_seconds") or defaults.openai_timeout_seconds), + "rag_enabled": bool(stored.get("rag_enabled") if "rag_enabled" in stored else defaults.rag_enabled), + "rag_top_k": int(stored.get("rag_top_k") or defaults.rag_top_k), + "rag_min_score": float(stored.get("rag_min_score") or defaults.rag_min_score), + "rag_embedding_provider": str(stored.get("rag_embedding_provider") or defaults.rag_embedding_provider), + "rag_embedding_model": str(stored.get("rag_embedding_model") or defaults.rag_embedding_model), + "rag_embedding_dimensions": int(stored.get("rag_embedding_dimensions") or defaults.rag_embedding_dimensions), + "rag_embedding_model_dir": str(stored.get("rag_embedding_model_dir") or defaults.rag_embedding_model_dir), + "rag_embedding_device": str(stored.get("rag_embedding_device") or defaults.rag_embedding_device), + "rag_auto_discover_terms": bool( + stored.get("rag_auto_discover_terms") if "rag_auto_discover_terms" in stored else defaults.rag_auto_discover_terms + ), + "rag_auto_learn_terms": bool(stored.get("rag_auto_learn_terms") if "rag_auto_learn_terms" in stored else defaults.rag_auto_learn_terms), + "rag_search_enabled": bool(stored.get("rag_search_enabled") if "rag_search_enabled" in stored else defaults.rag_search_enabled), + "rag_search_url": str(stored.get("rag_search_url") or defaults.rag_search_url), + "rag_domain": str(stored.get("rag_domain") or defaults.rag_domain), } @@ -70,6 +85,19 @@ def update_translate_settings(db: Session, defaults: SubtitleServiceSettings, up "default_batch_size", "default_max_retries", "default_enable_summary", + "rag_enabled", + "rag_top_k", + "rag_min_score", + "rag_embedding_provider", + "rag_embedding_model", + "rag_embedding_dimensions", + "rag_embedding_model_dir", + "rag_embedding_device", + "rag_auto_discover_terms", + "rag_auto_learn_terms", + "rag_search_enabled", + "rag_search_url", + "rag_domain", ]: if key not in update: continue @@ -120,6 +148,26 @@ def update_translate_settings(db: Session, defaults: SubtitleServiceSettings, up stored["default_max_retries"] = max(0, min(10, int(stored["default_max_retries"]))) except Exception: stored["default_max_retries"] = defaults.translate_max_retries + try: + stored["rag_top_k"] = max(0, min(30, int(stored.get("rag_top_k") or defaults.rag_top_k))) + except Exception: + stored["rag_top_k"] = defaults.rag_top_k + try: + stored["rag_min_score"] = max(0.0, min(1.0, float(stored.get("rag_min_score") or defaults.rag_min_score))) + except Exception: + stored["rag_min_score"] = defaults.rag_min_score + try: + stored["rag_embedding_dimensions"] = max(1, min(4096, int(stored.get("rag_embedding_dimensions") or defaults.rag_embedding_dimensions))) + except Exception: + stored["rag_embedding_dimensions"] = defaults.rag_embedding_dimensions + for bool_key in ["rag_enabled", "rag_auto_discover_terms", "rag_auto_learn_terms", "rag_search_enabled"]: + if bool_key in stored: + stored[bool_key] = bool(stored[bool_key]) + provider = str(stored.get("rag_embedding_provider") or defaults.rag_embedding_provider).strip().lower() + stored["rag_embedding_provider"] = provider if provider in {"openai", "local"} else defaults.rag_embedding_provider + for str_key in ["rag_embedding_model", "rag_embedding_model_dir", "rag_embedding_device", "rag_search_url", "rag_domain"]: + if str_key in stored and stored[str_key] is not None: + stored[str_key] = str(stored[str_key]).strip() try: if "openai_temperature" in update and update["openai_temperature"] is not None: openai["temperature"] = float(openai.get("temperature")) diff --git a/src/videoroll/apps/subtitle_service/worker.py b/src/videoroll/apps/subtitle_service/worker.py index 51ccad1..41536ae 100644 --- a/src/videoroll/apps/subtitle_service/worker.py +++ b/src/videoroll/apps/subtitle_service/worker.py @@ -74,6 +74,8 @@ from videoroll.apps.subtitle_service.model_downloads import ( download_model_snapshot, resolve_model_repo_id, ) +from videoroll.apps.subtitle_service.rag import build_rag_context, rag_settings_from_translate_settings +from videoroll.apps.subtitle_service.embeddings import embedding_settings_from_translate_settings from videoroll.apps.subtitle_service.task_title_store import set_task_titles from videoroll.apps.subtitle_service.translate_settings_store import get_translate_settings from videoroll.apps.publish_meta_draft import apply_publish_source_overrides, default_publish_meta @@ -1358,6 +1360,44 @@ def process_job(self: Any, job_id: str) -> dict[str, str]: ) checkpoint_segments = list(resume_prefix) + rag_settings = rag_settings_from_translate_settings(translate_settings) + rag_chat_config = openai_chat_config_from_settings(translate_settings) + rag_embedding_settings = embedding_settings_from_translate_settings(translate_settings) + + if rag_settings.enabled: + _safe_append_log_line( + log_path, + "translate rag: " + f"enabled top_k={rag_settings.top_k} min_score={rag_settings.min_score} " + f"embedding_provider={rag_settings.embedding_provider} embedding_model={rag_settings.embedding_model} " + f"domain={rag_settings.domain or '(any)'}", + ) + + def _rag_context_provider(batch_segments: list[Segment], start_idx: int, summary: str) -> dict[str, Any] | None: + del start_idx, summary + if not rag_settings.enabled: + return None + ctx = build_rag_context( + db, + segments=batch_segments, + target_lang=target_lang, + rag_settings=rag_settings, + embedding_settings=rag_embedding_settings, + chat_config=rag_chat_config, + task_id=str(task.id), + subtitle_job_id=str(job.id), + ) + if ctx.hits: + try: + db.commit() + except Exception: + db.rollback() + if not ctx.term_cards and not ctx.knowledge_cards: + return None + return { + "term_cards": ctx.term_cards, + "knowledge_cards": ctx.knowledge_cards, + } def _on_translate_batch_done(batch_segments: list[Segment], updated_summary: str, completed_count: int) -> None: del completed_count @@ -1375,6 +1415,7 @@ def process_job(self: Any, job_id: str) -> dict[str, str]: timeout_seconds=translate_settings["openai_timeout_seconds"], batch_size=batch_size, enable_summary=enable_summary, + rag_context_provider=_rag_context_provider, resume_from=resume_prefix, initial_summary=resume_summary, on_batch_done=_on_translate_batch_done, diff --git a/src/videoroll/config.py b/src/videoroll/config.py index 2c58b78..87f656c 100644 --- a/src/videoroll/config.py +++ b/src/videoroll/config.py @@ -81,6 +81,20 @@ class SubtitleServiceSettings(CommonSettings): openai_temperature: float = Field(0.2, alias="OPENAI_TEMPERATURE") openai_timeout_seconds: float = Field(180.0, alias="OPENAI_TIMEOUT_SECONDS") + rag_enabled: bool = Field(False, alias="SUBTITLE_RAG_ENABLED") + rag_top_k: int = Field(8, alias="SUBTITLE_RAG_TOP_K") + rag_min_score: float = Field(0.68, alias="SUBTITLE_RAG_MIN_SCORE") + rag_embedding_provider: str = Field("openai", alias="SUBTITLE_RAG_EMBEDDING_PROVIDER") + rag_embedding_model: str = Field("text-embedding-3-small", alias="SUBTITLE_RAG_EMBEDDING_MODEL") + rag_embedding_dimensions: int = Field(1536, alias="SUBTITLE_RAG_EMBEDDING_DIMENSIONS") + rag_embedding_model_dir: str = Field("/models/embeddings", alias="SUBTITLE_RAG_EMBEDDING_MODEL_DIR") + rag_embedding_device: str = Field("cpu", alias="SUBTITLE_RAG_EMBEDDING_DEVICE") + rag_auto_discover_terms: bool = Field(False, alias="SUBTITLE_RAG_AUTO_DISCOVER_TERMS") + rag_auto_learn_terms: bool = Field(False, alias="SUBTITLE_RAG_AUTO_LEARN_TERMS") + rag_search_enabled: bool = Field(False, alias="SUBTITLE_RAG_SEARCH_ENABLED") + rag_search_url: str = Field("", alias="SUBTITLE_RAG_SEARCH_URL") + rag_domain: str = Field("", alias="SUBTITLE_RAG_DOMAIN") + # Orchestrator API (used by subtitle worker for auto pipelines). orchestrator_url: str = Field("http://localhost:8000", alias="ORCHESTRATOR_URL") orchestrator_timeout_seconds: float = Field(1800.0, alias="ORCHESTRATOR_TIMEOUT_SECONDS") diff --git a/src/videoroll/db/auto_migrate.py b/src/videoroll/db/auto_migrate.py index bf10abf..c61afe3 100644 --- a/src/videoroll/db/auto_migrate.py +++ b/src/videoroll/db/auto_migrate.py @@ -91,6 +91,175 @@ def _ensure_youtube_sources_columns(engine: Engine) -> None: logger.warning("auto-migrated DB: added youtube_sources.%s", column) +def _ensure_pgvector_rag_tables(engine: Engine) -> None: + dialect = (engine.dialect.name or "").lower() + if dialect != "postgresql": + logger.warning("RAG tables require PostgreSQL/pgvector; skipping for dialect=%s", dialect) + return + + with engine.connect() as conn: + vector_type = conn.execute(text("SELECT to_regtype('vector')")).scalar() + try: + conn.commit() + except Exception: + pass + if vector_type is None: + try: + with engine.begin() as ext_conn: + ext_conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + except Exception as e: + logger.warning("pgvector extension is unavailable; skipping RAG table migration: %s", e) + return + + with engine.begin() as conn: + conn.execute( + text( + f""" + CREATE TABLE IF NOT EXISTS translation_knowledge_items ( + id UUID PRIMARY KEY, + item_type VARCHAR(32) NOT NULL DEFAULT 'document', + term TEXT NOT NULL DEFAULT '', + normalized_term TEXT NOT NULL DEFAULT '', + translation TEXT NOT NULL DEFAULT '', + target_lang VARCHAR(16) NOT NULL DEFAULT 'zh', + domain TEXT NOT NULL DEFAULT '', + aliases JSONB NOT NULL DEFAULT '[]'::jsonb, + title TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + sources JSONB NOT NULL DEFAULT '[]'::jsonb, + confidence DOUBLE PRECISION NOT NULL DEFAULT 0, + status VARCHAR(32) NOT NULL DEFAULT 'approved', + created_by VARCHAR(32) NOT NULL DEFAULT 'manual', + usage_count INTEGER NOT NULL DEFAULT 0, + embedding vector, + embedding_model TEXT NOT NULL DEFAULT '', + embedding_text_hash VARCHAR(64) NOT NULL DEFAULT '', + last_verified_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + try: + conn.execute(text("ALTER TABLE translation_knowledge_items ALTER COLUMN embedding TYPE vector")) + except Exception: + pass + conn.execute( + text( + """ + CREATE TABLE IF NOT EXISTS translation_term_evidence ( + id UUID PRIMARY KEY, + term TEXT NOT NULL DEFAULT '', + normalized_term TEXT NOT NULL DEFAULT '', + domain TEXT NOT NULL DEFAULT '', + query TEXT NOT NULL DEFAULT '', + results JSONB NOT NULL DEFAULT '[]'::jsonb, + summary TEXT NOT NULL DEFAULT '', + sources JSONB NOT NULL DEFAULT '[]'::jsonb, + confidence DOUBLE PRECISION NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + conn.execute( + text( + """ + CREATE TABLE IF NOT EXISTS translation_term_matches ( + id UUID PRIMARY KEY, + task_id UUID, + subtitle_job_id UUID, + knowledge_item_id UUID REFERENCES translation_knowledge_items(id) ON DELETE SET NULL, + term TEXT NOT NULL DEFAULT '', + normalized_term TEXT NOT NULL DEFAULT '', + segment_start DOUBLE PRECISION, + segment_end DOUBLE PRECISION, + raw_context TEXT NOT NULL DEFAULT '', + decision TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + conn.execute( + text( + """ + CREATE TABLE IF NOT EXISTS translation_agent_runs ( + id UUID PRIMARY KEY, + agent_type VARCHAR(64) NOT NULL DEFAULT 'rag_term_research', + status VARCHAR(32) NOT NULL DEFAULT 'running', + term TEXT NOT NULL DEFAULT '', + normalized_term TEXT NOT NULL DEFAULT '', + domain TEXT NOT NULL DEFAULT '', + target_lang VARCHAR(16) NOT NULL DEFAULT 'zh', + task_id UUID, + subtitle_job_id UUID, + query TEXT NOT NULL DEFAULT '', + steps JSONB NOT NULL DEFAULT '[]'::jsonb, + result JSONB NOT NULL DEFAULT '{}'::jsonb, + error TEXT NOT NULL DEFAULT '', + knowledge_item_id UUID REFERENCES translation_knowledge_items(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + conn.execute( + text( + """ + CREATE UNIQUE INDEX IF NOT EXISTS uq_translation_terms_lang_domain_norm + ON translation_knowledge_items (target_lang, domain, normalized_term) + WHERE item_type = 'term' AND normalized_term <> '' + """ + ) + ) + conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_translation_agent_runs_status_updated + ON translation_agent_runs (status, updated_at DESC) + """ + ) + ) + conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_translation_agent_runs_term + ON translation_agent_runs (target_lang, domain, normalized_term) + """ + ) + ) + conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_translation_knowledge_type_status + ON translation_knowledge_items (item_type, status, target_lang) + """ + ) + ) + conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_translation_knowledge_domain_norm + ON translation_knowledge_items (domain, normalized_term) + """ + ) + ) + conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_translation_matches_task + ON translation_term_matches (task_id, subtitle_job_id) + """ + ) + ) + + def auto_migrate_engine(engine: Engine) -> None: """ Small, targeted "auto migration" for deployments without Alembic. @@ -100,6 +269,7 @@ def auto_migrate_engine(engine: Engine) -> None: """ _ensure_tasks_lock_columns(engine) _ensure_youtube_sources_columns(engine) + _ensure_pgvector_rag_tables(engine) @lru_cache @@ -111,5 +281,9 @@ def _auto_migrate_cached(database_url: str, pid: int) -> None: auto_migrate_engine(engine) -def auto_migrate(database_url: str) -> None: +def auto_migrate(database_url: str, *, force: bool = False) -> None: + if force: + engine = get_engine(database_url) + auto_migrate_engine(engine) + return _auto_migrate_cached(database_url, os.getpid()) diff --git a/src/videoroll/utils/intel_gpu.py b/src/videoroll/utils/intel_gpu.py index bb667f5..2eba29e 100644 --- a/src/videoroll/utils/intel_gpu.py +++ b/src/videoroll/utils/intel_gpu.py @@ -3,6 +3,7 @@ from __future__ import annotations import os import shutil import subprocess +import time from pathlib import Path from typing import Any @@ -36,6 +37,56 @@ def detect_intel_hardware(render_device: str) -> dict[str, Any]: return _probe_intel_sysfs(sysfs_dir, device_path) +def sample_intel_gpu_usage(render_device: str, *, interval_seconds: float = 0.1) -> dict[str, Any]: + hardware = detect_intel_hardware(render_device) + out: dict[str, Any] = { + **hardware, + "usage_supported": False, + "usage_percent": None, + "engines": [], + } + if not bool(hardware.get("available")): + return out + + sysfs_dir = _render_sysfs_device_dir(Path(str(render_device or "").strip() or "/dev/dri/renderD128")) + if sysfs_dir is None: + out["detail"] = "未找到渲染设备的 sysfs 信息" + return out + + first = _read_engine_busy_snapshot(sysfs_dir) + if not first: + out["detail"] = f"{hardware.get('detail') or '已检测到 Intel 硬件'}；未找到 GPU busy 计数器" + return out + + interval = max(0.02, float(interval_seconds)) + time.sleep(interval) + second = _read_engine_busy_snapshot(sysfs_dir) + if not second: + out["detail"] = f"{hardware.get('detail') or '已检测到 Intel 硬件'}；GPU busy 计数器不可读" + return out + + elapsed_us = interval * 1_000_000.0 + engines: list[dict[str, Any]] = [] + max_percent = 0.0 + for name, start_busy in first.items(): + end_busy = second.get(name) + if end_busy is None: + continue + delta = max(0, end_busy - start_busy) + percent = max(0.0, min(100.0, (delta / elapsed_us) * 100.0)) + max_percent = max(max_percent, percent) + engines.append({"name": name, "percent": percent}) + + if not engines: + out["detail"] = f"{hardware.get('detail') or '已检测到 Intel 硬件'}；GPU busy 计数器无有效数据" + return out + + out["usage_supported"] = True + out["usage_percent"] = max_percent + out["engines"] = sorted(engines, key=lambda item: str(item.get("name") or "")) + return out + + def _probe_intel_sysfs(sysfs_dir: Path, render_device: str) -> dict[str, Any]: result: dict[str, Any] = { "checked": True, @@ -72,6 +123,33 @@ def _probe_intel_sysfs(sysfs_dir: Path, render_device: str) -> dict[str, Any]: return result +def _read_engine_busy_snapshot(sysfs_dir: Path) -> dict[str, int]: + engine_root = sysfs_dir / "engine" + out: dict[str, int] = {} + if engine_root.exists(): + for busy_path in engine_root.glob("*/busy"): + raw = _read_text(busy_path) + try: + value = int(str(raw or "").strip()) + except Exception: + continue + name = busy_path.parent.name.strip() or "engine" + out[name] = value + if out: + return out + + # Some kernels expose aggregate busy counters under the device directory. + for name in ["busy", "gt/gt0/rps_busy", "gt/gt0/busy"]: + path = sysfs_dir / name + raw = _read_text(path) + try: + value = int(str(raw or "").strip()) + except Exception: + continue + out[name.replace("/", ".")] = value + return out + + def _render_sysfs_device_dir(node: Path) -> Path | None: name = node.name if name: diff --git a/src/videoroll/utils/openai_compat.py b/src/videoroll/utils/openai_compat.py index 604251d..4747a81 100644 --- a/src/videoroll/utils/openai_compat.py +++ b/src/videoroll/utils/openai_compat.py @@ -55,3 +55,20 @@ def build_openai_chat_completions_url(base_url: str) -> str: base = normalize_openai_base_url(raw) return base.rstrip("/") + "/chat/completions" + +def build_openai_embeddings_url(base_url: str) -> str: + """ + Accepts either: + - a base URL (e.g. https://api.openai.com/v1), or + - a full endpoint URL ending with /embeddings. + """ + raw = (base_url or "").strip() + if not raw: + raw = DEFAULT_OPENAI_BASE_URL + raw = _ensure_scheme(raw).rstrip("/") + + if raw.endswith("/embeddings"): + return raw + + base = normalize_openai_base_url(raw) + return base.rstrip("/") + "/embeddings" diff --git a/src/videoroll/utils/resources.py b/src/videoroll/utils/resources.py new file mode 100644 index 0000000..e637ed9 --- /dev/null +++ b/src/videoroll/utils/resources.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import os +import time +from pathlib import Path +from typing import Any + +from videoroll.utils.cpu import process_cpu_count + + +def _read_text(path: Path) -> str | None: + try: + return path.read_text(encoding="utf-8").strip() + except Exception: + return None + + +def read_cpu_snapshot() -> tuple[int, int] | None: + raw = _read_text(Path("/proc/stat")) + if not raw: + return None + first = raw.splitlines()[0].split() + if len(first) < 5 or first[0] != "cpu": + return None + try: + values = [int(x) for x in first[1:]] + except Exception: + return None + idle = values[3] + (values[4] if len(values) > 4 else 0) + return sum(values), idle + + +def cpu_percent_between(prev: tuple[int, int], curr: tuple[int, int]) -> float | None: + total_delta = curr[0] - prev[0] + idle_delta = curr[1] - prev[1] + if total_delta <= 0: + return None + busy = max(0, total_delta - max(0, idle_delta)) + return max(0.0, min(100.0, (busy / total_delta) * 100.0)) + + +def sample_cpu_percent(*, interval_seconds: float = 0.05) -> float | None: + prev = read_cpu_snapshot() + if prev is None: + return None + time.sleep(max(0.0, float(interval_seconds))) + curr = read_cpu_snapshot() + if curr is None: + return None + return cpu_percent_between(prev, curr) + + +def read_load_average() -> list[float] | None: + try: + return [float(x) for x in os.getloadavg()] + except Exception: + return None + + +def _parse_meminfo(raw: str) -> dict[str, int]: + out: dict[str, int] = {} + for line in raw.splitlines(): + if ":" not in line: + continue + key, rest = line.split(":", 1) + parts = rest.strip().split() + if not parts: + continue + try: + value = int(parts[0]) + except Exception: + continue + unit = parts[1].lower() if len(parts) > 1 else "" + out[key] = value * 1024 if unit == "kb" else value + return out + + +def memory_stats_from_meminfo(raw: str) -> dict[str, int | float] | None: + data = _parse_meminfo(raw) + total = int(data.get("MemTotal") or 0) + if total <= 0: + return None + available = int(data.get("MemAvailable") or 0) + if available <= 0: + available = int(data.get("MemFree") or 0) + int(data.get("Buffers") or 0) + int(data.get("Cached") or 0) + available = max(0, min(total, available)) + used = max(0, total - available) + return { + "total_bytes": total, + "used_bytes": used, + "available_bytes": available, + "percent": (used / total) * 100.0, + } + + +def read_memory_stats() -> dict[str, int | float] | None: + raw = _read_text(Path("/proc/meminfo")) + if not raw: + return None + return memory_stats_from_meminfo(raw) + + +def read_cgroup_memory_stats() -> dict[str, int | float] | None: + # cgroup v2 + current = _read_int(Path("/sys/fs/cgroup/memory.current")) + max_raw = _read_text(Path("/sys/fs/cgroup/memory.max")) + total = None if max_raw in {None, "", "max"} else _parse_positive_int(max_raw) + if current is not None and total is not None and total > 0: + return _memory_stats_from_usage(current, total) + + # cgroup v1 + current = _read_int(Path("/sys/fs/cgroup/memory/memory.usage_in_bytes")) + total = _read_int(Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")) + if current is not None and total is not None and 0 < total < 1 << 60: + return _memory_stats_from_usage(current, total) + return None + + +def _read_int(path: Path) -> int | None: + return _parse_positive_int(_read_text(path)) + + +def _parse_positive_int(raw: str | None) -> int | None: + try: + n = int(str(raw or "").strip()) + except Exception: + return None + return n if n >= 0 else None + + +def _memory_stats_from_usage(used: int, total: int) -> dict[str, int | float]: + used = max(0, min(total, int(used))) + available = max(0, total - used) + return { + "total_bytes": total, + "used_bytes": used, + "available_bytes": available, + "percent": (used / total) * 100.0 if total > 0 else 0.0, + } + + +def process_cpu_summary() -> dict[str, Any]: + return { + "percent": sample_cpu_percent(), + "cores": process_cpu_count(), + "load_average": read_load_average(), + } diff --git a/src/web/index.html b/src/web/index.html index 079ad89..5b7d014 100644 --- a/src/web/index.html +++ b/src/web/index.html @@ -3,11 +3,20 @@ + VideoRoll +
- diff --git a/src/web/src/App.tsx b/src/web/src/App.tsx index cee039b..a212aa8 100644 --- a/src/web/src/App.tsx +++ b/src/web/src/App.tsx @@ -1,6 +1,7 @@ -import { useState } from "react"; +import { ReactNode, useEffect, useState } from "react"; import { Link, NavLink, Route, Routes } from "react-router-dom"; import AuthGate from "./components/AuthGate"; +import { FeedbackProvider } from "./components/Feedback"; import { fetchJson } from "./lib/http"; import { ORCHESTRATOR_URL } from "./lib/urls"; import DashboardPage from "./pages/DashboardPage"; @@ -25,8 +26,10 @@ function NavItem({ to, label }: { to: string; label: string }) { to={to} className={({ isActive }) => [ - "block rounded px-3 py-2 text-sm", - isActive ? "bg-slate-900 text-white" : "text-slate-700 hover:bg-slate-100", + "block rounded-md px-3 py-2 text-sm", + isActive + ? "bg-slate-900 text-white dark:bg-slate-100 dark:text-slate-950" + : "text-slate-700 hover:bg-slate-100 dark:text-slate-300 dark:hover:bg-slate-800", ].join(" ") } > @@ -35,6 +38,15 @@ function NavItem({ to, label }: { to: string; label: string }) { ); } +function NavGroup({ title, children }: { title: string; children: ReactNode }) { + return ( +
+
{title}
+
{children}
+
+ ); +} + export default function App() { const orchestratorDisplay = ORCHESTRATOR_URL.startsWith("http://") || ORCHESTRATOR_URL.startsWith("https://") @@ -44,6 +56,19 @@ export default function App() { : ORCHESTRATOR_URL; const [loggingOut, setLoggingOut] = useState(false); + const [darkMode, setDarkMode] = useState(() => { + if (typeof window === "undefined") return false; + const stored = window.localStorage.getItem("videoroll-theme"); + if (stored === "dark") return true; + if (stored === "light") return false; + return window.matchMedia?.("(prefers-color-scheme: dark)").matches ?? false; + }); + + useEffect(() => { + const root = document.documentElement; + root.classList.toggle("dark", darkMode); + window.localStorage.setItem("videoroll-theme", darkMode ? "dark" : "light"); + }, [darkMode]); async function logout() { if (loggingOut) return; @@ -60,22 +85,32 @@ export default function App() { } return ( - -
-
+ + +
+
VideoRoll
合规版 · MVP
+
- + + ); } diff --git a/src/web/src/components/Feedback.tsx b/src/web/src/components/Feedback.tsx new file mode 100644 index 0000000..fd9bec8 --- /dev/null +++ b/src/web/src/components/Feedback.tsx @@ -0,0 +1,104 @@ +import { PropsWithChildren, ReactNode, useCallback, useMemo, useState } from "react"; +import { ConfirmOptions, FeedbackContext, Toast, ToastKind } from "./feedbackContext"; + +type ConfirmState = ConfirmOptions & { + resolve: (value: boolean) => void; +}; + +const toastClasses: Record = { + success: "border-emerald-200 bg-emerald-50 text-emerald-900", + error: "border-rose-200 bg-rose-50 text-rose-900", + info: "border-sky-200 bg-sky-50 text-sky-900", + warning: "border-amber-200 bg-amber-50 text-amber-950", +}; + +function clampToastText(value: ReactNode): ReactNode { + if (typeof value !== "string") return value; + const text = value.trim(); + return text.length > 700 ? `${text.slice(0, 699)}...` : text; +} + +export function FeedbackProvider({ children }: PropsWithChildren) { + const [toasts, setToasts] = useState([]); + const [confirmState, setConfirmState] = useState(null); + + const dismissToast = useCallback((id: number) => { + setToasts((items) => items.filter((item) => item.id !== id)); + }, []); + + const toast = useCallback( + (next: Omit) => { + const id = Date.now() + Math.floor(Math.random() * 10000); + setToasts((items) => [...items, { ...next, id }].slice(-4)); + window.setTimeout(() => dismissToast(id), next.kind === "error" ? 9000 : 5000); + }, + [dismissToast], + ); + + const confirm = useCallback((options: ConfirmOptions) => { + return new Promise((resolve) => { + setConfirmState({ ...options, resolve }); + }); + }, []); + + const value = useMemo(() => ({ toast, confirm }), [toast, confirm]); + + function closeConfirm(result: boolean) { + const current = confirmState; + if (!current) return; + setConfirmState(null); + current.resolve(result); + } + + const confirmTone = confirmState?.tone ?? "default"; + const confirmButtonClass = + confirmTone === "danger" + ? "bg-rose-700 text-white hover:bg-rose-800" + : confirmTone === "warning" + ? "bg-amber-600 text-white hover:bg-amber-700" + : "bg-slate-900 text-white hover:bg-slate-800"; + + return ( + + {children} + +
+ {toasts.map((item) => ( +
+
+
+
{item.title}
+ {item.message ?
{clampToastText(item.message)}
: null} +
+ +
+
+ ))} +
+ + {confirmState ? ( +
+
+
{confirmState.title}
+ {confirmState.message ?
{confirmState.message}
: null} +
+ + +
+
+
+ ) : null} + + ); +} diff --git a/src/web/src/components/feedbackContext.ts b/src/web/src/components/feedbackContext.ts new file mode 100644 index 0000000..5d5d3fc --- /dev/null +++ b/src/web/src/components/feedbackContext.ts @@ -0,0 +1,37 @@ +import { createContext, ReactNode, useContext } from "react"; + +export type ToastKind = "success" | "error" | "info" | "warning"; + +export type Toast = { + id: number; + kind: ToastKind; + title: string; + message?: ReactNode; +}; + +export type ConfirmOptions = { + title: string; + message?: ReactNode; + confirmLabel?: string; + cancelLabel?: string; + tone?: "default" | "danger" | "warning"; +}; + +export type FeedbackContextValue = { + toast: (toast: Omit) => void; + confirm: (options: ConfirmOptions) => Promise; +}; + +export const FeedbackContext = createContext(null); + +export function useToast() { + const ctx = useContext(FeedbackContext); + if (!ctx) throw new Error("useToast must be used inside FeedbackProvider"); + return ctx.toast; +} + +export function useConfirm() { + const ctx = useContext(FeedbackContext); + if (!ctx) throw new Error("useConfirm must be used inside FeedbackProvider"); + return ctx.confirm; +} diff --git a/src/web/src/components/ui.tsx b/src/web/src/components/ui.tsx new file mode 100644 index 0000000..6d18226 --- /dev/null +++ b/src/web/src/components/ui.tsx @@ -0,0 +1,74 @@ +import { ButtonHTMLAttributes, PropsWithChildren, ReactNode } from "react"; + +type ButtonTone = "primary" | "secondary" | "danger" | "warning" | "ghost"; +type ButtonSize = "sm" | "xs"; + +const toneClasses: Record = { + primary: "border-slate-900 bg-slate-900 text-white hover:bg-slate-800", + secondary: "border-slate-300 bg-white text-slate-800 hover:bg-slate-50", + danger: "border-rose-300 bg-white text-rose-700 hover:bg-rose-50", + warning: "border-amber-300 bg-white text-amber-800 hover:bg-amber-50", + ghost: "border-transparent bg-transparent text-slate-700 hover:bg-slate-100", +}; + +const sizeClasses: Record = { + sm: "px-3 py-2 text-sm", + xs: "px-2 py-1 text-xs", +}; + +type ButtonProps = ButtonHTMLAttributes & { + tone?: ButtonTone; + size?: ButtonSize; +}; + +export function Button({ tone = "secondary", size = "sm", className = "", ...props }: ButtonProps) { + return ( + +
+
+
+
+
概览
+
+
status: {run.status}
+
type: {run.agent_type}
+
domain: {run.domain || "-"}
+
target: {run.target_lang}
+
query: {run.query || "-"}
+
started: {formatDate(run.started_at)}
+
updated: {formatDate(run.updated_at)}
+ {run.finished_at ?
finished: {formatDate(run.finished_at)}
: null} +
+
+
+
结果
+ {translation ? ( +
+
{translation}
+ {confidence !== null ?
confidence: {(confidence * 100).toFixed(0)}%
: null} + {knowledgeStatus ?
knowledge: {knowledgeStatus}
: null} + {failureCategory ?
failure: {failureCategory}
: null} + {run.knowledge_item_id ? ( + + item: {run.knowledge_item_id} + + ) : null} +
+ ) : ( +
暂无结果。
+ )} + {run.error ?
{run.error}
: null} +
+ {Object.keys(run.result || {}).length ? ( +
+
Result JSON
+ 
{prettyJson(run.result)}
+
+ ) : null} +
+
+
对话流
+
+ {run.steps.length === 0 ?
暂无步骤记录。
: null} + {run.steps.map((step, index) => )} +
+
+
+
+
+ ); +} + +function AgentRunCard({ run, onSelect }: { run: AgentRun; onSelect: (run: AgentRun) => void }) { + const translation = textValue(run.result?.translation); + const confidence = numberValue(run.result?.confidence); + const knowledgeStatus = textValue(run.result?.knowledge_status); + const failureCategory = textValue(run.result?.failure_category); + const opened = run.steps.filter((step) => step.action === "open_url" || step.action === "read_url").length; + const latestStep = run.steps[run.steps.length - 1]; + return ( + + ); +} + +function ResourceBar({ label, value, detail, tone = "sky" }: { label: string; value?: number | null; detail?: string; tone?: "sky" | "emerald" | "amber" }) { + const percent = clampPercent(value); + const color = + tone === "emerald" + ? "bg-emerald-500" + : tone === "amber" + ? "bg-amber-500" + : "bg-sky-500"; + return ( +
+
+ {label} + {formatPercent(value)} +
+
+
+
+ {detail ?
{detail}
: null} +
+ ); +} + +const runningStatuses = new Set(["INGESTED", "DOWNLOADED", "AUDIO_EXTRACTED", "ASR_DONE", "TRANSLATED", "SUBTITLE_READY", "RENDERED", "READY_FOR_REVIEW", "APPROVED", "PUBLISHING"]); + export default function DashboardPage() { const [tasks, setTasks] = useState(null); const [error, setError] = useState(null); const [videos, setVideos] = useState(null); const [videosError, setVideosError] = useState(null); + const [resources, setResources] = useState(null); + const [resourcesError, setResourcesError] = useState(null); + const [agentRuns, setAgentRuns] = useState(null); + const [agentRunsError, setAgentRunsError] = useState(null); + const [selectedAgentId, setSelectedAgentId] = useState(null); useEffect(() => { fetchJson(`${ORCHESTRATOR_URL}/tasks?limit=200`) @@ -34,43 +330,263 @@ export default function DashboardPage() { .catch((e: unknown) => setVideosError(e instanceof Error ? e.message : String(e))); }, []); + useEffect(() => { + let cancelled = false; + let timer: number | undefined; + const load = async () => { + try { + const data = await fetchJson(`${ORCHESTRATOR_URL}/system/resources`); + if (cancelled) return; + setResources(data); + setResourcesError(null); + } catch (e: unknown) { + if (cancelled) return; + setResourcesError(e instanceof Error ? e.message : String(e)); + } finally { + if (!cancelled) timer = window.setTimeout(load, 3000); + } + }; + load(); + return () => { + cancelled = true; + if (timer) window.clearTimeout(timer); + }; + }, []); + + useEffect(() => { + let cancelled = false; + let timer: number | undefined; + const load = async () => { + try { + const data = await fetchJson(`${SUBTITLE_SERVICE_URL}/subtitle/agents/runs?limit=12`); + if (cancelled) return; + setAgentRuns(data); + setAgentRunsError(null); + } catch (e: unknown) { + if (cancelled) return; + setAgentRunsError(e instanceof Error ? e.message : String(e)); + } finally { + if (!cancelled) timer = window.setTimeout(load, 5000); + } + }; + load(); + return () => { + cancelled = true; + if (timer) window.clearTimeout(timer); + }; + }, []); + const counts = useMemo(() => { const out = new Map(); for (const t of tasks ?? []) out.set(t.status, (out.get(t.status) ?? 0) + 1); return Array.from(out.entries()).sort((a, b) => b[1] - a[1]); }, [tasks]); + const total = tasks?.length ?? 0; + const failedTasks = useMemo(() => (tasks ?? []).filter((t) => t.status === "FAILED").slice(0, 6), [tasks]); + const runningTasks = useMemo(() => (tasks ?? []).filter((t) => runningStatuses.has(t.status)).slice(0, 6), [tasks]); + const runningAgents = useMemo(() => (agentRuns ?? []).filter((run) => run.status === "running"), [agentRuns]); + const recentFinishedAgents = useMemo(() => (agentRuns ?? []).filter((run) => run.status !== "running").slice(0, 6), [agentRuns]); + const selectedAgent = useMemo(() => (agentRuns ?? []).find((run) => run.id === selectedAgentId) ?? null, [agentRuns, selectedAgentId]); + const publishedCount = counts.find(([status]) => status === "PUBLISHED")?.[1] ?? 0; + const failedCount = counts.find(([status]) => status === "FAILED")?.[1] ?? 0; + const runningCount = (tasks ?? []).filter((t) => runningStatuses.has(t.status)).length; + return (
-
-
-
-
Dashboard
-
快速查看任务状态与入口
-
- + + + 查看失败 + + 新建任务 - -
-
+ + + } + /> -
-
最近任务（200 条内）状态统计
+
+
+
最近任务（200 条内）
+ {tasks ?
共 {total} 条
: null} +
{error ?
{error}
: null} {!tasks ?
加载中…
: null} {tasks ? ( -
+ <> +
+ +
全部任务
+
{total}
+ + +
运行中
+
{runningCount}
+ + +
失败
+
{failedCount}
+ + +
已发布
+
{publishedCount}
+ +
+
{counts.map(([status, n]) => ( -
+
{status}
-
{n}
-
+
{n}
+ ))} +
+ + ) : null} +
+ +
+
+
+
资源监控
+
每 3 秒刷新一次，数值来自当前运行后端的容器/主机视角。
+
+ {resources?.sampled_at ?
{new Date(resources.sampled_at).toLocaleTimeString()}
: null} +
+ {resourcesError ?
{resourcesError}
: null} + {!resources ?
加载中…
: null} + {resources ? ( +
+
+ n.toFixed(2)).join(" / ")}` : undefined} + /> +
+
+ + {resources.cgroup_memory ? ( +
host: {formatBytes(resources.memory.used_bytes)} / {formatBytes(resources.memory.total_bytes)}
+ ) : null} +
+ {resources.intel_gpu?.enabled ? ( +
+ + {resources.intel_gpu.engines?.length ? ( +
+ {resources.intel_gpu.engines.slice(0, 4).map((engine) => ( +
+ {engine.name} + {formatPercent(engine.percent)} +
+ ))} +
+ ) : null} +
+ ) : null}
) : null}
-
+
+
+
+
RAG Agent
+
显示术语发现、搜索、网页读取、总结入库的运行记录。
+
+ {agentRuns ?
运行中 {runningAgents.length}
: null} +
+ {agentRunsError ?
{agentRunsError}
: null} + {!agentRuns ?
加载中…
: null} + {agentRuns ? ( +
+
+
正在工作
+
+ {runningAgents.length === 0 ?
暂无运行中的 agent。
: null} + {runningAgents.map((run) => setSelectedAgentId(item.id)} />)} +
+
+
+
最近结果
+
+ {recentFinishedAgents.length === 0 ?
暂无 agent 结果。
: null} + {recentFinishedAgents.map((run) => setSelectedAgentId(item.id)} />)} +
+
+
+ ) : null} +
+ +
+
+
+
正在处理
+ + 队列 → + +
+
+ {!tasks ?
加载中…
: null} + {tasks && runningTasks.length === 0 ?
暂无运行中任务。
: null} + {runningTasks.map((t) => ( + +
+
+
{t.display_title?.trim() || t.source_url || t.id}
+
{t.id.slice(0, 8)} · {formatDate(t.updated_at)}
+
+ +
+ + ))} +
+
+ +
+
+
最近失败
+ + 全部失败 → + +
+
+ {!tasks ?
加载中…
: null} + {tasks && failedTasks.length === 0 ?
暂无失败任务。
: null} + {failedTasks.map((t) => ( + +
+
+
{t.display_title?.trim() || t.source_url || t.id}
+
{t.error_message || t.error_code || "FAILED"}
+
+ +
+ + ))} +
+
+
+ +
已转换视频（video_final）
@@ -81,7 +597,7 @@ export default function DashboardPage() { {videosError ?
{videosError}
: null} {!videos ?
加载中…
: null} {videos ? ( -
+
@@ -108,12 +624,12 @@ export default function DashboardPage() { + + + + + + + + + {embeddingModels.map((model) => ( + + + + + + + ))} + + + )} + -
- 提示：API Key 会加密存储在 DB 中，解密密钥保存在本地 `data/secrets/fernet.key`（容器内 `/secrets/fernet.key`）。 -
+
+ +
-
-
测试翻译
+
+
知识库导入
+
+ + + {knowledgeType === "term" ? ( + <> + + + + ) : ( + + )} +
Download - + Detail
@@ -132,6 +648,7 @@ export default function DashboardPage() { ) : null} + {selectedAgent ? setSelectedAgentId(null)} /> : null} ); } diff --git a/src/web/src/pages/RenderQueuePage.tsx b/src/web/src/pages/RenderQueuePage.tsx index 4b24bbe..c32cae1 100644 --- a/src/web/src/pages/RenderQueuePage.tsx +++ b/src/web/src/pages/RenderQueuePage.tsx @@ -50,7 +50,7 @@ export default function RenderQueuePage() { useEffect(() => { refresh(); - }, []); + }, [refresh]); const shouldPoll = useMemo(() => (queue?.tasks ?? []).some((t) => t.state === "queued" || t.state === "running"), [queue]); diff --git a/src/web/src/pages/SettingsASRPage.tsx b/src/web/src/pages/SettingsASRPage.tsx index 495fa0d..0756614 100644 --- a/src/web/src/pages/SettingsASRPage.tsx +++ b/src/web/src/pages/SettingsASRPage.tsx @@ -1,4 +1,5 @@ import { useEffect, useMemo, useState } from "react"; +import { useConfirm } from "../components/feedbackContext"; import { fetchJson } from "../lib/http"; import { SUBTITLE_SERVICE_URL } from "../lib/urls"; @@ -58,6 +59,7 @@ function formatBytes(n?: number | null): string { } export default function SettingsASRPage() { + const confirm = useConfirm(); const [settings, setSettings] = useState(null); const [asrDefaults, setAsrDefaults] = useState(null); const [models, setModels] = useState(null); @@ -417,7 +419,13 @@ export default function SettingsASRPage() { disabled={busy} className="rounded border border-rose-300 px-2 py-1 text-xs text-rose-700 hover:bg-rose-50 disabled:opacity-50" onClick={async () => { - if (!confirm(`确定删除模型：${m.name} ?`)) return; + const ok = await confirm({ + title: "删除模型", + message: `确定删除模型：${m.name} ?`, + confirmLabel: "删除", + tone: "danger", + }); + if (!ok) return; setBusy(true); setError(null); try { diff --git a/src/web/src/pages/SettingsApiPage.tsx b/src/web/src/pages/SettingsApiPage.tsx index a1124b5..f4076aa 100644 --- a/src/web/src/pages/SettingsApiPage.tsx +++ b/src/web/src/pages/SettingsApiPage.tsx @@ -1,4 +1,5 @@ import { useEffect, useMemo, useState } from "react"; +import { useConfirm } from "../components/feedbackContext"; import { fetchJson } from "../lib/http"; import { ORCHESTRATOR_URL } from "../lib/urls"; @@ -30,6 +31,7 @@ function orchestratorBaseUrl(): string { } export default function SettingsApiPage() { + const confirm = useConfirm(); const [settings, setSettings] = useState(null); const [tokenInput, setTokenInput] = useState(""); const [error, setError] = useState(null); @@ -148,7 +150,13 @@ export default function SettingsApiPage() { disabled={busy || !settings?.token_set} className="rounded border px-3 py-2 text-sm hover:bg-slate-50 disabled:opacity-50" onClick={async () => { - if (!window.confirm("确认清空远程管理 token？清空后外部将无法再调用该接口。")) return; + const ok = await confirm({ + title: "清空远程管理 token", + message: "清空后外部将无法再调用该接口。", + confirmLabel: "清空", + tone: "danger", + }); + if (!ok) return; setBusy(true); setError(null); try { diff --git a/src/web/src/pages/SettingsAutoPage.tsx b/src/web/src/pages/SettingsAutoPage.tsx index f6f4d49..effc407 100644 --- a/src/web/src/pages/SettingsAutoPage.tsx +++ b/src/web/src/pages/SettingsAutoPage.tsx @@ -1,5 +1,6 @@ import { Link } from "react-router-dom"; import { useEffect, useMemo, useState } from "react"; +import { useConfirm } from "../components/feedbackContext"; import { fetchJson } from "../lib/http"; import { SUBTITLE_SERVICE_URL } from "../lib/urls"; @@ -57,6 +58,7 @@ function normalizeYouTubeSubtitleMode(value: unknown, legacyPrefer?: boolean | n } export default function SettingsAutoPage() { + const confirm = useConfirm(); const [error, setError] = useState(null); const [busy, setBusy] = useState(false); @@ -605,8 +607,14 @@ export default function SettingsAutoPage() { } /> + {error ?
{error}
: null} -
+
-
当前配置（保存于后端配置/数据库）
- -
- - {!settings ? ( -
加载中…
- ) : ( -
-
-
default_provider
-
{settings.default_provider}
-
-
-
default_target_lang
-
{settings.default_target_lang}
-
-
-
default_style
-
{settings.default_style}
-
-
-
default_batch_size
-
{settings.default_batch_size}
-
-
-
default_max_retries
-
{settings.default_max_retries}
-
-
-
default_enable_summary
-
{settings.default_enable_summary ? "true" : "false"}
-
-
- )} -
提示：默认值用于前端初始选择；任务里仍可按需覆盖。
-
- -
-
-
保存配置
+
当前配置
OpenAI API Key：{settings?.openai_api_key_set ? 已设置 : 未设置}
+ {!settings ? ( +
加载中...
+ ) : ( +
+ {[ + ["provider", settings.default_provider], + ["target", settings.default_target_lang], + ["batch", settings.default_batch_size], + ["summary", settings.default_enable_summary ? "true" : "false"], + ["rag", settings.rag_enabled ? "enabled" : "disabled"], + ["embedding", `${settings.rag_embedding_provider}:${settings.rag_embedding_model}`], + ].map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+ )} + +
+
翻译与模型
- - - - - -
OpenAI（标准接口）
- - - - -
+
+
+
RAG / pgvector
+
+ + + + + + + + + + + + + +
- + + {embeddingTestResult ?
{embeddingTestResult}
: null} + {embeddingRebuildResult ?
{embeddingRebuildResult}
: null} +
+
+ +
+
本地 Embedding 模型
+
+ + +
+
+ + - - + 使用该模型 +
+ {embeddingModels.length === 0 ? ( + 暂无本地 embedding 模型 + ) : ( + +
NamePathSizeAction
{model.name}{model.path}{model.size_bytes ? `${Math.round(model.size_bytes / 1024 / 1024)} MB` : "-"} + +