From 910b2f83344f2d799ff4a4fc1fbbe6b8d653022a Mon Sep 17 00:00:00 2001 From: inliver233 Date: Mon, 23 Feb 2026 06:54:02 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=A1=A5=E5=85=A8=E5=B0=8F?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/public/images.py | 48 +++++++------ backend/app/api/public/random.py | 72 +++++++++++-------- backend/app/core/pixiv_urls.py | 16 +++-- backend/app/db/session.py | 36 ++++++++-- backend/app/jobs/handlers/hydrate_metadata.py | 10 ++- backend/app/main.py | 5 ++ 6 files changed, 122 insertions(+), 65 deletions(-) diff --git a/backend/app/api/public/images.py b/backend/app/api/public/images.py index 074bf22..111bd5b 100644 --- a/backend/app/api/public/images.py +++ b/backend/app/api/public/images.py @@ -4,7 +4,7 @@ import asyncio from datetime import datetime, timezone from typing import Any -from fastapi import APIRouter, Query, Request +from fastapi import APIRouter, BackgroundTasks, Query, Request from fastapi.responses import JSONResponse from app.core.errors import ApiError, ErrorCode @@ -250,6 +250,7 @@ async def proxy_image( request: Request, image_id: int, ext: str, + background_tasks: BackgroundTasks, pixiv_cat: int = 0, pximg_mirror_host: str | None = None, ): @@ -305,15 +306,9 @@ async def proxy_image( use_pixiv_cat = bool(runtime.image_proxy_use_pixiv_cat) or int(pixiv_cat) == 1 mirror_host = mirror_host_override or str(getattr(runtime, "image_proxy_pximg_mirror_host", "") or "").strip() or "i.pixiv.cat" - def _spawn_best_effort(coro) -> None: - async def _run() -> None: - try: - await coro - except Exception: - pass - + async def _best_effort(fn, *args, timeout_s: float = 1.5, **kwargs) -> None: # type: ignore[no-untyped-def] try: - asyncio.create_task(_run()) + await asyncio.wait_for(fn(*args, **kwargs), timeout=float(timeout_s)) except Exception: pass @@ -340,12 +335,21 @@ async def proxy_image( range_header=request.headers.get("Range"), ) if should_mark_ok: - _spawn_best_effort(mark_image_ok(engine, image_id=int(image.id), now=now)) + background_tasks.add_task(_best_effort, mark_image_ok, engine, image_id=int(image.id), now=now, timeout_s=1.5) if needs_hydrate: - try: - await enqueue_opportunistic_hydrate_metadata(engine, illust_id=int(image.illust_id), reason="image_proxy") - except Exception: - pass + background_tasks.add_task( + _best_effort, + enqueue_opportunistic_hydrate_metadata, + engine, + illust_id=int(image.illust_id), + reason="image_proxy", + timeout_s=2.5, + ) + try: + if getattr(resp, "background", None) is None: + resp.background = background_tasks + except Exception: + pass return resp except ApiError as exc: if exc.code in { @@ -354,13 +358,13 @@ async def proxy_image( ErrorCode.UPSTREAM_404, ErrorCode.UPSTREAM_RATE_LIMIT, }: - _spawn_best_effort( - mark_image_failure( - engine, - image_id=int(image.id), - now=now, - error_code=exc.code.value, - error_message=exc.message, - ) + await _best_effort( + mark_image_failure, + engine, + image_id=int(image.id), + now=now, + error_code=exc.code.value, + error_message=exc.message, + timeout_s=1.5, ) raise diff --git a/backend/app/api/public/random.py b/backend/app/api/public/random.py index d8146ae..fa50aa6 100644 --- a/backend/app/api/public/random.py +++ b/backend/app/api/public/random.py @@ -7,7 +7,7 @@ import math from datetime import datetime, timedelta, timezone from typing import Any -from fastapi import APIRouter, Query, Request +from fastapi import APIRouter, BackgroundTasks, Query, Request from fastapi.responses import RedirectResponse from app.core.errors import ApiError, ErrorCode @@ -109,6 +109,7 @@ def _quality_score(image: Any, *, weights: dict[str, float] | None = None) -> fl @router.get("/random") async def random_image( request: Request, + background_tasks: BackgroundTasks, format: str = "image", redirect: int = 0, attempts: int | None = None, @@ -350,15 +351,9 @@ async def random_image( use_pixiv_cat = bool(runtime.image_proxy_use_pixiv_cat) or int(pixiv_cat) == 1 mirror_host = mirror_host_override or str(getattr(runtime, "image_proxy_pximg_mirror_host", "") or "").strip() or "i.pixiv.cat" - def _spawn_best_effort(coro) -> None: - async def _run() -> None: - try: - await coro - except Exception: - pass - + async def _best_effort(fn, *args, timeout_s: float = 1.5, **kwargs) -> None: # type: ignore[no-untyped-def] try: - asyncio.create_task(_run()) + await asyncio.wait_for(fn(*args, **kwargs), timeout=float(timeout_s)) except Exception: pass @@ -713,10 +708,14 @@ async def random_image( tags = await get_tag_names_for_image(session, image_id=image.id) if _needs_opportunistic_hydrate(image): - try: - await enqueue_opportunistic_hydrate_metadata(engine, illust_id=int(image.illust_id), reason="random") - except Exception: - pass + background_tasks.add_task( + _best_effort, + enqueue_opportunistic_hydrate_metadata, + engine, + illust_id=int(image.illust_id), + reason="random", + timeout_s=2.5, + ) if format == "image" and redirect == 1: qp: list[tuple[str, str]] = [] @@ -725,11 +724,17 @@ async def random_image( if mirror_host_override is not None: qp.append(("pximg_mirror_host", str(mirror_host_override))) qs = ("?" + "&".join([f"{k}={v}" for k, v in qp])) if qp else "" - return RedirectResponse( + resp = RedirectResponse( url=f"/i/{image.id}.{image.ext}{qs}", status_code=302, headers={"Cache-Control": "no-store"}, ) + try: + if getattr(resp, "background", None) is None: + resp.background = background_tasks + except Exception: + pass + return resp origin_url = None if runtime.hide_origin_url_in_public_json else image.original_url @@ -856,16 +861,21 @@ async def random_image( range_header=request.headers.get("Range"), ) if should_mark_ok: - _spawn_best_effort(mark_image_ok(engine, image_id=image_id, now=iso_utc_ms())) + background_tasks.add_task(_best_effort, mark_image_ok, engine, image_id=image_id, now=iso_utc_ms(), timeout_s=1.5) if needs_hydrate: - try: - await enqueue_opportunistic_hydrate_metadata( - engine, - illust_id=illust_id_for_hydrate, - reason="random", - ) - except Exception: - pass + background_tasks.add_task( + _best_effort, + enqueue_opportunistic_hydrate_metadata, + engine, + illust_id=illust_id_for_hydrate, + reason="random", + timeout_s=2.5, + ) + try: + if getattr(resp, "background", None) is None: + resp.background = background_tasks + except Exception: + pass return resp except ApiError as exc: if exc.code in { @@ -874,14 +884,14 @@ async def random_image( ErrorCode.UPSTREAM_404, ErrorCode.UPSTREAM_RATE_LIMIT, }: - _spawn_best_effort( - mark_image_failure( - engine, - image_id=image_id, - now=iso_utc_ms(), - error_code=exc.code.value, - error_message=exc.message, - ) + await _best_effort( + mark_image_failure, + engine, + image_id=image_id, + now=iso_utc_ms(), + error_code=exc.code.value, + error_message=exc.message, + timeout_s=1.5, ) tried_ids.add(image_id) last_error = exc diff --git a/backend/app/core/pixiv_urls.py b/backend/app/core/pixiv_urls.py index 9f56f93..7ba6714 100644 --- a/backend/app/core/pixiv_urls.py +++ b/backend/app/core/pixiv_urls.py @@ -12,10 +12,15 @@ class PixivOriginalUrl: ext: str -_PIXIV_P_RE = re.compile(r"(?P\d+)_p(?P\d+)(?:_master1200)?\.(?P[A-Za-z0-9]+)$") +_PIXIV_P_RE = re.compile( + r"(?P\d+)_p(?P\d+)(?:_(?:master|square|custom)\d+)?\.(?P[A-Za-z0-9]+)$" +) _PIXIV_UGOIRA_RE = re.compile(r"(?P\d+)_ugoira(?P\d+)\.(?P[A-Za-z0-9]+)$") +_PIXIV_UGOIRA_ZIP_RE = re.compile(r"(?P\d+)_ugoira(?:\d+x\d+)?\.(?P[A-Za-z0-9]+)$") -ALLOWED_IMAGE_EXTS = {"jpg", "jpeg", "png", "gif", "webp"} +ALLOWED_IMAGE_EXTS = {"jpg", "jpeg", "png", "gif", "webp", "zip"} + +ALLOWED_PXIMG_MIRROR_HOSTS = {"i.pixiv.cat", "i.pixiv.re", "i.pixiv.nl"} def parse_pixiv_original_url(url: str) -> PixivOriginalUrl: @@ -28,15 +33,16 @@ def parse_pixiv_original_url(url: str) -> PixivOriginalUrl: raise ValueError("unsupported scheme") host = (parsed.hostname or "").lower() - if not host.endswith("pximg.net"): + if not (host.endswith("pximg.net") or host in ALLOWED_PXIMG_MIRROR_HOSTS): raise ValueError("unsupported host") - m = _PIXIV_P_RE.search(parsed.path) or _PIXIV_UGOIRA_RE.search(parsed.path) + m = _PIXIV_P_RE.search(parsed.path) or _PIXIV_UGOIRA_RE.search(parsed.path) or _PIXIV_UGOIRA_ZIP_RE.search(parsed.path) if not m: raise ValueError("unsupported pixiv original url") illust_id = int(m.group("illust_id")) - page_index = int(m.group("page_index")) + page_index_raw = m.groupdict().get("page_index") + page_index = int(page_index_raw) if page_index_raw is not None else 0 ext = m.group("ext").lower() if ext not in ALLOWED_IMAGE_EXTS: raise ValueError("unsupported ext") diff --git a/backend/app/db/session.py b/backend/app/db/session.py index fd87f17..b1fe7d5 100644 --- a/backend/app/db/session.py +++ b/backend/app/db/session.py @@ -1,17 +1,21 @@ from __future__ import annotations import asyncio +import os +import random import sqlite3 from collections.abc import AsyncIterator, Awaitable, Callable from typing import TypeVar -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import OperationalError, TimeoutError from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker T = TypeVar("T") def is_sqlite_busy_error(exc: BaseException) -> bool: + if isinstance(exc, TimeoutError): + return True if isinstance(exc, sqlite3.OperationalError): msg = str(exc).lower() return ( @@ -29,17 +33,41 @@ def is_sqlite_busy_error(exc: BaseException) -> bool: async def with_sqlite_busy_retry( op: Callable[[], Awaitable[T]], *, - retries: int = 3, + retries: int = 8, base_delay_s: float = 0.05, ) -> T: + try: + env_retries = int((os.environ.get("SQLITE_BUSY_RETRIES") or "").strip() or retries) + except Exception: + env_retries = int(retries) + retries_i = max(0, min(int(env_retries), 50)) + + try: + env_base = float((os.environ.get("SQLITE_BUSY_BASE_DELAY_S") or "").strip() or base_delay_s) + except Exception: + env_base = float(base_delay_s) + base_delay = float(max(0.0, min(float(env_base), 5.0))) + + try: + env_max_delay = float((os.environ.get("SQLITE_BUSY_MAX_DELAY_S") or "").strip() or 2.0) + except Exception: + env_max_delay = 2.0 + max_delay = float(max(0.0, min(float(env_max_delay), 30.0))) + attempt = 0 while True: try: return await op() except Exception as exc: - if attempt >= retries or not is_sqlite_busy_error(exc): + if attempt >= retries_i or not is_sqlite_busy_error(exc): raise - await asyncio.sleep(base_delay_s * (2**attempt)) + delay = base_delay * (2**attempt) + if max_delay > 0: + delay = min(float(delay), float(max_delay)) + if delay > 0: + # Add a tiny jitter to avoid synchronized retries under load. + delay *= 0.9 + (random.random() * 0.2) + await asyncio.sleep(float(delay)) attempt += 1 diff --git a/backend/app/jobs/handlers/hydrate_metadata.py b/backend/app/jobs/handlers/hydrate_metadata.py index d1b2291..25231c7 100644 --- a/backend/app/jobs/handlers/hydrate_metadata.py +++ b/backend/app/jobs/handlers/hydrate_metadata.py @@ -1312,11 +1312,15 @@ LIMIT 1; pages: list[_IllustPage] = [] for idx, url in enumerate(urls): + url_s = str(url) try: - ext = _parse_pximg_ext(url) + ext = _parse_pximg_ext(url_s) except Exception as exc: - raise JobPermanentError("Pixiv illust detail invalid original url") from exc - pages.append(_IllustPage(page_index=int(idx), original_url=str(url), ext=ext)) + bad = redact_text(url_s) + if len(bad) > 300: + bad = bad[:297] + "..." + raise JobPermanentError(f"Pixiv illust detail invalid original url: {bad}") from exc + pages.append(_IllustPage(page_index=int(idx), original_url=url_s, ext=ext)) width = _as_int(illust.get("width")) height = _as_int(illust.get("height")) diff --git a/backend/app/main.py b/backend/app/main.py index 1071e1a..5c49153 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -4,6 +4,7 @@ import json import time from fastapi import FastAPI, Request +from fastapi.responses import Response from app.api.admin.router import router as admin_router from app.api.metrics import router as metrics_router @@ -233,6 +234,10 @@ def create_app() -> FastAPI: if engine is not None: await engine.dispose() + @app.get("/favicon.ico", include_in_schema=False) + async def _favicon() -> Response: # type: ignore[no-redef] + return Response(status_code=204, headers={"Cache-Control": "public, max-age=86400"}) + app.include_router(healthz_router) app.include_router(docs_page_router) app.include_router(authors_router)