From 968f37ad94800b7462d7e99cbf5be666a97ed934 Mon Sep 17 00:00:00 2001 From: inliver233 Date: Sun, 22 Feb 2026 18:58:55 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B4=E7=90=86=E4=BB=A3=E7=A0=81=E5=B9=B6?= =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 24 + .github/workflows/ci.yml | 74 + .gitignore | 68 + README.md | 122 + backend/Dockerfile | 33 + backend/alembic.ini | 38 + backend/alembic/env.py | 82 + backend/alembic/versions/.gitkeep | 1 + .../versions/20260210_0001_create_imports.py | 36 + .../versions/20260210_0002_create_images.py | 83 + .../versions/20260210_0003_create_tags.py | 36 + .../20260210_0004_create_image_tags.py | 35 + .../20260210_0005_create_pixiv_tokens.py | 47 + .../20260210_0006_create_proxy_endpoints.py | 55 + .../20260210_0007_create_proxy_pools.py | 40 + ...260210_0008_create_proxy_pool_endpoints.py | 63 + ...260210_0009_create_token_proxy_bindings.py | 66 + .../20260210_0010_create_runtime_settings.py | 30 + .../versions/20260210_0011_create_jobs.py | 61 + .../20260210_0012_create_hydration_runs.py | 56 + .../20260210_0013_create_request_logs.py | 42 + .../20260210_0014_create_admin_audit.py | 44 + .../versions/20260211_0015_create_api_keys.py | 45 + .../20260211_0016_create_fts_search.py | 134 + ...260215_0017_add_image_popularity_fields.py | 23 + .../20260218_0018_add_images_illust_type.py | 21 + backend/app/__init__.py | 1 + backend/app/api/admin/__init__.py | 1 + backend/app/api/admin/api_keys.py | 202 + backend/app/api/admin/audit.py | 83 + backend/app/api/admin/auth.py | 51 + backend/app/api/admin/bindings.py | 403 ++ backend/app/api/admin/deps.py | 17 + backend/app/api/admin/hydration_runs.py | 500 ++ backend/app/api/admin/images.py | 343 + backend/app/api/admin/imports.py | 435 ++ backend/app/api/admin/jobs.py | 251 + backend/app/api/admin/maintenance.py | 118 + backend/app/api/admin/proxies.py | 1095 +++ backend/app/api/admin/proxy_pools.py | 318 + backend/app/api/admin/router.py | 36 + backend/app/api/admin/settings.py | 444 ++ backend/app/api/admin/stats.py | 40 + backend/app/api/admin/summary.py | 156 + backend/app/api/admin/tokens.py | 450 ++ backend/app/api/metrics.py | 88 + backend/app/api/public/authors.py | 62 + backend/app/api/public/docs_page.py | 361 + backend/app/api/public/healthz.py | 180 + backend/app/api/public/images.py | 333 + backend/app/api/public/legacy.py | 100 + backend/app/api/public/random.py | 861 +++ backend/app/api/public/tags.py | 54 + backend/app/api/public/version.py | 35 + backend/app/core/api_keys.py | 182 + backend/app/core/bindings_recompute.py | 204 + backend/app/core/config.py | 195 + backend/app/core/crypto.py | 44 + backend/app/core/data_files.py | 57 + backend/app/core/effective_settings.py | 22 + backend/app/core/errors.py | 187 + backend/app/core/failover.py | 76 + backend/app/core/http_stream.py | 98 + backend/app/core/imgproxy.py | 148 + backend/app/core/logging.py | 30 + backend/app/core/metrics.py | 170 + backend/app/core/pixiv_urls.py | 44 + backend/app/core/proxy_routing.py | 539 ++ backend/app/core/proxy_uri.py | 100 + backend/app/core/random_request_stats.py | 83 + backend/app/core/redact.py | 95 + backend/app/core/request_id.py | 67 + backend/app/core/runtime_settings.py | 205 + backend/app/core/security.py | 147 + backend/app/core/time.py | 11 + backend/app/db/authors_list.py | 99 + backend/app/db/engine.py | 72 + backend/app/db/images_get.py | 12 + backend/app/db/images_get_by_illust.py | 25 + backend/app/db/images_list.py | 91 + backend/app/db/images_mark.py | 62 + backend/app/db/images_upsert.py | 44 + backend/app/db/models/admin_audit.py | 32 + backend/app/db/models/api_keys.py | 38 + backend/app/db/models/base.py | 26 + backend/app/db/models/hydration_runs.py | 42 + backend/app/db/models/image_tags.py | 23 + backend/app/db/models/images.py | 78 + backend/app/db/models/imports.py | 27 + backend/app/db/models/jobs.py | 49 + backend/app/db/models/pixiv_tokens.py | 41 + backend/app/db/models/proxy_endpoints.py | 47 + backend/app/db/models/proxy_pool_endpoints.py | 41 + backend/app/db/models/proxy_pools.py | 32 + backend/app/db/models/request_logs.py | 31 + backend/app/db/models/runtime_settings.py | 21 + backend/app/db/models/tags.py | 26 + backend/app/db/models/token_proxy_bindings.py | 51 + backend/app/db/random_pick.py | 366 + backend/app/db/request_logs_cleanup.py | 130 + backend/app/db/session.py | 52 + backend/app/db/sqlite_utils.py | 21 + backend/app/db/tags_get.py | 19 + backend/app/db/tags_list.py | 101 + backend/app/easy_proxies/__init__.py | 2 + backend/app/easy_proxies/auto_refresh.py | 133 + backend/app/easy_proxies/client.py | 105 + backend/app/easy_proxies/normalize.py | 39 + backend/app/jobs/backoff.py | 23 + backend/app/jobs/claim.py | 95 + backend/app/jobs/dispatch.py | 33 + backend/app/jobs/enqueue.py | 73 + backend/app/jobs/errors.py | 11 + backend/app/jobs/executor.py | 147 + .../app/jobs/handlers/easy_proxies_import.py | 289 + backend/app/jobs/handlers/heal_url.py | 66 + backend/app/jobs/handlers/hydrate_metadata.py | 1495 +++++ backend/app/jobs/handlers/import_images.py | 276 + backend/app/jobs/handlers/proxy_probe.py | 255 + backend/app/jobs/model.py | 113 + backend/app/main.py | 252 + backend/app/pixiv/__init__.py | 2 + backend/app/pixiv/access_token_cache.py | 97 + backend/app/pixiv/oauth.py | 151 + backend/app/pixiv/refresh_backoff.py | 40 + backend/app/pixiv/token_strategy.py | 116 + backend/app/web/__init__.py | 1 + backend/app/web/admin_ui.py | 40 + backend/app/worker.py | 451 ++ backend/tests/conftest.py | 7 + backend/tests/test_access_token_cache.py | 47 + backend/tests/test_admin_api_keys.py | 76 + backend/tests/test_admin_audit_list.py | 146 + .../test_admin_bindings_clear_override.py | 128 + backend/tests/test_admin_bindings_list.py | 103 + backend/tests/test_admin_bindings_override.py | 235 + .../tests/test_admin_bindings_recompute.py | 394 ++ .../test_admin_hydration_runs_actions.py | 78 + .../tests/test_admin_hydration_runs_create.py | 69 + ...st_admin_hydration_runs_list_and_manual.py | 218 + backend/tests/test_admin_hydration_stats.py | 96 + backend/tests/test_admin_images_delete.py | 119 + .../tests/test_admin_images_list_filters.py | 157 + backend/tests/test_admin_imports_get.py | 92 + backend/tests/test_admin_imports_json.py | 363 + backend/tests/test_admin_jobs_actions.py | 130 + .../test_admin_jobs_detail_and_actions.py | 78 + backend/tests/test_admin_jobs_list.py | 135 + backend/tests/test_admin_jwt.py | 51 + backend/tests/test_admin_login.py | 136 + .../test_admin_proxies_easy_proxies_import.py | 251 + backend/tests/test_admin_proxies_import.py | 168 + backend/tests/test_admin_proxies_list.py | 86 + backend/tests/test_admin_proxies_probe.py | 58 + .../test_admin_proxies_reset_failures.py | 85 + ..._update_enabled_and_passwordless_import.py | 137 + .../tests/test_admin_proxy_pools_create.py | 54 + .../tests/test_admin_proxy_pools_endpoints.py | 161 + backend/tests/test_admin_proxy_pools_list.py | 52 + .../tests/test_admin_proxy_pools_update.py | 88 + backend/tests/test_admin_random_stats.py | 63 + .../tests/test_admin_request_logs_cleanup.py | 138 + backend/tests/test_admin_settings_get.py | 84 + backend/tests/test_admin_settings_put.py | 157 + backend/tests/test_admin_tokens_create.py | 123 + backend/tests/test_admin_tokens_delete.py | 99 + backend/tests/test_admin_tokens_list.py | 64 + .../tests/test_admin_tokens_reset_failures.py | 87 + .../tests/test_admin_tokens_test_refresh.py | 196 + backend/tests/test_admin_tokens_update.py | 80 + backend/tests/test_authors_list.py | 190 + backend/tests/test_config.py | 48 + backend/tests/test_crypto.py | 43 + .../tests/test_e2e_minimal_import_random.py | 75 + .../tests/test_easy_proxies_auto_refresh.py | 64 + backend/tests/test_easy_proxies_client.py | 60 + backend/tests/test_effective_settings.py | 47 + backend/tests/test_error_codes.py | 386 ++ backend/tests/test_error_schema.py | 15 + backend/tests/test_failover_policy.py | 50 + backend/tests/test_healthz.py | 96 + backend/tests/test_http_stream.py | 144 + ...test_hydrate_persists_popularity_fields.py | 189 + backend/tests/test_image_proxy.py | 269 + backend/tests/test_images_get.py | 109 + backend/tests/test_images_list.py | 151 + backend/tests/test_images_upsert.py | 61 + backend/tests/test_imgproxy_signing.py | 84 + backend/tests/test_job_dispatch.py | 32 + backend/tests/test_job_fsm.py | 46 + .../test_job_handler_easy_proxies_import.py | 298 + backend/tests/test_job_handler_heal_url.py | 155 + .../test_job_handler_hydrate_metadata.py | 697 ++ ...test_job_handler_hydration_run_backfill.py | 197 + .../tests/test_job_handler_import_images.py | 249 + backend/tests/test_job_handler_proxy_probe.py | 113 + backend/tests/test_jobs_claim.py | 159 + .../tests/test_jobs_defer_on_sqlite_busy.py | 83 + backend/tests/test_legacy_single.py | 180 + backend/tests/test_log_redaction.py | 68 + backend/tests/test_metrics.py | 81 + backend/tests/test_models_admin_audit.py | 38 + backend/tests/test_models_hydration_runs.py | 39 + backend/tests/test_models_image_tags.py | 58 + backend/tests/test_models_images.py | 53 + backend/tests/test_models_imports.py | 40 + backend/tests/test_models_jobs.py | 39 + backend/tests/test_models_pixiv_tokens.py | 46 + backend/tests/test_models_proxy_endpoints.py | 39 + .../tests/test_models_proxy_pool_endpoints.py | 45 + backend/tests/test_models_proxy_pools.py | 38 + backend/tests/test_models_request_logs.py | 47 + backend/tests/test_models_runtime_settings.py | 37 + backend/tests/test_models_tags.py | 38 + .../tests/test_models_token_proxy_bindings.py | 56 + backend/tests/test_opportunistic_hydrate.py | 195 + backend/tests/test_pixiv_oauth.py | 98 + backend/tests/test_pixiv_urls.py | 52 + backend/tests/test_proxy_pool_routing.py | 456 ++ backend/tests/test_proxy_uri.py | 45 + backend/tests/test_public_api_key_required.py | 89 + backend/tests/test_random_attempts_retry.py | 114 + .../test_random_contract_seed_ai_type.py | 131 + backend/tests/test_random_created_filters.py | 71 + backend/tests/test_random_fail_cooldown.py | 84 + backend/tests/test_random_filters.py | 165 + backend/tests/test_random_geometry_filters.py | 110 + backend/tests/test_random_image_mode.py | 177 + backend/tests/test_random_json_tags.py | 56 + backend/tests/test_random_key_index.py | 30 + backend/tests/test_random_no_match_hints.py | 50 + backend/tests/test_random_quality_strategy.py | 295 + backend/tests/test_random_r18_filter.py | 90 + backend/tests/test_random_runtime_defaults.py | 143 + ...test_random_settings_defaults_alignment.py | 79 + backend/tests/test_random_tag_filters.py | 173 + .../tests/test_random_user_illust_filters.py | 62 + backend/tests/test_refresh_backoff.py | 22 + backend/tests/test_request_id.py | 42 + backend/tests/test_sensitive_no_echo.py | 148 + backend/tests/test_smoke.py | 3 + backend/tests/test_sqlite_pragmas.py | 59 + backend/tests/test_tags_list.py | 197 + backend/tests/test_token_strategy.py | 98 + .../tests/test_unhandled_exception_json.py | 34 + backend/tests/test_version.py | 27 + backend/tests/test_worker_jobs_loop.py | 77 + backend/tests/test_worker_smoke.py | 11 + deploy/.env.example | 50 + deploy/docker-compose.yml | 44 + frontend/.eslintrc.cjs | 25 + frontend/index.html | 12 + frontend/package-lock.json | 5973 +++++++++++++++++ frontend/package.json | 38 + frontend/src/api/client.ts | 152 + frontend/src/app/AdminLayout.tsx | 135 + frontend/src/app/App.tsx | 130 + frontend/src/auth/tokenStorage.test.ts | 18 + frontend/src/auth/tokenStorage.ts | 16 + frontend/src/main.tsx | 18 + frontend/src/pages/AuditPage.test.tsx | 60 + frontend/src/pages/AuditPage.tsx | 90 + frontend/src/pages/AuthorsPage.test.tsx | 49 + frontend/src/pages/AuthorsPage.tsx | 73 + frontend/src/pages/BindingsPage.test.tsx | 202 + frontend/src/pages/BindingsPage.tsx | 467 ++ frontend/src/pages/DashboardPage.test.tsx | 191 + frontend/src/pages/DashboardPage.tsx | 372 + frontend/src/pages/HydrationPage.test.tsx | 169 + frontend/src/pages/HydrationPage.tsx | 628 ++ frontend/src/pages/ImagesPage.test.tsx | 76 + frontend/src/pages/ImagesPage.tsx | 377 ++ frontend/src/pages/ImportDetailPage.test.tsx | 121 + frontend/src/pages/ImportDetailPage.tsx | 189 + frontend/src/pages/ImportPage.test.tsx | 128 + frontend/src/pages/ImportPage.tsx | 253 + frontend/src/pages/JobsPage.test.tsx | 66 + frontend/src/pages/JobsPage.tsx | 329 + frontend/src/pages/LoginPage.test.tsx | 86 + frontend/src/pages/LoginPage.tsx | 106 + frontend/src/pages/PlaceholderPage.tsx | 6 + frontend/src/pages/PlaygroundPage.test.tsx | 93 + frontend/src/pages/PlaygroundPage.tsx | 542 ++ frontend/src/pages/ProxiesPage.test.tsx | 218 + frontend/src/pages/ProxiesPage.tsx | 843 +++ frontend/src/pages/ProxyPoolsPage.test.tsx | 97 + frontend/src/pages/ProxyPoolsPage.tsx | 495 ++ .../src/pages/RecommendationPage.test.tsx | 119 + frontend/src/pages/RecommendationPage.tsx | 449 ++ frontend/src/pages/SettingsPage.test.tsx | 101 + frontend/src/pages/SettingsPage.tsx | 327 + frontend/src/pages/TagsPage.test.tsx | 84 + frontend/src/pages/TagsPage.tsx | 97 + frontend/src/pages/TokensPage.test.tsx | 183 + frontend/src/pages/TokensPage.tsx | 458 ++ frontend/src/test/setup.ts | 23 + frontend/src/vite-env.d.ts | 2 + frontend/tsconfig.json | 17 + frontend/tsconfig.node.json | 11 + frontend/vite.config.ts | 30 + requirements-dev.in | 3 + requirements-dev.txt | 106 + requirements.in | 9 + requirements.txt | 95 + scripts/perf/bench_random_asgi.py | 263 + scripts/test_all.ps1 | 21 + scripts/test_backend.ps1 | 22 + scripts/test_frontend.ps1 | 32 + scripts/verify_backend_install.ps1 | 36 + 309 files changed, 46134 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 backend/Dockerfile create mode 100644 backend/alembic.ini create mode 100644 backend/alembic/env.py create mode 100644 backend/alembic/versions/.gitkeep create mode 100644 backend/alembic/versions/20260210_0001_create_imports.py create mode 100644 backend/alembic/versions/20260210_0002_create_images.py create mode 100644 backend/alembic/versions/20260210_0003_create_tags.py create mode 100644 backend/alembic/versions/20260210_0004_create_image_tags.py create mode 100644 backend/alembic/versions/20260210_0005_create_pixiv_tokens.py create mode 100644 backend/alembic/versions/20260210_0006_create_proxy_endpoints.py create mode 100644 backend/alembic/versions/20260210_0007_create_proxy_pools.py create mode 100644 backend/alembic/versions/20260210_0008_create_proxy_pool_endpoints.py create mode 100644 backend/alembic/versions/20260210_0009_create_token_proxy_bindings.py create mode 100644 backend/alembic/versions/20260210_0010_create_runtime_settings.py create mode 100644 backend/alembic/versions/20260210_0011_create_jobs.py create mode 100644 backend/alembic/versions/20260210_0012_create_hydration_runs.py create mode 100644 backend/alembic/versions/20260210_0013_create_request_logs.py create mode 100644 backend/alembic/versions/20260210_0014_create_admin_audit.py create mode 100644 backend/alembic/versions/20260211_0015_create_api_keys.py create mode 100644 backend/alembic/versions/20260211_0016_create_fts_search.py create mode 100644 backend/alembic/versions/20260215_0017_add_image_popularity_fields.py create mode 100644 backend/alembic/versions/20260218_0018_add_images_illust_type.py create mode 100644 backend/app/__init__.py create mode 100644 backend/app/api/admin/__init__.py create mode 100644 backend/app/api/admin/api_keys.py create mode 100644 backend/app/api/admin/audit.py create mode 100644 backend/app/api/admin/auth.py create mode 100644 backend/app/api/admin/bindings.py create mode 100644 backend/app/api/admin/deps.py create mode 100644 backend/app/api/admin/hydration_runs.py create mode 100644 backend/app/api/admin/images.py create mode 100644 backend/app/api/admin/imports.py create mode 100644 backend/app/api/admin/jobs.py create mode 100644 backend/app/api/admin/maintenance.py create mode 100644 backend/app/api/admin/proxies.py create mode 100644 backend/app/api/admin/proxy_pools.py create mode 100644 backend/app/api/admin/router.py create mode 100644 backend/app/api/admin/settings.py create mode 100644 backend/app/api/admin/stats.py create mode 100644 backend/app/api/admin/summary.py create mode 100644 backend/app/api/admin/tokens.py create mode 100644 backend/app/api/metrics.py create mode 100644 backend/app/api/public/authors.py create mode 100644 backend/app/api/public/docs_page.py create mode 100644 backend/app/api/public/healthz.py create mode 100644 backend/app/api/public/images.py create mode 100644 backend/app/api/public/legacy.py create mode 100644 backend/app/api/public/random.py create mode 100644 backend/app/api/public/tags.py create mode 100644 backend/app/api/public/version.py create mode 100644 backend/app/core/api_keys.py create mode 100644 backend/app/core/bindings_recompute.py create mode 100644 backend/app/core/config.py create mode 100644 backend/app/core/crypto.py create mode 100644 backend/app/core/data_files.py create mode 100644 backend/app/core/effective_settings.py create mode 100644 backend/app/core/errors.py create mode 100644 backend/app/core/failover.py create mode 100644 backend/app/core/http_stream.py create mode 100644 backend/app/core/imgproxy.py create mode 100644 backend/app/core/logging.py create mode 100644 backend/app/core/metrics.py create mode 100644 backend/app/core/pixiv_urls.py create mode 100644 backend/app/core/proxy_routing.py create mode 100644 backend/app/core/proxy_uri.py create mode 100644 backend/app/core/random_request_stats.py create mode 100644 backend/app/core/redact.py create mode 100644 backend/app/core/request_id.py create mode 100644 backend/app/core/runtime_settings.py create mode 100644 backend/app/core/security.py create mode 100644 backend/app/core/time.py create mode 100644 backend/app/db/authors_list.py create mode 100644 backend/app/db/engine.py create mode 100644 backend/app/db/images_get.py create mode 100644 backend/app/db/images_get_by_illust.py create mode 100644 backend/app/db/images_list.py create mode 100644 backend/app/db/images_mark.py create mode 100644 backend/app/db/images_upsert.py create mode 100644 backend/app/db/models/admin_audit.py create mode 100644 backend/app/db/models/api_keys.py create mode 100644 backend/app/db/models/base.py create mode 100644 backend/app/db/models/hydration_runs.py create mode 100644 backend/app/db/models/image_tags.py create mode 100644 backend/app/db/models/images.py create mode 100644 backend/app/db/models/imports.py create mode 100644 backend/app/db/models/jobs.py create mode 100644 backend/app/db/models/pixiv_tokens.py create mode 100644 backend/app/db/models/proxy_endpoints.py create mode 100644 backend/app/db/models/proxy_pool_endpoints.py create mode 100644 backend/app/db/models/proxy_pools.py create mode 100644 backend/app/db/models/request_logs.py create mode 100644 backend/app/db/models/runtime_settings.py create mode 100644 backend/app/db/models/tags.py create mode 100644 backend/app/db/models/token_proxy_bindings.py create mode 100644 backend/app/db/random_pick.py create mode 100644 backend/app/db/request_logs_cleanup.py create mode 100644 backend/app/db/session.py create mode 100644 backend/app/db/sqlite_utils.py create mode 100644 backend/app/db/tags_get.py create mode 100644 backend/app/db/tags_list.py create mode 100644 backend/app/easy_proxies/__init__.py create mode 100644 backend/app/easy_proxies/auto_refresh.py create mode 100644 backend/app/easy_proxies/client.py create mode 100644 backend/app/easy_proxies/normalize.py create mode 100644 backend/app/jobs/backoff.py create mode 100644 backend/app/jobs/claim.py create mode 100644 backend/app/jobs/dispatch.py create mode 100644 backend/app/jobs/enqueue.py create mode 100644 backend/app/jobs/errors.py create mode 100644 backend/app/jobs/executor.py create mode 100644 backend/app/jobs/handlers/easy_proxies_import.py create mode 100644 backend/app/jobs/handlers/heal_url.py create mode 100644 backend/app/jobs/handlers/hydrate_metadata.py create mode 100644 backend/app/jobs/handlers/import_images.py create mode 100644 backend/app/jobs/handlers/proxy_probe.py create mode 100644 backend/app/jobs/model.py create mode 100644 backend/app/main.py create mode 100644 backend/app/pixiv/__init__.py create mode 100644 backend/app/pixiv/access_token_cache.py create mode 100644 backend/app/pixiv/oauth.py create mode 100644 backend/app/pixiv/refresh_backoff.py create mode 100644 backend/app/pixiv/token_strategy.py create mode 100644 backend/app/web/__init__.py create mode 100644 backend/app/web/admin_ui.py create mode 100644 backend/app/worker.py create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/test_access_token_cache.py create mode 100644 backend/tests/test_admin_api_keys.py create mode 100644 backend/tests/test_admin_audit_list.py create mode 100644 backend/tests/test_admin_bindings_clear_override.py create mode 100644 backend/tests/test_admin_bindings_list.py create mode 100644 backend/tests/test_admin_bindings_override.py create mode 100644 backend/tests/test_admin_bindings_recompute.py create mode 100644 backend/tests/test_admin_hydration_runs_actions.py create mode 100644 backend/tests/test_admin_hydration_runs_create.py create mode 100644 backend/tests/test_admin_hydration_runs_list_and_manual.py create mode 100644 backend/tests/test_admin_hydration_stats.py create mode 100644 backend/tests/test_admin_images_delete.py create mode 100644 backend/tests/test_admin_images_list_filters.py create mode 100644 backend/tests/test_admin_imports_get.py create mode 100644 backend/tests/test_admin_imports_json.py create mode 100644 backend/tests/test_admin_jobs_actions.py create mode 100644 backend/tests/test_admin_jobs_detail_and_actions.py create mode 100644 backend/tests/test_admin_jobs_list.py create mode 100644 backend/tests/test_admin_jwt.py create mode 100644 backend/tests/test_admin_login.py create mode 100644 backend/tests/test_admin_proxies_easy_proxies_import.py create mode 100644 backend/tests/test_admin_proxies_import.py create mode 100644 backend/tests/test_admin_proxies_list.py create mode 100644 backend/tests/test_admin_proxies_probe.py create mode 100644 backend/tests/test_admin_proxies_reset_failures.py create mode 100644 backend/tests/test_admin_proxies_update_enabled_and_passwordless_import.py create mode 100644 backend/tests/test_admin_proxy_pools_create.py create mode 100644 backend/tests/test_admin_proxy_pools_endpoints.py create mode 100644 backend/tests/test_admin_proxy_pools_list.py create mode 100644 backend/tests/test_admin_proxy_pools_update.py create mode 100644 backend/tests/test_admin_random_stats.py create mode 100644 backend/tests/test_admin_request_logs_cleanup.py create mode 100644 backend/tests/test_admin_settings_get.py create mode 100644 backend/tests/test_admin_settings_put.py create mode 100644 backend/tests/test_admin_tokens_create.py create mode 100644 backend/tests/test_admin_tokens_delete.py create mode 100644 backend/tests/test_admin_tokens_list.py create mode 100644 backend/tests/test_admin_tokens_reset_failures.py create mode 100644 backend/tests/test_admin_tokens_test_refresh.py create mode 100644 backend/tests/test_admin_tokens_update.py create mode 100644 backend/tests/test_authors_list.py create mode 100644 backend/tests/test_config.py create mode 100644 backend/tests/test_crypto.py create mode 100644 backend/tests/test_e2e_minimal_import_random.py create mode 100644 backend/tests/test_easy_proxies_auto_refresh.py create mode 100644 backend/tests/test_easy_proxies_client.py create mode 100644 backend/tests/test_effective_settings.py create mode 100644 backend/tests/test_error_codes.py create mode 100644 backend/tests/test_error_schema.py create mode 100644 backend/tests/test_failover_policy.py create mode 100644 backend/tests/test_healthz.py create mode 100644 backend/tests/test_http_stream.py create mode 100644 backend/tests/test_hydrate_persists_popularity_fields.py create mode 100644 backend/tests/test_image_proxy.py create mode 100644 backend/tests/test_images_get.py create mode 100644 backend/tests/test_images_list.py create mode 100644 backend/tests/test_images_upsert.py create mode 100644 backend/tests/test_imgproxy_signing.py create mode 100644 backend/tests/test_job_dispatch.py create mode 100644 backend/tests/test_job_fsm.py create mode 100644 backend/tests/test_job_handler_easy_proxies_import.py create mode 100644 backend/tests/test_job_handler_heal_url.py create mode 100644 backend/tests/test_job_handler_hydrate_metadata.py create mode 100644 backend/tests/test_job_handler_hydration_run_backfill.py create mode 100644 backend/tests/test_job_handler_import_images.py create mode 100644 backend/tests/test_job_handler_proxy_probe.py create mode 100644 backend/tests/test_jobs_claim.py create mode 100644 backend/tests/test_jobs_defer_on_sqlite_busy.py create mode 100644 backend/tests/test_legacy_single.py create mode 100644 backend/tests/test_log_redaction.py create mode 100644 backend/tests/test_metrics.py create mode 100644 backend/tests/test_models_admin_audit.py create mode 100644 backend/tests/test_models_hydration_runs.py create mode 100644 backend/tests/test_models_image_tags.py create mode 100644 backend/tests/test_models_images.py create mode 100644 backend/tests/test_models_imports.py create mode 100644 backend/tests/test_models_jobs.py create mode 100644 backend/tests/test_models_pixiv_tokens.py create mode 100644 backend/tests/test_models_proxy_endpoints.py create mode 100644 backend/tests/test_models_proxy_pool_endpoints.py create mode 100644 backend/tests/test_models_proxy_pools.py create mode 100644 backend/tests/test_models_request_logs.py create mode 100644 backend/tests/test_models_runtime_settings.py create mode 100644 backend/tests/test_models_tags.py create mode 100644 backend/tests/test_models_token_proxy_bindings.py create mode 100644 backend/tests/test_opportunistic_hydrate.py create mode 100644 backend/tests/test_pixiv_oauth.py create mode 100644 backend/tests/test_pixiv_urls.py create mode 100644 backend/tests/test_proxy_pool_routing.py create mode 100644 backend/tests/test_proxy_uri.py create mode 100644 backend/tests/test_public_api_key_required.py create mode 100644 backend/tests/test_random_attempts_retry.py create mode 100644 backend/tests/test_random_contract_seed_ai_type.py create mode 100644 backend/tests/test_random_created_filters.py create mode 100644 backend/tests/test_random_fail_cooldown.py create mode 100644 backend/tests/test_random_filters.py create mode 100644 backend/tests/test_random_geometry_filters.py create mode 100644 backend/tests/test_random_image_mode.py create mode 100644 backend/tests/test_random_json_tags.py create mode 100644 backend/tests/test_random_key_index.py create mode 100644 backend/tests/test_random_no_match_hints.py create mode 100644 backend/tests/test_random_quality_strategy.py create mode 100644 backend/tests/test_random_r18_filter.py create mode 100644 backend/tests/test_random_runtime_defaults.py create mode 100644 backend/tests/test_random_settings_defaults_alignment.py create mode 100644 backend/tests/test_random_tag_filters.py create mode 100644 backend/tests/test_random_user_illust_filters.py create mode 100644 backend/tests/test_refresh_backoff.py create mode 100644 backend/tests/test_request_id.py create mode 100644 backend/tests/test_sensitive_no_echo.py create mode 100644 backend/tests/test_smoke.py create mode 100644 backend/tests/test_sqlite_pragmas.py create mode 100644 backend/tests/test_tags_list.py create mode 100644 backend/tests/test_token_strategy.py create mode 100644 backend/tests/test_unhandled_exception_json.py create mode 100644 backend/tests/test_version.py create mode 100644 backend/tests/test_worker_jobs_loop.py create mode 100644 backend/tests/test_worker_smoke.py create mode 100644 deploy/.env.example create mode 100644 deploy/docker-compose.yml create mode 100644 frontend/.eslintrc.cjs create mode 100644 frontend/index.html create mode 100644 frontend/package-lock.json create mode 100644 frontend/package.json create mode 100644 frontend/src/api/client.ts create mode 100644 frontend/src/app/AdminLayout.tsx create mode 100644 frontend/src/app/App.tsx create mode 100644 frontend/src/auth/tokenStorage.test.ts create mode 100644 frontend/src/auth/tokenStorage.ts create mode 100644 frontend/src/main.tsx create mode 100644 frontend/src/pages/AuditPage.test.tsx create mode 100644 frontend/src/pages/AuditPage.tsx create mode 100644 frontend/src/pages/AuthorsPage.test.tsx create mode 100644 frontend/src/pages/AuthorsPage.tsx create mode 100644 frontend/src/pages/BindingsPage.test.tsx create mode 100644 frontend/src/pages/BindingsPage.tsx create mode 100644 frontend/src/pages/DashboardPage.test.tsx create mode 100644 frontend/src/pages/DashboardPage.tsx create mode 100644 frontend/src/pages/HydrationPage.test.tsx create mode 100644 frontend/src/pages/HydrationPage.tsx create mode 100644 frontend/src/pages/ImagesPage.test.tsx create mode 100644 frontend/src/pages/ImagesPage.tsx create mode 100644 frontend/src/pages/ImportDetailPage.test.tsx create mode 100644 frontend/src/pages/ImportDetailPage.tsx create mode 100644 frontend/src/pages/ImportPage.test.tsx create mode 100644 frontend/src/pages/ImportPage.tsx create mode 100644 frontend/src/pages/JobsPage.test.tsx create mode 100644 frontend/src/pages/JobsPage.tsx create mode 100644 frontend/src/pages/LoginPage.test.tsx create mode 100644 frontend/src/pages/LoginPage.tsx create mode 100644 frontend/src/pages/PlaceholderPage.tsx create mode 100644 frontend/src/pages/PlaygroundPage.test.tsx create mode 100644 frontend/src/pages/PlaygroundPage.tsx create mode 100644 frontend/src/pages/ProxiesPage.test.tsx create mode 100644 frontend/src/pages/ProxiesPage.tsx create mode 100644 frontend/src/pages/ProxyPoolsPage.test.tsx create mode 100644 frontend/src/pages/ProxyPoolsPage.tsx create mode 100644 frontend/src/pages/RecommendationPage.test.tsx create mode 100644 frontend/src/pages/RecommendationPage.tsx create mode 100644 frontend/src/pages/SettingsPage.test.tsx create mode 100644 frontend/src/pages/SettingsPage.tsx create mode 100644 frontend/src/pages/TagsPage.test.tsx create mode 100644 frontend/src/pages/TagsPage.tsx create mode 100644 frontend/src/pages/TokensPage.test.tsx create mode 100644 frontend/src/pages/TokensPage.tsx create mode 100644 frontend/src/test/setup.ts create mode 100644 frontend/src/vite-env.d.ts create mode 100644 frontend/tsconfig.json create mode 100644 frontend/tsconfig.node.json create mode 100644 frontend/vite.config.ts create mode 100644 requirements-dev.in create mode 100644 requirements-dev.txt create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 scripts/perf/bench_random_asgi.py create mode 100644 scripts/test_all.ps1 create mode 100644 scripts/test_backend.ps1 create mode 100644 scripts/test_frontend.ps1 create mode 100644 scripts/verify_backend_install.ps1 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cf258d7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,24 @@ +.git +.github +.codex +.venv +.pytest_cache +**/__pycache__/ + +# Env / secrets +.env +.env.* +!deploy/.env.example + +data/ +issues/ +plan/ +docs/ +参考资料/ + +frontend/node_modules/ +frontend/dist/ + +backend/.pytest_cache/ + +_tmp_append_test.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..59c5284 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,74 @@ +name: ci + +on: + push: + branches: + - main + - test + pull_request: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + backend-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: requirements-dev.txt + + - name: Install dependencies + run: python -m pip install -r requirements-dev.txt + + - name: Compile + run: python -m compileall -q backend + + - name: Tests + run: python -m pytest -q backend/tests + + frontend-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + working-directory: frontend + + - name: Lint + run: npm run lint + working-directory: frontend + + - name: Typecheck + run: npm run typecheck + working-directory: frontend + + - name: Tests + run: npm test + working-directory: frontend + + compose-config: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Prepare deploy env file + run: | + if [ ! -f deploy/.env ]; then + cp deploy/.env.example deploy/.env + fi + + - name: Validate docker compose config + run: docker compose -f deploy/docker-compose.yml config > /dev/null diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e4f7713 --- /dev/null +++ b/.gitignore @@ -0,0 +1,68 @@ +## Python +__pycache__/ +*.py[cod] +*$py.class +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +*.egg-info/ +.coverage +.coverage.* +htmlcov/ +coverage.xml + +## Virtualenv +.venv/ +venv/ +ENV/ +env/ + +## Node +node_modules/ +dist/ +.vite/ + +## Runtime data / local state (never commit) +data/ +backend/data/ +deploy/.env +.env +.env.* +!.env.example +!deploy/.env.example +*.db +*.sqlite +*.sqlite3 + +## Codex / AI tooling (repo-local) +.codex/ +plan/ +issues/ +docs/ +参考资料/ + +## Data exports / plans +*.csv +*.tsv +*.plan.md + +## OS / IDE +.DS_Store +Thumbs.db +desktop.ini +.idea/ +.vscode/ +*.iml + +## Local temp / scratch +*.log +*.tmp +要求.txt +_tmp_easy_export.txt +_tmp_*.md +_tmp_*.txt + +## Keep public-facing docs at repo root +/*.md +!README.md +!LICENSE* diff --git a/README.md b/README.md new file mode 100644 index 0000000..8b43dad --- /dev/null +++ b/README.md @@ -0,0 +1,122 @@ +# Random Mage (new-pixiv-api) + +一个自托管的随机图片 API：把 Pixiv 原图链接导入到本地数据库后，即可通过 `/random` 按 **标签 / 热度 / 分辨率 / R18 / AI / 作品类型** 等条件组合筛选并随机出图；同时提供管理后台（Web UI）+ 后台 Worker，用于导入、补全元数据、代理探测等任务。 + +> 免责声明：本项目为非官方实现；请遵守 Pixiv ToS/版权与当地法律法规，自行承担使用风险。不要将任何 refresh_token / 密钥 / 代理密码提交到仓库。 + +## 功能概览 + +- 公共 API + - `GET /random`：默认直接返回图片流；也可 `format=json|simple_json` 返回 JSON；支持大量筛选参数 + - `GET /images` / `GET /images/{id}`：按条件分页查看已入库图片 + - `GET /i/{image_id}.{ext}`：本地代理/缓存路径（`/random?redirect=1` 会跳转到这里） + - `GET /tags`、`GET /authors`、`GET /healthz`、`GET /version`、`GET /docs`（人类可读文档页） + - Swagger / OpenAPI：`GET /api/docs`、`GET /openapi.json` +- 管理后台（`/admin`） + - 登录后可管理：Pixiv refresh_token、代理池、导入、任务队列、运行时开关、审计/统计等 +- Worker（后台任务） + - 导入图片 URL、补全元数据（tags/宽高/浏览收藏等）、修复失效 URL、代理探测等 +- 可选能力 + - 公网访问保护：启用 Public API Key + 限流 + - `imgproxy`：对外提供签名处理 URL（可隐藏 origin URL） + - 代理路由：为上游图片/接口选择代理 + +## 一键部署（Docker Compose，推荐） + +前置：安装 Docker + Docker Compose。 + +1) 准备环境变量 + +```bash +cp deploy/.env.example deploy/.env +``` + +然后编辑 `deploy/.env`，至少建议修改： + +- `SECRET_KEY`：JWT/API Key 哈希用（生产环境必须修改） +- `ADMIN_PASSWORD`：后台密码（生产环境必须修改） +- `FIELD_ENCRYPTION_KEY`：用于加密保存 refresh_token（生产环境必须提供；建议用 secret 挂载或文件方式） + +2) 启动 + +```bash +docker compose -f deploy/docker-compose.yml up -d --build +``` + +3) 访问 + +- API 基址：`http://localhost:23222` +- 健康检查：`GET http://localhost:23222/healthz` +- 文档页：`GET http://localhost:23222/docs` +- 后台 UI：`http://localhost:23222/admin` +- Swagger：`http://localhost:23222/api/docs` + +## 首次使用（导入数据 + 补全元数据） + +这个项目不会“自动替你抓全站图片”，你需要先把图片记录导入数据库。 + +1) 登录后台 + +- 打开 `http://localhost:23222/admin` +- 使用 `deploy/.env` 中的 `ADMIN_USERNAME` / `ADMIN_PASSWORD` 登录 + +2) 导入图片原图 URL + +后台提供“导入”入口，支持： + +- 直接粘贴多行文本（每行一个 URL） +- 上传 `.txt` 文件（同样每行一个 URL） + +导入内容应为 Pixiv 原图链接（例如 `i.pximg.net/.../img-original/...` 这类）。导入时会自动去重（按 illust_id + page）。 + +3) 配置 Pixiv refresh_token（可选，但强烈建议） + +若你希望 Worker 自动补全/刷新元数据（tags、宽高、浏览/收藏/评论、R18 标记、AI 类型等），需要： + +- 在后台添加一个或多个 refresh_token +- 配置 `PIXIV_OAUTH_CLIENT_ID` / `PIXIV_OAUTH_CLIENT_SECRET`（以及可选的 `PIXIV_OAUTH_HASH_SECRET`） + +> 安全提示：refresh_token 属于敏感凭据；请只放在运行环境里（`deploy/.env` / secret 管理），不要写进代码/仓库。 + +4) 确认 Worker 正常 + +`/healthz` 会同时检查 DB、队列、以及 Worker 心跳；如果 `worker_ok=false`，通常意味着 Worker 没启动或无法连接数据库卷。 + +## 配置说明（节选） + +所有配置均通过环境变量注入（见 `deploy/.env.example`）： + +- `APP_ENV`：`dev` / `prod`（`prod` 会启用更严格的必填校验） +- `DATABASE_URL`：默认 SQLite（容器内 `/app/data/app.db`，通过 `../data:/app/data` 挂载） +- `PUBLIC_API_KEY_REQUIRED` / `PUBLIC_API_KEY_RPM` / `PUBLIC_API_KEY_BURST`：公网访问保护（可选） +- `WORKER_*`、`IMPORT_*`、`SQLITE_BUSY_TIMEOUT_MS`：Worker/导入/SQLite 运行参数 +- `IMGPROXY_*`：imgproxy 集成（可选） + +## 本地开发（可选） + +后端（Python 3.11）： + +```bash +cd backend +python -m venv .venv +.\.venv\Scripts\activate +python -m pip install -r ..\requirements-dev.txt +alembic upgrade head +uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 +``` + +前端管理台（Node 20）： + +```bash +cd frontend +npm ci +npm run dev +``` + +如需让前端直连本地后端，设置 `VITE_API_BASE_URL`（例如 `http://localhost:8000`）。 + +## 运行测试 + +- 后端：`python -m pytest -q backend/tests` +- 前端：`cd frontend && npm test` + diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..ae78e08 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,33 @@ +FROM node:20-slim AS frontend-build + +WORKDIR /frontend + +COPY frontend/package.json frontend/package-lock.json /frontend/ +RUN npm ci --no-audit --no-fund + +COPY frontend/ /frontend/ +RUN npm run build + + +FROM python:3.11-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN python -m pip install --no-cache-dir --upgrade pip + +COPY requirements.txt /app/requirements.txt +RUN python -m pip install --no-cache-dir -r /app/requirements.txt + +RUN mkdir -p /app/data /app/web + +COPY backend/app /app/app +COPY backend/alembic.ini /app/alembic.ini +COPY backend/alembic /app/alembic +COPY --from=frontend-build /frontend/dist /app/web/dist + +EXPOSE 8000 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 0000000..c5eed18 --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,38 @@ +[alembic] +script_location = %(here)s/alembic +prepend_sys_path = %(here)s +sqlalchemy.url = sqlite:///./data/app.db + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S + diff --git a/backend/alembic/env.py b/backend/alembic/env.py new file mode 100644 index 0000000..3515c25 --- /dev/null +++ b/backend/alembic/env.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import os +import sys +from logging.config import fileConfig +from pathlib import Path + +from alembic import context +from sqlalchemy import engine_from_config, pool +from sqlalchemy.engine.url import make_url + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +BACKEND_DIR = Path(__file__).resolve().parents[1] +if str(BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(BACKEND_DIR)) + +target_metadata = None + + +def _get_database_url() -> str: + url = (os.environ.get("DATABASE_URL") or config.get_main_option("sqlalchemy.url") or "").strip() + if not url: + raise RuntimeError("DATABASE_URL (or sqlalchemy.url) is required") + return url.replace("+aiosqlite", "") + + +def _ensure_sqlite_dir(url: str) -> None: + parsed = make_url(url) + if parsed.get_backend_name() != "sqlite": + return + db_path = parsed.database + if not db_path or db_path == ":memory:": + return + Path(db_path).expanduser().resolve().parent.mkdir(parents=True, exist_ok=True) + + +def run_migrations_offline() -> None: + url = _get_database_url() + _ensure_sqlite_dir(url) + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + render_as_batch=url.startswith("sqlite"), + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + url = _get_database_url() + _ensure_sqlite_dir(url) + + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + url=url, + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch=url.startswith("sqlite"), + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() + diff --git a/backend/alembic/versions/.gitkeep b/backend/alembic/versions/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/alembic/versions/.gitkeep @@ -0,0 +1 @@ + diff --git a/backend/alembic/versions/20260210_0001_create_imports.py b/backend/alembic/versions/20260210_0001_create_imports.py new file mode 100644 index 0000000..71bf715 --- /dev/null +++ b/backend/alembic/versions/20260210_0001_create_imports.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0001" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "imports", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("created_by", sa.Text(), nullable=True), + sa.Column("source", sa.Text(), nullable=True), + sa.Column("total", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("accepted", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("success", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("failed", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("detail_json", sa.Text(), nullable=True), + ) + op.create_index("idx_imports_created_at", "imports", ["created_at"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_imports_created_at", table_name="imports") + op.drop_table("imports") + diff --git a/backend/alembic/versions/20260210_0002_create_images.py b/backend/alembic/versions/20260210_0002_create_images.py new file mode 100644 index 0000000..8b514c8 --- /dev/null +++ b/backend/alembic/versions/20260210_0002_create_images.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0002" +down_revision = "20260210_0001" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "images", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column("illust_id", sa.Integer(), nullable=False), + sa.Column("page_index", sa.Integer(), nullable=False), + sa.Column("ext", sa.Text(), nullable=False), + sa.Column("original_url", sa.Text(), nullable=False), + sa.Column("proxy_path", sa.Text(), nullable=False), + sa.Column("random_key", sa.Float(), nullable=False), + sa.Column("width", sa.Integer(), nullable=True), + sa.Column("height", sa.Integer(), nullable=True), + sa.Column("aspect_ratio", sa.Float(), nullable=True), + sa.Column("orientation", sa.Integer(), nullable=True), + sa.Column("x_restrict", sa.Integer(), nullable=True), + sa.Column("ai_type", sa.Integer(), nullable=True), + sa.Column("user_id", sa.Integer(), nullable=True), + sa.Column("user_name", sa.Text(), nullable=True), + sa.Column("title", sa.Text(), nullable=True), + sa.Column("created_at_pixiv", sa.Text(), nullable=True), + sa.Column("status", sa.Integer(), nullable=False, server_default=sa.text("1")), + sa.Column("fail_count", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("last_fail_at", sa.Text(), nullable=True), + sa.Column("last_ok_at", sa.Text(), nullable=True), + sa.Column("last_error_code", sa.Text(), nullable=True), + sa.Column("last_error_msg", sa.Text(), nullable=True), + sa.Column( + "created_import_id", + sa.Integer(), + sa.ForeignKey("imports.id", name="fk_images_import", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "added_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.CheckConstraint("status IN (1,2,3,4)", name="ck_images_status"), + sa.CheckConstraint("random_key >= 0.0 AND random_key < 1.0", name="ck_images_random_key"), + sa.UniqueConstraint("illust_id", "page_index", name="uq_images_illust_page"), + ) + + op.create_index( + "idx_images_filter", + "images", + ["status", "x_restrict", "orientation", "width", "height", "random_key"], + unique=False, + ) + op.create_index( + "idx_images_user_random", + "images", + ["status", "user_id", "random_key"], + unique=False, + ) + op.create_index("idx_images_created_at_pixiv", "images", ["created_at_pixiv"], unique=False) + op.create_index("idx_images_created_import_id", "images", ["created_import_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_images_created_import_id", table_name="images") + op.drop_index("idx_images_created_at_pixiv", table_name="images") + op.drop_index("idx_images_user_random", table_name="images") + op.drop_index("idx_images_filter", table_name="images") + op.drop_table("images") + diff --git a/backend/alembic/versions/20260210_0003_create_tags.py b/backend/alembic/versions/20260210_0003_create_tags.py new file mode 100644 index 0000000..5b8845e --- /dev/null +++ b/backend/alembic/versions/20260210_0003_create_tags.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0003" +down_revision = "20260210_0002" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "tags", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("translated_name", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.UniqueConstraint("name", name="uq_tags_name"), + ) + + +def downgrade() -> None: + op.drop_table("tags") + diff --git a/backend/alembic/versions/20260210_0004_create_image_tags.py b/backend/alembic/versions/20260210_0004_create_image_tags.py new file mode 100644 index 0000000..946cd99 --- /dev/null +++ b/backend/alembic/versions/20260210_0004_create_image_tags.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0004" +down_revision = "20260210_0003" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "image_tags", + sa.Column( + "image_id", + sa.Integer(), + sa.ForeignKey("images.id", name="fk_image_tags_image", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "tag_id", + sa.Integer(), + sa.ForeignKey("tags.id", name="fk_image_tags_tag", ondelete="CASCADE"), + nullable=False, + ), + sa.PrimaryKeyConstraint("image_id", "tag_id"), + ) + op.create_index("idx_image_tags_tag_image", "image_tags", ["tag_id", "image_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_image_tags_tag_image", table_name="image_tags") + op.drop_table("image_tags") + diff --git a/backend/alembic/versions/20260210_0005_create_pixiv_tokens.py b/backend/alembic/versions/20260210_0005_create_pixiv_tokens.py new file mode 100644 index 0000000..12815bc --- /dev/null +++ b/backend/alembic/versions/20260210_0005_create_pixiv_tokens.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0005" +down_revision = "20260210_0004" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "pixiv_tokens", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("label", sa.Text(), nullable=True), + sa.Column("enabled", sa.Integer(), nullable=False, server_default=sa.text("1")), + sa.Column("refresh_token_enc", sa.Text(), nullable=False), + sa.Column("refresh_token_masked", sa.Text(), nullable=False), + sa.Column("weight", sa.Float(), nullable=False, server_default=sa.text("1.0")), + sa.Column("error_count", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("backoff_until", sa.Text(), nullable=True), + sa.Column("last_ok_at", sa.Text(), nullable=True), + sa.Column("last_fail_at", sa.Text(), nullable=True), + sa.Column("last_error_code", sa.Text(), nullable=True), + sa.Column("last_error_msg", sa.Text(), nullable=True), + sa.CheckConstraint("enabled IN (0,1)", name="ck_pixiv_tokens_enabled"), + ) + op.create_index("idx_pixiv_tokens_enabled", "pixiv_tokens", ["enabled"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_pixiv_tokens_enabled", table_name="pixiv_tokens") + op.drop_table("pixiv_tokens") + diff --git a/backend/alembic/versions/20260210_0006_create_proxy_endpoints.py b/backend/alembic/versions/20260210_0006_create_proxy_endpoints.py new file mode 100644 index 0000000..5fe2116 --- /dev/null +++ b/backend/alembic/versions/20260210_0006_create_proxy_endpoints.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0006" +down_revision = "20260210_0005" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "proxy_endpoints", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("scheme", sa.Text(), nullable=False), + sa.Column("host", sa.Text(), nullable=False), + sa.Column("port", sa.Integer(), nullable=False), + sa.Column("username", sa.Text(), nullable=False, server_default=sa.text("''")), + sa.Column("password_enc", sa.Text(), nullable=False, server_default=sa.text("''")), + sa.Column("enabled", sa.Integer(), nullable=False, server_default=sa.text("1")), + sa.Column("source", sa.Text(), nullable=False, server_default=sa.text("'manual'")), + sa.Column("source_ref", sa.Text(), nullable=True), + sa.Column("last_latency_ms", sa.Float(), nullable=True), + sa.Column("last_ok_at", sa.Text(), nullable=True), + sa.Column("last_fail_at", sa.Text(), nullable=True), + sa.Column("success_count", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("failure_count", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("blacklisted_until", sa.Text(), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.UniqueConstraint("scheme", "host", "port", "username", name="uq_proxy_identity"), + sa.CheckConstraint("enabled IN (0,1)", name="ck_proxy_enabled"), + ) + + op.create_index("idx_proxy_endpoints_enabled", "proxy_endpoints", ["enabled"], unique=False) + op.create_index("idx_proxy_endpoints_source", "proxy_endpoints", ["source"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_proxy_endpoints_source", table_name="proxy_endpoints") + op.drop_index("idx_proxy_endpoints_enabled", table_name="proxy_endpoints") + op.drop_table("proxy_endpoints") + diff --git a/backend/alembic/versions/20260210_0007_create_proxy_pools.py b/backend/alembic/versions/20260210_0007_create_proxy_pools.py new file mode 100644 index 0000000..67ef514 --- /dev/null +++ b/backend/alembic/versions/20260210_0007_create_proxy_pools.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0007" +down_revision = "20260210_0006" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "proxy_pools", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("enabled", sa.Integer(), nullable=False, server_default=sa.text("1")), + sa.UniqueConstraint("name", name="uq_proxy_pools_name"), + sa.CheckConstraint("enabled IN (0,1)", name="ck_proxy_pools_enabled"), + ) + op.create_index("idx_proxy_pools_enabled", "proxy_pools", ["enabled"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_proxy_pools_enabled", table_name="proxy_pools") + op.drop_table("proxy_pools") + diff --git a/backend/alembic/versions/20260210_0008_create_proxy_pool_endpoints.py b/backend/alembic/versions/20260210_0008_create_proxy_pool_endpoints.py new file mode 100644 index 0000000..ad1745e --- /dev/null +++ b/backend/alembic/versions/20260210_0008_create_proxy_pool_endpoints.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0008" +down_revision = "20260210_0007" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "proxy_pool_endpoints", + sa.Column( + "pool_id", + sa.Integer(), + sa.ForeignKey("proxy_pools.id", name="fk_ppe_pool", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "endpoint_id", + sa.Integer(), + sa.ForeignKey("proxy_endpoints.id", name="fk_ppe_ep", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("enabled", sa.Integer(), nullable=False, server_default=sa.text("1")), + sa.Column("weight", sa.Integer(), nullable=False, server_default=sa.text("1")), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.PrimaryKeyConstraint("pool_id", "endpoint_id"), + sa.CheckConstraint("enabled IN (0,1)", name="ck_ppe_enabled"), + ) + + op.create_index( + "idx_ppe_pool_enabled", + "proxy_pool_endpoints", + ["pool_id", "enabled"], + unique=False, + ) + op.create_index( + "idx_ppe_endpoint_pool", + "proxy_pool_endpoints", + ["endpoint_id", "pool_id"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("idx_ppe_endpoint_pool", table_name="proxy_pool_endpoints") + op.drop_index("idx_ppe_pool_enabled", table_name="proxy_pool_endpoints") + op.drop_table("proxy_pool_endpoints") + diff --git a/backend/alembic/versions/20260210_0009_create_token_proxy_bindings.py b/backend/alembic/versions/20260210_0009_create_token_proxy_bindings.py new file mode 100644 index 0000000..da49012 --- /dev/null +++ b/backend/alembic/versions/20260210_0009_create_token_proxy_bindings.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0009" +down_revision = "20260210_0008" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "token_proxy_bindings", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "token_id", + sa.Integer(), + sa.ForeignKey("pixiv_tokens.id", name="fk_tpb_token", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "pool_id", + sa.Integer(), + sa.ForeignKey("proxy_pools.id", name="fk_tpb_pool", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "primary_proxy_id", + sa.Integer(), + sa.ForeignKey("proxy_endpoints.id", name="fk_tpb_primary", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column( + "override_proxy_id", + sa.Integer(), + sa.ForeignKey("proxy_endpoints.id", name="fk_tpb_override", ondelete="SET NULL"), + nullable=True, + ), + sa.Column("override_expires_at", sa.Text(), nullable=True), + sa.UniqueConstraint("token_id", "pool_id", name="uq_token_pool"), + ) + + op.create_index("idx_tpb_pool", "token_proxy_bindings", ["pool_id"], unique=False) + op.create_index("idx_tpb_primary", "token_proxy_bindings", ["primary_proxy_id"], unique=False) + op.create_index("idx_tpb_override", "token_proxy_bindings", ["override_proxy_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_tpb_override", table_name="token_proxy_bindings") + op.drop_index("idx_tpb_primary", table_name="token_proxy_bindings") + op.drop_index("idx_tpb_pool", table_name="token_proxy_bindings") + op.drop_table("token_proxy_bindings") + diff --git a/backend/alembic/versions/20260210_0010_create_runtime_settings.py b/backend/alembic/versions/20260210_0010_create_runtime_settings.py new file mode 100644 index 0000000..667dd66 --- /dev/null +++ b/backend/alembic/versions/20260210_0010_create_runtime_settings.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0010" +down_revision = "20260210_0009" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "runtime_settings", + sa.Column("key", sa.Text(), primary_key=True), + sa.Column("value_json", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("updated_by", sa.Text(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_table("runtime_settings") + diff --git a/backend/alembic/versions/20260210_0011_create_jobs.py b/backend/alembic/versions/20260210_0011_create_jobs.py new file mode 100644 index 0000000..1917c36 --- /dev/null +++ b/backend/alembic/versions/20260210_0011_create_jobs.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0011" +down_revision = "20260210_0010" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "jobs", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("type", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("priority", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("run_after", sa.Text(), nullable=True), + sa.Column("attempt", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("max_attempts", sa.Integer(), nullable=False, server_default=sa.text("3")), + sa.Column("payload_json", sa.Text(), nullable=False), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column("locked_by", sa.Text(), nullable=True), + sa.Column("locked_at", sa.Text(), nullable=True), + sa.Column("ref_type", sa.Text(), nullable=True), + sa.Column("ref_id", sa.Text(), nullable=True), + sa.CheckConstraint( + "status IN ('pending','running','paused','canceled','completed','failed','dlq')", + name="ck_jobs_status", + ), + ) + + op.create_index( + "idx_jobs_status_priority", + "jobs", + ["status", "priority", "id"], + unique=False, + ) + op.create_index("idx_jobs_run_after", "jobs", ["run_after"], unique=False) + op.create_index("idx_jobs_ref", "jobs", ["ref_type", "ref_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_jobs_ref", table_name="jobs") + op.drop_index("idx_jobs_run_after", table_name="jobs") + op.drop_index("idx_jobs_status_priority", table_name="jobs") + op.drop_table("jobs") + diff --git a/backend/alembic/versions/20260210_0012_create_hydration_runs.py b/backend/alembic/versions/20260210_0012_create_hydration_runs.py new file mode 100644 index 0000000..6e7daf9 --- /dev/null +++ b/backend/alembic/versions/20260210_0012_create_hydration_runs.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0012" +down_revision = "20260210_0011" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "hydration_runs", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("type", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("criteria_json", sa.Text(), nullable=True), + sa.Column("cursor_json", sa.Text(), nullable=True), + sa.Column("total", sa.Integer(), nullable=True), + sa.Column("processed", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("success", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("failed", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("started_at", sa.Text(), nullable=True), + sa.Column("finished_at", sa.Text(), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.CheckConstraint( + "status IN ('pending','running','paused','canceled','completed','failed')", + name="ck_hr_status", + ), + ) + + op.create_index( + "idx_hr_status_updated", + "hydration_runs", + ["status", "updated_at"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("idx_hr_status_updated", table_name="hydration_runs") + op.drop_table("hydration_runs") + diff --git a/backend/alembic/versions/20260210_0013_create_request_logs.py b/backend/alembic/versions/20260210_0013_create_request_logs.py new file mode 100644 index 0000000..93c6f6f --- /dev/null +++ b/backend/alembic/versions/20260210_0013_create_request_logs.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0013" +down_revision = "20260210_0012" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "request_logs", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("request_id", sa.Text(), nullable=True), + sa.Column("method", sa.Text(), nullable=False), + sa.Column("route", sa.Text(), nullable=False), + sa.Column("status", sa.Integer(), nullable=False), + sa.Column("duration_ms", sa.Integer(), nullable=False), + sa.Column("ip", sa.Text(), nullable=True), + sa.Column("user_agent", sa.Text(), nullable=True), + sa.Column("sample_rate", sa.Float(), nullable=True), + ) + + op.create_index("idx_request_logs_created_at", "request_logs", ["created_at"], unique=False) + op.create_index("idx_request_logs_route", "request_logs", ["route"], unique=False) + op.create_index("idx_request_logs_status", "request_logs", ["status"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_request_logs_status", table_name="request_logs") + op.drop_index("idx_request_logs_route", table_name="request_logs") + op.drop_index("idx_request_logs_created_at", table_name="request_logs") + op.drop_table("request_logs") + diff --git a/backend/alembic/versions/20260210_0014_create_admin_audit.py b/backend/alembic/versions/20260210_0014_create_admin_audit.py new file mode 100644 index 0000000..359f2df --- /dev/null +++ b/backend/alembic/versions/20260210_0014_create_admin_audit.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260210_0014" +down_revision = "20260210_0013" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "admin_audit", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("actor", sa.Text(), nullable=True), + sa.Column("action", sa.Text(), nullable=False), + sa.Column("resource", sa.Text(), nullable=False), + sa.Column("record_id", sa.Text(), nullable=True), + sa.Column("request_id", sa.Text(), nullable=True), + sa.Column("ip", sa.Text(), nullable=True), + sa.Column("user_agent", sa.Text(), nullable=True), + sa.Column("detail_json", sa.Text(), nullable=True), + ) + + op.create_index("idx_admin_audit_created_at", "admin_audit", ["created_at"], unique=False) + op.create_index("idx_admin_audit_action", "admin_audit", ["action"], unique=False) + op.create_index("idx_admin_audit_resource", "admin_audit", ["resource"], unique=False) + op.create_index("idx_admin_audit_record_id", "admin_audit", ["record_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_admin_audit_record_id", table_name="admin_audit") + op.drop_index("idx_admin_audit_resource", table_name="admin_audit") + op.drop_index("idx_admin_audit_action", table_name="admin_audit") + op.drop_index("idx_admin_audit_created_at", table_name="admin_audit") + op.drop_table("admin_audit") + diff --git a/backend/alembic/versions/20260211_0015_create_api_keys.py b/backend/alembic/versions/20260211_0015_create_api_keys.py new file mode 100644 index 0000000..15ede64 --- /dev/null +++ b/backend/alembic/versions/20260211_0015_create_api_keys.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260211_0015" +down_revision = "20260210_0014" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "api_keys", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column( + "created_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column( + "updated_at", + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("key_hash", sa.Text(), nullable=False), + sa.Column("hint", sa.Text(), nullable=False, server_default=sa.text("''")), + sa.Column("enabled", sa.Integer(), nullable=False, server_default=sa.text("1")), + sa.Column("last_used_at", sa.Text(), nullable=True), + sa.UniqueConstraint("name", name="uq_api_keys_name"), + sa.CheckConstraint("enabled IN (0,1)", name="ck_api_keys_enabled"), + ) + + op.create_index("idx_api_keys_enabled", "api_keys", ["enabled"], unique=False) + op.create_index("idx_api_keys_created_at", "api_keys", ["created_at"], unique=False) + + +def downgrade() -> None: + op.drop_index("idx_api_keys_created_at", table_name="api_keys") + op.drop_index("idx_api_keys_enabled", table_name="api_keys") + op.drop_table("api_keys") diff --git a/backend/alembic/versions/20260211_0016_create_fts_search.py b/backend/alembic/versions/20260211_0016_create_fts_search.py new file mode 100644 index 0000000..a1256b9 --- /dev/null +++ b/backend/alembic/versions/20260211_0016_create_fts_search.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260211_0016" +down_revision = "20260211_0015" +branch_labels = None +depends_on = None + + +def _try_create_fts5(conn: sa.Connection, *, table_name: str, columns_sql: str) -> bool: + candidates = [ + f"CREATE VIRTUAL TABLE {table_name} USING fts5({columns_sql}, tokenize='trigram');", + f"CREATE VIRTUAL TABLE {table_name} USING fts5({columns_sql});", + ] + for sql in candidates: + try: + conn.exec_driver_sql(sql) + return True + except Exception: + continue + return False + + +def upgrade() -> None: + conn = op.get_bind() + + tags_ok = _try_create_fts5(conn, table_name="tags_fts", columns_sql="name, translated_name") + if tags_ok: + conn.exec_driver_sql( + "INSERT INTO tags_fts(rowid, name, translated_name) " + "SELECT id, name, COALESCE(translated_name,'') FROM tags;" + ) + conn.exec_driver_sql( + """ +CREATE TRIGGER tags_ai_fts AFTER INSERT ON tags BEGIN + INSERT INTO tags_fts(rowid, name, translated_name) + VALUES (new.id, new.name, COALESCE(new.translated_name,'')); +END; +""".strip() + ) + conn.exec_driver_sql( + """ +CREATE TRIGGER tags_ad_fts AFTER DELETE ON tags BEGIN + DELETE FROM tags_fts WHERE rowid = old.id; +END; +""".strip() + ) + conn.exec_driver_sql( + """ +CREATE TRIGGER tags_au_fts AFTER UPDATE OF name, translated_name ON tags BEGIN + DELETE FROM tags_fts WHERE rowid = old.id; + INSERT INTO tags_fts(rowid, name, translated_name) + VALUES (new.id, new.name, COALESCE(new.translated_name,'')); +END; +""".strip() + ) + + authors_ok = _try_create_fts5(conn, table_name="authors_fts", columns_sql="user_name") + if authors_ok: + conn.exec_driver_sql( + "INSERT INTO authors_fts(rowid, user_name) " + "SELECT user_id, MAX(user_name) " + "FROM images " + "WHERE user_id IS NOT NULL AND user_name IS NOT NULL AND status=1 " + "GROUP BY user_id;" + ) + conn.exec_driver_sql( + """ +CREATE TRIGGER images_ai_authors_fts AFTER INSERT ON images +WHEN new.user_id IS NOT NULL +BEGIN + DELETE FROM authors_fts WHERE rowid = new.user_id; + INSERT INTO authors_fts(rowid, user_name) + SELECT new.user_id, MAX(user_name) + FROM images + WHERE user_id = new.user_id AND status = 1 AND user_name IS NOT NULL + HAVING MAX(user_name) IS NOT NULL; +END; +""".strip() + ) + conn.exec_driver_sql( + """ +CREATE TRIGGER images_au_authors_fts AFTER UPDATE OF user_id, user_name, status ON images +BEGIN + DELETE FROM authors_fts WHERE rowid = old.user_id; + INSERT INTO authors_fts(rowid, user_name) + SELECT old.user_id, MAX(user_name) + FROM images + WHERE old.user_id IS NOT NULL AND user_id = old.user_id AND status = 1 AND user_name IS NOT NULL + HAVING MAX(user_name) IS NOT NULL; + + DELETE FROM authors_fts + WHERE rowid = new.user_id AND new.user_id IS NOT NULL AND (old.user_id IS NULL OR new.user_id != old.user_id); + INSERT INTO authors_fts(rowid, user_name) + SELECT new.user_id, MAX(user_name) + FROM images + WHERE new.user_id IS NOT NULL + AND (old.user_id IS NULL OR new.user_id != old.user_id) + AND user_id = new.user_id AND status = 1 AND user_name IS NOT NULL + HAVING MAX(user_name) IS NOT NULL; +END; +""".strip() + ) + conn.exec_driver_sql( + """ +CREATE TRIGGER images_ad_authors_fts AFTER DELETE ON images +WHEN old.user_id IS NOT NULL +BEGIN + DELETE FROM authors_fts WHERE rowid = old.user_id; + INSERT INTO authors_fts(rowid, user_name) + SELECT old.user_id, MAX(user_name) + FROM images + WHERE user_id = old.user_id AND status = 1 AND user_name IS NOT NULL + HAVING MAX(user_name) IS NOT NULL; +END; +""".strip() + ) + + +def downgrade() -> None: + conn = op.get_bind() + + conn.exec_driver_sql("DROP TRIGGER IF EXISTS images_ad_authors_fts;") + conn.exec_driver_sql("DROP TRIGGER IF EXISTS images_au_authors_fts;") + conn.exec_driver_sql("DROP TRIGGER IF EXISTS images_ai_authors_fts;") + conn.exec_driver_sql("DROP TABLE IF EXISTS authors_fts;") + + conn.exec_driver_sql("DROP TRIGGER IF EXISTS tags_au_fts;") + conn.exec_driver_sql("DROP TRIGGER IF EXISTS tags_ad_fts;") + conn.exec_driver_sql("DROP TRIGGER IF EXISTS tags_ai_fts;") + conn.exec_driver_sql("DROP TABLE IF EXISTS tags_fts;") + diff --git a/backend/alembic/versions/20260215_0017_add_image_popularity_fields.py b/backend/alembic/versions/20260215_0017_add_image_popularity_fields.py new file mode 100644 index 0000000..e331d5d --- /dev/null +++ b/backend/alembic/versions/20260215_0017_add_image_popularity_fields.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260215_0017" +down_revision = "20260211_0016" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("images", sa.Column("bookmark_count", sa.Integer(), nullable=True)) + op.add_column("images", sa.Column("view_count", sa.Integer(), nullable=True)) + op.add_column("images", sa.Column("comment_count", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table("images") as batch_op: + batch_op.drop_column("comment_count") + batch_op.drop_column("view_count") + batch_op.drop_column("bookmark_count") + diff --git a/backend/alembic/versions/20260218_0018_add_images_illust_type.py b/backend/alembic/versions/20260218_0018_add_images_illust_type.py new file mode 100644 index 0000000..45abcbb --- /dev/null +++ b/backend/alembic/versions/20260218_0018_add_images_illust_type.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "20260218_0018" +down_revision = "20260215_0017" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("images", sa.Column("illust_type", sa.Integer(), nullable=True)) + op.create_index("idx_images_illust_type_random", "images", ["status", "illust_type", "random_key"]) + + +def downgrade() -> None: + op.drop_index("idx_images_illust_type_random", table_name="images") + with op.batch_alter_table("images") as batch_op: + batch_op.drop_column("illust_type") + diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/app/__init__.py @@ -0,0 +1 @@ + diff --git a/backend/app/api/admin/__init__.py b/backend/app/api/admin/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/app/api/admin/__init__.py @@ -0,0 +1 @@ + diff --git a/backend/app/api/admin/api_keys.py b/backend/app/api/admin/api_keys.py new file mode 100644 index 0000000..3ddbd0c --- /dev/null +++ b/backend/app/api/admin/api_keys.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +from typing import Any + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Request +from sqlalchemy.exc import IntegrityError + +from app.api.admin.deps import get_admin_claims +from app.core.api_keys import api_key_hint, hmac_sha256_hex +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.core.time import iso_utc_ms +from app.db.models.api_keys import ApiKey +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + +router = APIRouter() + + +def _as_bool(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"true", "1", "yes", "y", "on"}: + return True + if v in {"false", "0", "no", "n", "off"}: + return False + return None + + +async def _load_json_object(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + return data + + +@router.get("/api-keys") +async def list_api_keys( + request: Request, + limit: int = 50, + cursor: str | None = None, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if limit < 1 or limit > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported limit", status_code=400) + + cursor_i: int | None = None + cursor_raw = (cursor or "").strip() + if cursor_raw: + if not cursor_raw.isdigit(): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + cursor_i = int(cursor_raw) + if cursor_i <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + stmt = sa.select(ApiKey).order_by(ApiKey.id.desc()).limit(limit + 1) + if cursor_i is not None: + stmt = stmt.where(ApiKey.id < cursor_i) + + async with Session() as session: + rows = ((await session.execute(stmt)).scalars().all()) + + items_rows = rows[:limit] + next_cursor = int(items_rows[-1].id) if len(rows) > limit and items_rows else None + + items = [ + { + "id": str(row.id), + "name": row.name, + "description": row.description, + "enabled": bool(row.enabled), + "hint": row.hint, + "created_at": row.created_at, + "updated_at": row.updated_at, + "last_used_at": row.last_used_at, + } + for row in items_rows + ] + + return { + "ok": True, + "items": items, + "next_cursor": str(next_cursor) if next_cursor is not None else "", + "request_id": rid, + } + + +@router.post("/api-keys") +async def create_api_key( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + + body = await _load_json_object(request) + name = str(body.get("name") or "").strip() + api_key = str(body.get("api_key") or "").strip() + description = str(body.get("description") or "").strip() or None + enabled_v = _as_bool(body.get("enabled")) + enabled = bool(enabled_v) if enabled_v is not None else True + + if not name or len(name) > 100: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid name", status_code=400) + if not api_key or len(api_key) < 20 or len(api_key) > 500: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid api_key", status_code=400) + if description is not None and len(description) > 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid description", status_code=400) + + settings = request.app.state.settings + try: + key_hash = hmac_sha256_hex(secret_key=settings.secret_key, message=api_key) + except Exception as exc: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="API key hashing not configured", status_code=500) from exc + + hint = api_key_hint(api_key) + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> int: + async with Session() as session: + row = ApiKey( + name=name, + description=description, + key_hash=key_hash, + hint=hint, + enabled=1 if enabled else 0, + updated_at=now, + ) + session.add(row) + try: + await session.commit() + except IntegrityError as exc: + await session.rollback() + raise ApiError(code=ErrorCode.BAD_REQUEST, message="API key name exists", status_code=400) from exc + await session.refresh(row) + return int(row.id) + + api_key_id = await with_sqlite_busy_retry(_op) + return {"ok": True, "api_key_id": str(api_key_id), "hint": hint, "request_id": rid} + + +@router.put("/api-keys/{api_key_id}") +async def update_api_key( + api_key_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if api_key_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid api_key_id", status_code=400) + + rid = get_or_create_request_id(request) + body = await _load_json_object(request) + + enabled_v = _as_bool(body.get("enabled")) if "enabled" in body else None + description_raw = body.get("description") if "description" in body else None + description = str(description_raw).strip() if isinstance(description_raw, str) else None + if description is not None and not description: + description = None + if description is not None and len(description) > 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid description", status_code=400) + + if enabled_v is None and "description" not in body: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing fields", status_code=400) + + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + row = await session.get(ApiKey, int(api_key_id)) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="API key not found", status_code=404) + if enabled_v is not None: + row.enabled = 1 if bool(enabled_v) else 0 + if "description" in body: + row.description = description + row.updated_at = now + await session.commit() + + return {"ok": True, "api_key_id": str(api_key_id), "request_id": rid} + + return await with_sqlite_busy_retry(_op) + diff --git a/backend/app/api/admin/audit.py b/backend/app/api/admin/audit.py new file mode 100644 index 0000000..a678298 --- /dev/null +++ b/backend/app/api/admin/audit.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import json +from typing import Any + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Request + +from app.api.admin.deps import get_admin_claims +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.db.models.admin_audit import AdminAudit +from app.db.session import create_sessionmaker + +router = APIRouter() + + +@router.get("/audit") +async def list_admin_audit( + request: Request, + limit: int = 50, + cursor: str | None = None, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if limit < 1 or limit > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported limit", status_code=400) + + cursor_i: int | None = None + cursor_raw = (cursor or "").strip() + if cursor_raw: + if not cursor_raw.isdigit(): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + cursor_i = int(cursor_raw) + if cursor_i <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + stmt = sa.select(AdminAudit).order_by(AdminAudit.id.desc()).limit(limit + 1) + if cursor_i is not None: + stmt = stmt.where(AdminAudit.id < cursor_i) + + async with Session() as session: + rows = (await session.execute(stmt)).scalars().all() + + items_rows = rows[:limit] + next_cursor = int(items_rows[-1].id) if len(rows) > limit and items_rows else None + + def _parse_detail(text: str | None) -> dict[str, Any] | None: + raw = str(text or "").strip() + if not raw: + return None + try: + data = json.loads(raw) + except Exception: + return {"raw": raw} + return data if isinstance(data, dict) else {"value": data} + + items = [ + { + "id": str(row.id), + "created_at": row.created_at, + "actor": row.actor, + "action": row.action, + "resource": row.resource, + "record_id": row.record_id, + "request_id": row.request_id, + "detail_json": _parse_detail(row.detail_json), + } + for row in items_rows + ] + + return { + "ok": True, + "items": items, + "next_cursor": str(next_cursor) if next_cursor is not None else "", + "request_id": rid, + } + diff --git a/backend/app/api/admin/auth.py b/backend/app/api/admin/auth.py new file mode 100644 index 0000000..adfc84f --- /dev/null +++ b/backend/app/api/admin/auth.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import hmac +from typing import Any + +from fastapi import APIRouter, Request + +from app.api.admin.deps import get_admin_claims +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.core.security import create_jwt +from fastapi import Depends + +router = APIRouter() + + +async def _load_login_json(request: Request) -> tuple[str, str]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + username = str(data.get("username") or "").strip() + password = str(data.get("password") or "") + if not username or not password: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing credentials", status_code=400) + + return username, password + + +@router.post("/login") +async def login(request: Request) -> dict[str, Any]: + username, password = await _load_login_json(request) + settings = request.app.state.settings + + if username != settings.admin_username or not hmac.compare_digest(password, settings.admin_password): + raise ApiError(code=ErrorCode.UNAUTHORIZED, message="Invalid credentials", status_code=401) + + token = create_jwt(secret_key=settings.secret_key, subject=settings.admin_username, ttl_s=3600) + rid = get_or_create_request_id(request) + return {"ok": True, "token": token, "request_id": rid} + + +@router.post("/logout") +async def logout(request: Request, _claims: dict[str, Any] = Depends(get_admin_claims)) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + return {"ok": True, "request_id": rid} diff --git a/backend/app/api/admin/bindings.py b/backend/app/api/admin/bindings.py new file mode 100644 index 0000000..28ec761 --- /dev/null +++ b/backend/app/api/admin/bindings.py @@ -0,0 +1,403 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Request +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.orm import aliased + +from app.api.admin.deps import get_admin_claims +from app.core.bindings_recompute import recompute_token_proxy_bindings +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.core.time import iso_utc_ms +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + +router = APIRouter() + + +def _parse_bool(value: Any, *, default: bool) -> bool: + if value is None: + return bool(default) + if isinstance(value, bool): + return bool(value) + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return bool(default) + + +def _fnv1a64(text: str) -> int: + h = 14695981039346656037 + prime = 1099511628211 + for b in text.encode("utf-8"): + h ^= b + h = (h * prime) & 0xFFFFFFFFFFFFFFFF + return h + + +def _rendezvous_proxy_order(*, token_id: int, proxy_ids: list[int], salt: str) -> list[int]: + scored = [(_fnv1a64(f"{token_id}|{pid}|{salt}"), pid) for pid in proxy_ids] + scored.sort(key=lambda x: (-x[0], x[1])) + return [pid for _, pid in scored] + + +def _compute_primary_assignments( + *, + token_ids: list[int], + proxy_ids: list[int], + capacity_by_proxy_id: dict[int, int], + salt: str, +) -> dict[int, int]: + remaining = {pid: int(capacity_by_proxy_id.get(int(pid), 0)) for pid in proxy_ids} + out: dict[int, int] = {} + for token_id in token_ids: + for pid in _rendezvous_proxy_order(token_id=token_id, proxy_ids=proxy_ids, salt=salt): + if remaining.get(pid, 0) > 0: + out[token_id] = pid + remaining[pid] -= 1 + break + return out + + +def _compute_primary_assignments_soft( + *, + token_ids: list[int], + proxy_ids: list[int], + capacity_by_proxy_id: dict[int, int], + salt: str, +) -> tuple[dict[int, int], int]: + out = _compute_primary_assignments( + token_ids=token_ids, + proxy_ids=proxy_ids, + capacity_by_proxy_id=capacity_by_proxy_id, + salt=salt, + ) + + over_capacity = 0 + for token_id in token_ids: + if token_id in out: + continue + order = _rendezvous_proxy_order(token_id=token_id, proxy_ids=proxy_ids, salt=salt) + if not order: + continue + out[token_id] = int(order[0]) + over_capacity += 1 + + return out, int(over_capacity) + + +async def _load_recompute_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + try: + pool_id = int(data.get("pool_id")) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid pool_id", status_code=400) from exc + if pool_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid pool_id", status_code=400) + + raw_max = data.get("max_tokens_per_proxy", 2) + try: + max_tokens_per_proxy = int(raw_max) + except Exception as exc: + raise ApiError( + code=ErrorCode.BAD_REQUEST, + message="Invalid max_tokens_per_proxy", + status_code=400, + ) from exc + + if max_tokens_per_proxy <= 0 or max_tokens_per_proxy > 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid max_tokens_per_proxy", status_code=400) + + strict = _parse_bool(data.get("strict"), default=True) + + return {"pool_id": pool_id, "max_tokens_per_proxy": max_tokens_per_proxy, "strict": strict} + + +async def _load_override_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + try: + override_proxy_id = int(data.get("override_proxy_id")) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid override_proxy_id", status_code=400) from exc + if override_proxy_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid override_proxy_id", status_code=400) + + try: + ttl_ms = int(data.get("ttl_ms")) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid ttl_ms", status_code=400) from exc + if ttl_ms <= 0 or ttl_ms > 30 * 24 * 60 * 60 * 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid ttl_ms", status_code=400) + + reason = str(data.get("reason") or "").strip() + if reason and len(reason) > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid reason", status_code=400) + + return {"override_proxy_id": override_proxy_id, "ttl_ms": ttl_ms, "reason": reason} + + +@router.get("/bindings") +async def list_bindings( + request: Request, + pool_id: int, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if int(pool_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid pool_id", status_code=400) + + rid = get_or_create_request_id(request) + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + Primary = aliased(ProxyEndpoint) + Override = aliased(ProxyEndpoint) + + async with Session() as session: + rows = ( + ( + await session.execute( + sa.select(TokenProxyBinding, PixivToken.label, ProxyPool.name, Primary, Override) + .join(PixivToken, PixivToken.id == TokenProxyBinding.token_id) + .join(ProxyPool, ProxyPool.id == TokenProxyBinding.pool_id) + .join(Primary, Primary.id == TokenProxyBinding.primary_proxy_id) + .outerjoin(Override, Override.id == TokenProxyBinding.override_proxy_id) + .where(TokenProxyBinding.pool_id == int(pool_id)) + .order_by(TokenProxyBinding.id.asc()) + ) + ) + .all() + ) + + counts = ( + ( + await session.execute( + sa.select( + sa.func.count(ProxyPoolEndpoint.endpoint_id), + sa.func.sum( + sa.case( + (sa.and_(ProxyPoolEndpoint.enabled == 1, ProxyEndpoint.enabled == 1), 1), + else_=0, + ) + ), + ) + .select_from(ProxyPoolEndpoint) + .join(ProxyEndpoint, ProxyEndpoint.id == ProxyPoolEndpoint.endpoint_id) + .where(ProxyPoolEndpoint.pool_id == int(pool_id)) + ) + ) + .first() + ) + endpoints_total = int(counts[0] or 0) if counts is not None else 0 + endpoints_enabled = int(counts[1] or 0) if counts is not None else 0 + + items: list[dict[str, Any]] = [] + for binding, token_label, pool_name, primary_proxy, override_proxy in rows: + override_active = False + if binding.override_proxy_id is not None and binding.override_expires_at: + override_active = str(binding.override_expires_at) > now + + effective_proxy_id = binding.override_proxy_id if override_active else binding.primary_proxy_id + effective_mode = "override" if override_active else "primary" + + items.append( + { + "id": str(binding.id), + "created_at": binding.created_at, + "updated_at": binding.updated_at, + "token": {"id": str(binding.token_id), "label": token_label}, + "pool": {"id": str(binding.pool_id), "name": pool_name}, + "primary_proxy": { + "id": str(primary_proxy.id), + "scheme": primary_proxy.scheme, + "host": primary_proxy.host, + "port": int(primary_proxy.port), + "username": primary_proxy.username, + }, + "override_proxy": ( + { + "id": str(override_proxy.id), + "scheme": override_proxy.scheme, + "host": override_proxy.host, + "port": int(override_proxy.port), + "username": override_proxy.username, + } + if override_proxy is not None + else None + ), + "override_expires_at": binding.override_expires_at, + "effective_proxy_id": str(effective_proxy_id), + "effective_mode": effective_mode, + } + ) + + return { + "ok": True, + "items": items, + "summary": { + "pool_id": str(pool_id), + "pool_endpoints_total": int(endpoints_total), + "pool_endpoints_enabled": int(endpoints_enabled), + }, + "request_id": rid, + } + + +@router.post("/bindings/recompute") +async def recompute_bindings( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + now = iso_utc_ms() + body = await _load_recompute_json(request) + + pool_id = int(body["pool_id"]) + max_tokens_per_proxy = int(body["max_tokens_per_proxy"]) + strict = bool(body.get("strict", True)) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + pool = await session.get(ProxyPool, pool_id) + if pool is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Proxy pool not found", status_code=404) + + result = await recompute_token_proxy_bindings( + session, + pool_id=int(pool_id), + now=str(now), + max_tokens_per_proxy=int(max_tokens_per_proxy), + strict=bool(strict), + ) + await session.commit() + + return {"ok": True, "pool_id": str(pool_id), "request_id": rid, **result} + + return await with_sqlite_busy_retry(_op) + + +@router.post("/bindings/{binding_id}/override") +async def set_binding_override( + binding_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if binding_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid binding id", status_code=400) + + rid = get_or_create_request_id(request) + now = iso_utc_ms() + body = await _load_override_json(request) + + override_proxy_id = int(body["override_proxy_id"]) + ttl_ms = int(body["ttl_ms"]) + _reason = str(body["reason"] or "") + + expires_at = iso_utc_ms(datetime.now(timezone.utc) + timedelta(milliseconds=ttl_ms)) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + binding = await session.get(TokenProxyBinding, binding_id) + if binding is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Binding not found", status_code=404) + + proxy = await session.get(ProxyEndpoint, override_proxy_id) + if proxy is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Proxy endpoint not found", status_code=404) + if not bool(proxy.enabled): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Proxy endpoint disabled", status_code=400) + + in_pool = ( + await session.execute( + sa.select(ProxyPoolEndpoint) + .where(ProxyPoolEndpoint.pool_id == int(binding.pool_id)) + .where(ProxyPoolEndpoint.endpoint_id == int(override_proxy_id)) + .where(ProxyPoolEndpoint.enabled == 1) + ) + ).scalars().first() + if in_pool is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Override proxy not in pool", status_code=400) + + binding.override_proxy_id = int(override_proxy_id) + binding.override_expires_at = expires_at + binding.updated_at = now + await session.commit() + + return { + "ok": True, + "binding_id": str(binding_id), + "override_proxy_id": str(override_proxy_id), + "override_expires_at": expires_at, + "request_id": rid, + } + + return await with_sqlite_busy_retry(_op) + + +@router.post("/bindings/{binding_id}/clear-override") +async def clear_binding_override( + binding_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if binding_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid binding id", status_code=400) + + rid = get_or_create_request_id(request) + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + binding = await session.get(TokenProxyBinding, binding_id) + if binding is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Binding not found", status_code=404) + + binding.override_proxy_id = None + binding.override_expires_at = None + binding.updated_at = now + await session.commit() + + return {"ok": True, "binding_id": str(binding_id), "request_id": rid} + + return await with_sqlite_busy_retry(_op) diff --git a/backend/app/api/admin/deps.py b/backend/app/api/admin/deps.py new file mode 100644 index 0000000..91bfd92 --- /dev/null +++ b/backend/app/api/admin/deps.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import Request + +from app.core.security import require_admin + + +def get_admin_claims(request: Request) -> dict[str, Any]: + settings = request.app.state.settings + return require_admin( + request.headers, + secret_key=settings.secret_key, + admin_username=settings.admin_username, + ) + diff --git a/backend/app/api/admin/hydration_runs.py b/backend/app/api/admin/hydration_runs.py new file mode 100644 index 0000000..e2f5ac2 --- /dev/null +++ b/backend/app/api/admin/hydration_runs.py @@ -0,0 +1,500 @@ +from __future__ import annotations + +import json +from typing import Any + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Request + +from app.api.admin.deps import get_admin_claims +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.core.time import iso_utc_ms +from app.db.models.hydration_runs import HydrationRun +from app.db.models.images import Image +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + +router = APIRouter() + +_ALLOWED_RUN_STATUSES = {"pending", "running", "paused", "canceled", "completed", "failed"} + + +def _parse_json_dict(value: str | None) -> dict[str, Any]: + raw = str(value or "").strip() + if not raw: + return {} + try: + loaded = json.loads(raw) + except Exception: + return {} + if isinstance(loaded, dict): + return loaded + return {} + + +def _serialize_job(job: JobRow | None) -> dict[str, Any] | None: + if job is None: + return None + return { + "id": str(job.id), + "status": job.status, + "attempt": int(job.attempt), + "max_attempts": int(job.max_attempts), + "run_after": job.run_after, + "last_error": job.last_error, + "locked_by": job.locked_by, + "locked_at": job.locked_at, + "updated_at": job.updated_at, + } + + +def _serialize_run(run: HydrationRun, *, latest_job: JobRow | None = None) -> dict[str, Any]: + return { + "id": str(run.id), + "type": run.type, + "status": run.status, + "criteria": _parse_json_dict(run.criteria_json), + "cursor": _parse_json_dict(run.cursor_json), + "total": int(run.total) if run.total is not None else None, + "processed": int(run.processed), + "success": int(run.success), + "failed": int(run.failed), + "started_at": run.started_at, + "finished_at": run.finished_at, + "last_error": run.last_error, + "created_at": run.created_at, + "updated_at": run.updated_at, + "latest_job": _serialize_job(latest_job), + } + + +async def _latest_jobs_by_run_ids(session, *, run_ids: list[str]) -> dict[str, JobRow]: + ids = [str(v).strip() for v in run_ids if str(v).strip()] + if not ids: + return {} + + latest_per_ref = ( + sa.select(JobRow.ref_id.label("ref_id"), sa.func.max(JobRow.id).label("max_job_id")) + .where(JobRow.ref_type == "hydration_run", JobRow.ref_id.in_(ids)) + .group_by(JobRow.ref_id) + .subquery() + ) + + rows = ( + await session.execute( + sa.select(JobRow, latest_per_ref.c.ref_id) + .join(latest_per_ref, JobRow.id == latest_per_ref.c.max_job_id) + .order_by(JobRow.id.desc()) + ) + ).all() + + out: dict[str, JobRow] = {} + for job, ref_id in rows: + key = str(ref_id or "").strip() + if not key: + continue + out[key] = job + return out + + +async def _load_create_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + run_type = str(data.get("type") or "backfill").strip().lower() or "backfill" + if run_type not in {"backfill", "manual"}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid type", status_code=400) + + criteria = data.get("criteria") + if criteria is None: + criteria = {} + if not isinstance(criteria, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid criteria", status_code=400) + + return {"type": run_type, "criteria": criteria} + + +async def _load_manual_job_json(request: Request) -> dict[str, int | None]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + illust_id_raw = data.get("illust_id") + image_id_raw = data.get("image_id") + + if illust_id_raw is None and image_id_raw is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Either illust_id or image_id is required", status_code=400) + + if illust_id_raw is not None and image_id_raw is not None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Provide only one of illust_id or image_id", status_code=400) + + illust_id: int | None = None + image_id: int | None = None + if illust_id_raw is not None: + try: + illust_id = int(illust_id_raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid illust_id", status_code=400) from exc + if illust_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid illust_id", status_code=400) + + if image_id_raw is not None: + try: + image_id = int(image_id_raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid image_id", status_code=400) from exc + if image_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid image_id", status_code=400) + + return {"illust_id": illust_id, "image_id": image_id} + + +@router.get("/hydration-runs") +async def list_hydration_runs( + request: Request, + limit: int = 20, + cursor: str | None = None, + status: str | None = None, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if limit < 1 or limit > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported limit", status_code=400) + + cursor_i: int | None = None + cursor_raw = str(cursor or "").strip() + if cursor_raw: + if not cursor_raw.isdigit(): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + cursor_i = int(cursor_raw) + if cursor_i <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + + status_norm = str(status or "").strip().lower() or None + if status_norm is not None and status_norm not in _ALLOWED_RUN_STATUSES: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported status", status_code=400) + + rid = get_or_create_request_id(request) + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + stmt = sa.select(HydrationRun).order_by(HydrationRun.id.desc()).limit(limit + 1) + if cursor_i is not None: + stmt = stmt.where(HydrationRun.id < cursor_i) + if status_norm is not None: + stmt = stmt.where(HydrationRun.status == status_norm) + + rows = (await session.execute(stmt)).scalars().all() + current_rows = rows[:limit] + run_ids = [str(int(r.id)) for r in current_rows] + jobs_by_run_id = await _latest_jobs_by_run_ids(session, run_ids=run_ids) + + next_cursor = int(current_rows[-1].id) if len(rows) > limit and current_rows else None + items = [_serialize_run(row, latest_job=jobs_by_run_id.get(str(int(row.id)))) for row in current_rows] + + return { + "ok": True, + "items": items, + "next_cursor": str(next_cursor) if next_cursor is not None else "", + "request_id": rid, + } + + +@router.get("/hydration-runs/{run_id}") +async def get_hydration_run( + run_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if run_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid hydration_run id", status_code=400) + + rid = get_or_create_request_id(request) + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + run = await session.get(HydrationRun, run_id) + if run is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Hydration run not found", status_code=404) + + jobs = await _latest_jobs_by_run_ids(session, run_ids=[str(run_id)]) + item = _serialize_run(run, latest_job=jobs.get(str(run_id))) + + return {"ok": True, "item": item, "request_id": rid} + + +@router.post("/hydration-runs/manual") +async def create_manual_hydration_job( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + now = iso_utc_ms() + body = await _load_manual_job_json(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + illust_id = body.get("illust_id") + image_id = body.get("image_id") + if illust_id is None and image_id is not None: + image = await session.get(Image, int(image_id)) + if image is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Image not found", status_code=404) + illust_id = int(image.illust_id) + + if illust_id is None or int(illust_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid illust_id", status_code=400) + + existing = ( + ( + await session.execute( + sa.select(JobRow) + .where( + JobRow.type == "hydrate_metadata", + JobRow.ref_type == "manual_hydrate", + JobRow.ref_id == str(int(illust_id)), + JobRow.status.in_(["pending", "running"]), + ) + .order_by(JobRow.id.desc()) + .limit(1) + ) + ) + .scalars() + .first() + ) + + if existing is not None: + return { + "ok": True, + "created": False, + "job_id": str(int(existing.id)), + "illust_id": str(int(illust_id)), + "request_id": rid, + } + + job = JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps({"illust_id": int(illust_id)}, separators=(",", ":"), ensure_ascii=False), + last_error=None, + priority=0, + run_after=None, + attempt=0, + max_attempts=3, + locked_by=None, + locked_at=None, + ref_type="manual_hydrate", + ref_id=str(int(illust_id)), + updated_at=now, + ) + session.add(job) + await session.flush() + await session.commit() + + return { + "ok": True, + "created": True, + "job_id": str(int(job.id)), + "illust_id": str(int(illust_id)), + "request_id": rid, + } + + return await with_sqlite_busy_retry(_op) + + +@router.post("/hydration-runs") +async def create_hydration_run( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + now = iso_utc_ms() + body = await _load_create_json(request) + + run_type = str(body["type"]) + criteria = dict(body["criteria"]) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> tuple[int, int]: + async with Session() as session: + run = HydrationRun( + type=run_type, + status="pending", + criteria_json=json.dumps(criteria, separators=(",", ":"), ensure_ascii=False), + cursor_json=None, + total=None, + processed=0, + success=0, + failed=0, + started_at=None, + finished_at=None, + last_error=None, + updated_at=now, + ) + session.add(run) + await session.flush() + + job = JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps( + {"hydration_run_id": int(run.id), "criteria": criteria}, + separators=(",", ":"), + ensure_ascii=False, + ), + last_error=None, + priority=0, + run_after=None, + attempt=0, + max_attempts=3, + locked_by=None, + locked_at=None, + ref_type="hydration_run", + ref_id=str(int(run.id)), + updated_at=now, + ) + session.add(job) + await session.flush() + await session.commit() + return int(run.id), int(job.id) + + run_id, job_id = await with_sqlite_busy_retry(_op) + + return { + "ok": True, + "hydration_run_id": str(run_id), + "job_id": str(job_id), + "request_id": rid, + } + + +async def _set_run_and_job_status( + request: Request, + *, + run_id: int, + target_status: str, + allowed_from: set[str], + job_status: str, +) -> dict[str, Any]: + if run_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid hydration_run id", status_code=400) + + rid = get_or_create_request_id(request) + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + run = await session.get(HydrationRun, run_id) + if run is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Hydration run not found", status_code=404) + + if str(run.status) not in allowed_from: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported status transition", status_code=400) + + run.status = target_status + run.updated_at = now + if target_status in {"canceled", "completed", "failed"} and run.finished_at is None: + run.finished_at = now + + job = ( + ( + await session.execute( + sa.select(JobRow) + .where(JobRow.ref_type == "hydration_run", JobRow.ref_id == str(run_id)) + .order_by(JobRow.id.desc()) + .limit(1) + ) + ) + .scalars() + .first() + ) + if job is not None: + job.status = job_status + if job_status in {"pending", "canceled", "paused", "dlq"}: + job.run_after = None + job.locked_by = None + job.locked_at = None + job.updated_at = now + + await session.commit() + + return { + "ok": True, + "hydration_run_id": str(run_id), + "status": target_status, + "job_status": job_status if job is not None else "", + "request_id": rid, + } + + return await with_sqlite_busy_retry(_op) + + +@router.post("/hydration-runs/{run_id}/pause") +async def pause_hydration_run( + run_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + return await _set_run_and_job_status( + request, + run_id=run_id, + target_status="paused", + allowed_from={"pending", "running"}, + job_status="paused", + ) + + +@router.post("/hydration-runs/{run_id}/resume") +async def resume_hydration_run( + run_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + return await _set_run_and_job_status( + request, + run_id=run_id, + target_status="pending", + allowed_from={"paused"}, + job_status="pending", + ) + + +@router.post("/hydration-runs/{run_id}/cancel") +async def cancel_hydration_run( + run_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + return await _set_run_and_job_status( + request, + run_id=run_id, + target_status="canceled", + allowed_from={"pending", "running", "paused"}, + job_status="canceled", + ) diff --git a/backend/app/api/admin/images.py b/backend/app/api/admin/images.py new file mode 100644 index 0000000..e445322 --- /dev/null +++ b/backend/app/api/admin/images.py @@ -0,0 +1,343 @@ +from __future__ import annotations + +from typing import Any + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Query, Request + +from app.api.admin.deps import get_admin_claims +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + +router = APIRouter() + +_ALLOWED_MISSING = {"tags", "geometry", "r18", "ai", "illust_type", "user", "title", "created_at", "popularity"} + + +def _parse_missing(values: list[str] | None) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for raw in values or []: + for part in str(raw or "").replace(",", "|").split("|"): + key = part.strip().lower() + if not key or key in seen: + continue + if key not in _ALLOWED_MISSING: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported missing", status_code=400) + seen.add(key) + out.append(key) + return out + + +@router.get("/images") +async def list_admin_images( + request: Request, + limit: int = 50, + cursor: str | None = None, + missing: list[str] | None = Query(default=None), + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if limit < 1 or limit > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported limit", status_code=400) + + cursor_i: int | None = None + cursor_raw = (cursor or "").strip() + if cursor_raw: + if not cursor_raw.isdigit(): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + cursor_i = int(cursor_raw) + if cursor_i <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + + missing_keys = _parse_missing(missing) + + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + tag_counts = ( + sa.select(ImageTag.image_id.label("image_id"), sa.func.count().label("tag_count")) + .group_by(ImageTag.image_id) + .subquery() + ) + tag_count_col = sa.func.coalesce(tag_counts.c.tag_count, 0).label("tag_count") + + stmt = ( + sa.select(Image, tag_count_col) + .outerjoin(tag_counts, tag_counts.c.image_id == Image.id) + .where(Image.status == 1) + .order_by(Image.id.desc()) + .limit(int(limit) + 1) + ) + if cursor_i is not None: + stmt = stmt.where(Image.id < int(cursor_i)) + + for key in missing_keys: + if key == "tags": + stmt = stmt.where(tag_count_col == 0) + elif key == "geometry": + stmt = stmt.where((Image.width.is_(None)) | (Image.height.is_(None))) + elif key == "r18": + stmt = stmt.where(Image.x_restrict.is_(None)) + elif key == "ai": + stmt = stmt.where(Image.ai_type.is_(None)) + elif key == "illust_type": + stmt = stmt.where(Image.illust_type.is_(None)) + elif key == "user": + stmt = stmt.where(Image.user_id.is_(None)) + elif key == "title": + stmt = stmt.where((Image.title.is_(None)) | (sa.func.trim(Image.title) == "")) + elif key == "created_at": + stmt = stmt.where((Image.created_at_pixiv.is_(None)) | (sa.func.trim(Image.created_at_pixiv) == "")) + elif key == "popularity": + stmt = stmt.where( + (Image.bookmark_count.is_(None)) | (Image.view_count.is_(None)) | (Image.comment_count.is_(None)) + ) + + async with Session() as session: + rows = (await session.execute(stmt)).all() + + rows_page = rows[: int(limit)] + next_cursor = int(rows_page[-1][0].id) if len(rows) > int(limit) and rows_page else None + + items: list[dict[str, Any]] = [] + for img, tag_count in rows_page: + tag_count_i = int(tag_count or 0) + missing_list: list[str] = [] + if tag_count_i <= 0: + missing_list.append("tags") + if img.width is None or img.height is None: + missing_list.append("geometry") + if img.x_restrict is None: + missing_list.append("r18") + if img.ai_type is None: + missing_list.append("ai") + if getattr(img, "illust_type", None) is None: + missing_list.append("illust_type") + if img.user_id is None: + missing_list.append("user") + if img.title is None or not str(img.title).strip(): + missing_list.append("title") + if img.created_at_pixiv is None or not str(img.created_at_pixiv).strip(): + missing_list.append("created_at") + if img.bookmark_count is None or img.view_count is None or img.comment_count is None: + missing_list.append("popularity") + + items.append( + { + "id": str(img.id), + "illust_id": str(img.illust_id), + "page_index": int(img.page_index), + "ext": img.ext, + "status": int(img.status), + "width": img.width, + "height": img.height, + "orientation": img.orientation, + "x_restrict": img.x_restrict, + "ai_type": img.ai_type, + "illust_type": getattr(img, "illust_type", None), + "bookmark_count": img.bookmark_count, + "view_count": img.view_count, + "comment_count": img.comment_count, + "user": { + "id": str(img.user_id) if img.user_id is not None else None, + "name": img.user_name, + }, + "title": img.title, + "created_at_pixiv": img.created_at_pixiv, + "original_url": img.original_url, + "proxy_path": img.proxy_path, + "tag_count": tag_count_i, + "missing": missing_list, + } + ) + + return { + "ok": True, + "items": items, + "next_cursor": str(next_cursor) if next_cursor is not None else "", + "request_id": rid, + } + + +async def _load_bulk_delete_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + raw_ids = data.get("image_ids", None) + if raw_ids is None: + raw_ids = data.get("ids", None) + if raw_ids is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing image_ids", status_code=400) + if not isinstance(raw_ids, list): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported image_ids", status_code=400) + + ids: list[int] = [] + seen: set[int] = set() + for raw in raw_ids: + try: + i = int(raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported image_ids", status_code=400) from exc + if i <= 0 or i in seen: + continue + seen.add(i) + ids.append(i) + + if not ids: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Empty image_ids", status_code=400) + if len(ids) > 20_000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Too many image_ids", status_code=400) + + return {"image_ids": ids} + + +def _chunks(values: list[int], *, chunk_size: int) -> list[list[int]]: + if chunk_size <= 0: + return [values] + out: list[list[int]] = [] + for i in range(0, len(values), chunk_size): + out.append(values[i : i + chunk_size]) + return out + + +def _safe_rowcount(result: Any) -> int: + try: + rc = int(getattr(result, "rowcount", 0) or 0) + except Exception: + return 0 + return rc if rc > 0 else 0 + + +@router.delete("/images/{image_id}") +async def delete_admin_image( + image_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if int(image_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid image id", status_code=400) + + rid = get_or_create_request_id(request) + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + row = await session.get(Image, int(image_id)) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Image not found", status_code=404) + await session.execute(sa.delete(ImageTag).where(ImageTag.image_id == int(image_id))) + await session.delete(row) + await session.commit() + + return {"ok": True, "image_id": str(int(image_id)), "request_id": rid} + + return await with_sqlite_busy_retry(_op) + + +@router.post("/images/bulk-delete") +async def bulk_delete_admin_images( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + body = await _load_bulk_delete_json(request) + ids = list(body["image_ids"]) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + deleted = 0 + found = 0 + async with Session() as session: + for chunk in _chunks(ids, chunk_size=900): + rows = (await session.execute(sa.select(Image.id).where(Image.id.in_(chunk)))).scalars().all() + found += len(rows) + + for chunk in _chunks(ids, chunk_size=900): + await session.execute(sa.delete(ImageTag).where(ImageTag.image_id.in_(chunk))) + result = await session.execute(sa.delete(Image).where(Image.id.in_(chunk))) + deleted += _safe_rowcount(result) + + await session.commit() + + missing = max(0, int(len(ids)) - int(found)) + return { + "ok": True, + "requested": int(len(ids)), + "deleted": int(deleted), + "missing": int(missing), + "request_id": rid, + } + + return await with_sqlite_busy_retry(_op) + + +async def _load_clear_images_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + confirm = data.get("confirm", False) + if confirm not in {True, 1, "1", "true", "yes", "y", "on"}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing confirm", status_code=400) + + delete_tags = data.get("delete_tags", True) + delete_tags_bool = ( + bool(delete_tags) + if isinstance(delete_tags, (bool, int)) + else str(delete_tags).strip().lower() in {"1", "true", "yes", "y", "on"} + ) + + return {"delete_tags": bool(delete_tags_bool)} + + +@router.post("/images/clear") +async def clear_admin_images( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + body = await _load_clear_images_json(request) + delete_tags = bool(body["delete_tags"]) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + result_links = await session.execute(sa.delete(ImageTag)) + result_images = await session.execute(sa.delete(Image)) + result_tags = None + if delete_tags: + result_tags = await session.execute(sa.delete(Tag)) + + await session.commit() + + return { + "ok": True, + "deleted_image_tags": _safe_rowcount(result_links), + "deleted_images": _safe_rowcount(result_images), + "deleted_tags": _safe_rowcount(result_tags) if result_tags is not None else 0, + "request_id": rid, + } + + return await with_sqlite_busy_retry(_op) diff --git a/backend/app/api/admin/imports.py b/backend/app/api/admin/imports.py new file mode 100644 index 0000000..f0badda --- /dev/null +++ b/backend/app/api/admin/imports.py @@ -0,0 +1,435 @@ +from __future__ import annotations + +import json +import os +from dataclasses import asdict, dataclass +from typing import Any, Literal +from uuid import uuid4 + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Request +from pydantic import BaseModel, Field +from pydantic import ValidationError +from starlette.datastructures import UploadFile + +from app.api.admin.deps import get_admin_claims +from app.core.data_files import get_sqlite_db_dir, make_file_ref +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.core.time import iso_utc_ms +from app.core.pixiv_urls import parse_pixiv_original_url +from app.db.models.images import Image +from app.db.models.imports import Import +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker, with_sqlite_busy_retry +from app.jobs.dispatch import JobDispatcher +from app.jobs.executor import execute_claimed_job +from app.jobs.handlers.import_images import build_import_images_handler + +router = APIRouter() + + +class ImportCreateRequest(BaseModel): + text: str = Field(min_length=1) + dry_run: bool = False + hydrate_on_import: bool = False + source: str = "manual" + + +class ImportRollbackRequest(BaseModel): + mode: Literal["disable", "delete"] + + +@dataclass(frozen=True, slots=True) +class ImportErrorItem: + line: int + url: str + code: str + message: str + + +def _max_import_text_bytes() -> int: + default = 50 * 1024 * 1024 + raw = (os.environ.get("IMPORT_MAX_BYTES") or "").strip() + if not raw: + return default + try: + value = int(raw) + except Exception: + return default + return max(1024, min(int(value), 50 * 1024 * 1024)) + + +def _import_inline_max_accepted() -> int: + default = 200 + raw = (os.environ.get("IMPORT_INLINE_MAX_ACCEPTED") or "").strip() + if not raw: + return default + try: + value = int(raw) + except Exception: + return default + return max(0, min(int(value), 10_000)) + + +async def _claim_job_by_id(engine, *, job_id: int, worker_id: str, now: str) -> dict[str, Any] | None: + sql = """ +UPDATE jobs +SET status='running', + locked_by=:worker_id, + locked_at=:now, + updated_at=:now +WHERE id=:id AND status='pending' +RETURNING *; +""".strip() + + async def _op() -> dict[str, Any] | None: + async with engine.begin() as conn: + result = await conn.exec_driver_sql(sql, {"id": int(job_id), "worker_id": worker_id, "now": now}) + row = result.mappings().first() + return dict(row) if row else None + + return await with_sqlite_busy_retry(_op) + + +def _parse_import_text( + text: str, *, preview_limit: int = 20 +) -> tuple[int, int, int, int, list[ImportErrorItem], list[dict[str, Any]]]: + total = 0 + accepted = 0 + deduped = 0 + error_total = 0 + errors: list[ImportErrorItem] = [] + seen: set[tuple[int, int]] = set() + preview: list[dict[str, Any]] = [] + + for line_no, raw in enumerate(text.splitlines(), start=1): + url = raw.strip() + if not url: + continue + total += 1 + try: + parsed = parse_pixiv_original_url(url) + except Exception as exc: + error_total += 1 + if len(errors) < 200: + errors.append( + ImportErrorItem( + line=line_no, + url=url, + code=ErrorCode.UNSUPPORTED_URL.value, + message=str(exc) or ErrorCode.UNSUPPORTED_URL.value, + ) + ) + continue + + key = (parsed.illust_id, parsed.page_index) + if key in seen: + deduped += 1 + continue + seen.add(key) + accepted += 1 + + if len(preview) < int(preview_limit): + preview.append( + { + "illust_id": parsed.illust_id, + "page_index": parsed.page_index, + "ext": parsed.ext, + "url": url, + } + ) + + return total, accepted, deduped, error_total, errors, preview + + +def _parse_bool(value: Any, *, default: bool = False) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return default + + +def _validate_import_create(data: dict[str, Any]) -> ImportCreateRequest: + try: + return ImportCreateRequest(**data) + except ValidationError as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="导入请求体无效", status_code=400) from exc + + +async def _load_import_request(request: Request) -> ImportCreateRequest: + content_type = (request.headers.get("content-type") or "").lower() + max_bytes = _max_import_text_bytes() + + if content_type.startswith("application/json"): + data = await request.json() + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + body = _validate_import_create(data) + if len(body.text.encode("utf-8", errors="ignore")) > max_bytes: + raise ApiError(code=ErrorCode.PAYLOAD_TOO_LARGE, message="Payload too large", status_code=413) + return body + + if content_type.startswith("multipart/form-data"): + form = await request.form() + file_obj = form.get("file") + if not isinstance(file_obj, UploadFile): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing file", status_code=400) + + filename = (file_obj.filename or "").strip().lower() + if filename and not filename.endswith(".txt"): + raise ApiError(code=ErrorCode.INVALID_UPLOAD_TYPE, message="Unsupported upload type", status_code=400) + + raw = await file_obj.read() + if len(raw) > max_bytes: + raise ApiError(code=ErrorCode.PAYLOAD_TOO_LARGE, message="Payload too large", status_code=413) + text = raw.decode("utf-8", errors="replace") + + return _validate_import_create( + { + "text": text, + "dry_run": _parse_bool(form.get("dry_run"), default=False), + "hydrate_on_import": _parse_bool(form.get("hydrate_on_import"), default=False), + "source": str(form.get("source") or "manual"), + } + ) + + raise ApiError(code=ErrorCode.INVALID_UPLOAD_TYPE, message="Unsupported content type", status_code=400) + + +@router.post("/imports") +async def create_import( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + body = await _load_import_request(request) + rid = get_or_create_request_id(request) + + total, accepted, deduped, error_total, errors, preview = _parse_import_text(body.text, preview_limit=20) + + if body.dry_run: + return { + "ok": True, + "import_id": "", + "job_id": "", + "accepted": accepted, + "deduped": deduped, + "errors": [asdict(e) for e in errors[:200]], + "preview": preview, + "request_id": rid, + } + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + settings = request.app.state.settings + db_dir = get_sqlite_db_dir(settings.database_url) + payload_dir = db_dir / "imports_payloads" + payload_dir.mkdir(parents=True, exist_ok=True) + + payload_path = payload_dir / f"import_payload_{uuid4().hex}.txt" + try: + payload_path.write_text(body.text, encoding="utf-8") + except OSError as exc: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="写入导入内容失败", status_code=500) from exc + + file_ref = make_file_ref(payload_path, base_dir=db_dir) + + async def _op() -> tuple[int, int]: + async with Session() as session: + imp = Import(created_by=str(_claims.get("sub") or ""), source=body.source) + session.add(imp) + await session.flush() + + imp.total = int(total) + imp.accepted = int(accepted) + imp.success = 0 + imp.failed = int(error_total) + imp.detail_json = json.dumps( + { + "deduped": int(deduped), + "errors": [asdict(e) for e in errors[:200]], + }, + ensure_ascii=False, + ) + + job = JobRow( + type="import_images", + status="pending", + payload_json=json.dumps( + { + "import_id": int(imp.id), + "file_ref": file_ref, + "hydrate_on_import": bool(body.hydrate_on_import), + }, + ensure_ascii=False, + separators=(",", ":"), + ), + ref_type="import", + ref_id=str(imp.id), + ) + session.add(job) + await session.flush() + + await session.commit() + return int(imp.id), int(job.id) + + try: + import_id, job_id = await with_sqlite_busy_retry(_op) + except Exception: + try: + payload_path.unlink(missing_ok=True) + except Exception: + pass + raise + + inline_max = _import_inline_max_accepted() + executed_inline = False + if accepted > 0 and accepted <= inline_max: + now = iso_utc_ms() + actor = str(_claims.get("sub") or "admin").strip() or "admin" + worker_id = f"inline-import:{actor}" + claimed = await _claim_job_by_id(engine, job_id=int(job_id), worker_id=worker_id, now=now) + if claimed is not None: + dispatcher = JobDispatcher() + dispatcher.register("import_images", build_import_images_handler(engine)) + await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id=worker_id) + executed_inline = True + + return { + "ok": True, + "import_id": str(import_id), + "job_id": str(job_id), + "executed_inline": executed_inline, + "accepted": accepted, + "deduped": deduped, + "errors": [asdict(e) for e in errors[:200]], + "preview": preview, + "request_id": rid, + } + + +@router.post("/imports/{import_id}/rollback") +async def rollback_import( + import_id: int, + body: ImportRollbackRequest, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if import_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid import id", status_code=400) + + rid = get_or_create_request_id(request) + + target_status = 2 if body.mode == "disable" else 4 + now_expr = sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))") + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> int: + async with Session() as session: + imp = await session.get(Import, import_id) + if imp is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Import not found", status_code=404) + + result = await session.execute( + sa.update(Image) + .where(Image.created_import_id == import_id) + .values(status=target_status, updated_at=now_expr) + ) + updated = int(result.rowcount or 0) + await session.commit() + return int(updated) + + updated = await with_sqlite_busy_retry(_op) + + return { + "ok": True, + "mode": body.mode, + "updated": updated, + "request_id": rid, + } + + +@router.get("/imports/{import_id}") +async def get_import( + import_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if import_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid import id", status_code=400) + + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + imp = await session.get(Import, import_id) + if imp is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Import not found", status_code=404) + + job = ( + ( + await session.execute( + sa.select(JobRow) + .where(JobRow.ref_type == "import", JobRow.ref_id == str(import_id)) + .order_by(JobRow.id.desc()) + .limit(1) + ) + ) + .scalars() + .first() + ) + + detail: dict[str, Any] = {} + if imp.detail_json: + try: + parsed = json.loads(imp.detail_json) + if isinstance(parsed, dict): + detail = parsed + except Exception: + detail = {} + + return { + "ok": True, + "item": { + "import": { + "id": str(imp.id), + "created_at": imp.created_at, + "created_by": imp.created_by, + "source": imp.source, + "total": int(imp.total or 0), + "accepted": int(imp.accepted or 0), + "success": int(imp.success or 0), + "failed": int(imp.failed or 0), + }, + "job": ( + { + "id": str(job.id), + "type": job.type, + "status": job.status, + "attempt": job.attempt, + "max_attempts": job.max_attempts, + "last_error": job.last_error, + } + if job is not None + else None + ), + "detail": detail, + }, + "request_id": rid, + } diff --git a/backend/app/api/admin/jobs.py b/backend/app/api/admin/jobs.py new file mode 100644 index 0000000..21bd510 --- /dev/null +++ b/backend/app/api/admin/jobs.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import json +from typing import Any + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Request + +from app.api.admin.deps import get_admin_claims +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.core.time import iso_utc_ms +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + +router = APIRouter() + +_ALLOWED_JOB_STATUSES = {"pending", "running", "paused", "canceled", "completed", "failed", "dlq"} + + +@router.get("/jobs") +async def list_jobs( + request: Request, + limit: int = 50, + cursor: str | None = None, + status: str | None = None, + type: str | None = None, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if limit < 1 or limit > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported limit", status_code=400) + + cursor_i: int | None = None + cursor_raw = (cursor or "").strip() + if cursor_raw: + if not cursor_raw.isdigit(): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + cursor_i = int(cursor_raw) + if cursor_i <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + + status_norm = (status or "").strip().lower() or None + if status_norm is not None and status_norm not in _ALLOWED_JOB_STATUSES: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported status", status_code=400) + + type_norm = (type or "").strip() or None + if type_norm is not None and len(type_norm) > 100: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported type", status_code=400) + + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + stmt = sa.select(JobRow).order_by(JobRow.id.desc()).limit(limit + 1) + if cursor_i is not None: + stmt = stmt.where(JobRow.id < cursor_i) + if status_norm is not None: + stmt = stmt.where(JobRow.status == status_norm) + if type_norm is not None: + stmt = stmt.where(JobRow.type == type_norm) + + async with Session() as session: + rows = ((await session.execute(stmt)).scalars().all()) + + items_rows = rows[:limit] + next_cursor = int(items_rows[-1].id) if len(rows) > limit and items_rows else None + + items = [ + { + "id": str(row.id), + "type": row.type, + "status": row.status, + "priority": int(row.priority), + "run_after": row.run_after, + "attempt": int(row.attempt), + "max_attempts": int(row.max_attempts), + "last_error": row.last_error, + "locked_by": row.locked_by, + "locked_at": row.locked_at, + "ref_type": row.ref_type, + "ref_id": row.ref_id, + "created_at": row.created_at, + "updated_at": row.updated_at, + } + for row in items_rows + ] + + return { + "ok": True, + "items": items, + "next_cursor": str(next_cursor) if next_cursor is not None else "", + "request_id": rid, + } + + +@router.get("/jobs/{job_id}") +async def get_job( + job_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if job_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid job id", status_code=400) + + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + row = await session.get(JobRow, job_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Job not found", status_code=404) + + payload_json = str(row.payload_json or "") + payload: Any = None + if payload_json.strip(): + try: + payload = json.loads(payload_json) + except Exception: + payload = None + + return { + "ok": True, + "item": { + "id": str(row.id), + "type": row.type, + "status": row.status, + "priority": int(row.priority), + "run_after": row.run_after, + "attempt": int(row.attempt), + "max_attempts": int(row.max_attempts), + "payload": payload, + "payload_json": payload_json, + "last_error": row.last_error, + "locked_by": row.locked_by, + "locked_at": row.locked_at, + "ref_type": row.ref_type, + "ref_id": row.ref_id, + "created_at": row.created_at, + "updated_at": row.updated_at, + }, + "request_id": rid, + } + + +@router.post("/jobs/{job_id}/retry") +async def retry_job( + job_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if job_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid job id", status_code=400) + + rid = get_or_create_request_id(request) + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + row = await session.get(JobRow, job_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Job not found", status_code=404) + + if row.status == "running": + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Job is running", status_code=400) + + row.status = "pending" + row.run_after = None + row.locked_by = None + row.locked_at = None + row.updated_at = now + await session.commit() + + return {"ok": True, "job_id": str(job_id), "status": "pending", "request_id": rid} + + return await with_sqlite_busy_retry(_op) + + +@router.post("/jobs/{job_id}/cancel") +async def cancel_job( + job_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if job_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid job id", status_code=400) + + rid = get_or_create_request_id(request) + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + row = await session.get(JobRow, job_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Job not found", status_code=404) + + row.status = "canceled" + row.locked_by = None + row.locked_at = None + row.updated_at = now + await session.commit() + + return {"ok": True, "job_id": str(job_id), "status": "canceled", "request_id": rid} + + return await with_sqlite_busy_retry(_op) + + +@router.post("/jobs/{job_id}/move-to-dlq") +async def move_job_to_dlq( + job_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if job_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid job id", status_code=400) + + rid = get_or_create_request_id(request) + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + row = await session.get(JobRow, job_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Job not found", status_code=404) + + row.status = "dlq" + row.run_after = None + row.locked_by = None + row.locked_at = None + row.updated_at = now + await session.commit() + + return {"ok": True, "job_id": str(job_id), "status": "dlq", "request_id": rid} + + return await with_sqlite_busy_retry(_op) diff --git a/backend/app/api/admin/maintenance.py b/backend/app/api/admin/maintenance.py new file mode 100644 index 0000000..8971706 --- /dev/null +++ b/backend/app/api/admin/maintenance.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, Request + +from app.api.admin.deps import get_admin_claims +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.db.request_logs_cleanup import ( + DEFAULT_REQUEST_LOGS_CHUNK_SIZE, + DEFAULT_REQUEST_LOGS_KEEP_DAYS, + DEFAULT_REQUEST_LOGS_MAX_DELETE_ROWS, + cleanup_request_logs, + preview_request_logs_cleanup, +) + +router = APIRouter() + + +def _parse_bool(value: Any, *, default: bool) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return default + + +async def _load_cleanup_request_logs_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + keep_days_raw = data.get("keep_days", DEFAULT_REQUEST_LOGS_KEEP_DAYS) + try: + keep_days = int(keep_days_raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported keep_days", status_code=400) from exc + if keep_days < 0 or keep_days > 36500: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported keep_days", status_code=400) + + max_delete_rows_raw = data.get("max_delete_rows", DEFAULT_REQUEST_LOGS_MAX_DELETE_ROWS) + try: + max_delete_rows = int(max_delete_rows_raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported max_delete_rows", status_code=400) from exc + if max_delete_rows < 1 or max_delete_rows > 10_000_000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported max_delete_rows", status_code=400) + + chunk_size_raw = data.get("chunk_size", DEFAULT_REQUEST_LOGS_CHUNK_SIZE) + try: + chunk_size = int(chunk_size_raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported chunk_size", status_code=400) from exc + if chunk_size < 1 or chunk_size > 100_000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported chunk_size", status_code=400) + + dry_run = _parse_bool(data.get("dry_run"), default=False) + + return { + "keep_days": int(keep_days), + "max_delete_rows": int(max_delete_rows), + "chunk_size": int(chunk_size), + "dry_run": bool(dry_run), + } + + +@router.post("/maintenance/request-logs/cleanup") +async def cleanup_request_logs_endpoint( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + cfg = await _load_cleanup_request_logs_json(request) + + engine = request.app.state.engine + if bool(cfg["dry_run"]): + preview = await preview_request_logs_cleanup( + engine, + keep_days=int(cfg["keep_days"]), + max_delete_rows=int(cfg["max_delete_rows"]), + ) + return { + "ok": True, + "dry_run": True, + "cutoff": preview.cutoff, + "would_delete": int(preview.would_delete), + "has_more": bool(preview.has_more), + "request_id": rid, + } + + result = await cleanup_request_logs( + engine, + keep_days=int(cfg["keep_days"]), + max_delete_rows=int(cfg["max_delete_rows"]), + chunk_size=int(cfg["chunk_size"]), + ) + return { + "ok": True, + "dry_run": False, + "cutoff": result.cutoff, + "deleted": int(result.deleted), + "has_more": bool(result.has_more), + "request_id": rid, + } + diff --git a/backend/app/api/admin/proxies.py b/backend/app/api/admin/proxies.py new file mode 100644 index 0000000..cf3c739 --- /dev/null +++ b/backend/app/api/admin/proxies.py @@ -0,0 +1,1095 @@ +from __future__ import annotations + +import json +from typing import Any +from urllib.parse import urlparse, urlunparse + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Request +from sqlalchemy.dialects.sqlite import insert as sqlite_insert + +from app.api.admin.deps import get_admin_claims +from app.core.bindings_recompute import recompute_token_proxy_bindings +from app.core.crypto import FieldEncryptor +from app.core.errors import ApiError, ErrorCode +from app.core.proxy_uri import parse_proxy_uri +from app.core.request_id import get_or_create_request_id +from app.core.time import iso_utc_ms +from app.db.models.jobs import JobRow +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker, with_sqlite_busy_retry +from app.easy_proxies.client import EasyProxiesError, easy_proxies_auth, easy_proxies_export +from app.easy_proxies.normalize import normalize_exported_proxy_host, resolve_export_host + +router = APIRouter() + + +def _mask_proxy_uri(*, scheme: str, host: str, port: int, username: str, password_set: bool) -> str: + scheme = (scheme or "").strip().lower() + host = (host or "").strip() + username = (username or "").strip() + if ":" in host and not host.startswith("["): + host = f"[{host}]" + + auth = "" + if username: + auth = f"{username}@" + if password_set: + auth = f"{username}:***@" + + return f"{scheme}://{auth}{host}:{int(port)}" + + +def _sanitize_source_ref(value: str | None) -> str | None: + raw = str(value or "").strip() + if not raw: + return None + + try: + parsed = urlparse(raw) + except Exception: + return raw[:200] + + if not parsed.scheme or not parsed.netloc: + return raw[:200] + + host = parsed.hostname + if not host: + return raw[:200] + + port = parsed.port + netloc = f"{host}:{int(port)}" if port else host + path = parsed.path or "" + + return urlunparse((parsed.scheme, netloc, path, "", "", "")) + + +@router.get("/proxies/endpoints") +async def list_proxy_endpoints( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + async with Session() as session: + endpoints = ( + ( + await session.execute(sa.select(ProxyEndpoint).order_by(ProxyEndpoint.id.desc())) + ) + .scalars() + .all() + ) + + endpoint_ids = [int(p.id) for p in endpoints] + + pools_by_eid: dict[int, list[dict[str, Any]]] = {} + if endpoint_ids: + pool_rows = ( + ( + await session.execute( + sa.select( + ProxyPoolEndpoint.endpoint_id, + ProxyPoolEndpoint.pool_id, + ProxyPoolEndpoint.enabled, + ProxyPoolEndpoint.weight, + ProxyPool.name, + ProxyPool.enabled, + ) + .join(ProxyPool, ProxyPool.id == ProxyPoolEndpoint.pool_id) + .where(ProxyPoolEndpoint.endpoint_id.in_(endpoint_ids)) + .order_by(ProxyPoolEndpoint.pool_id.asc()) + ) + ) + .all() + ) + for endpoint_id, pool_id, member_enabled, weight, pool_name, pool_enabled in pool_rows: + pools_by_eid.setdefault(int(endpoint_id), []).append( + { + "id": str(pool_id), + "name": str(pool_name), + "pool_enabled": bool(pool_enabled), + "member_enabled": bool(member_enabled), + "weight": int(weight or 0), + } + ) + + primary_counts: dict[int, int] = {} + override_counts: dict[int, int] = {} + if endpoint_ids: + rows = ( + ( + await session.execute( + sa.select(TokenProxyBinding.primary_proxy_id, sa.func.count()) + .where(TokenProxyBinding.primary_proxy_id.in_(endpoint_ids)) + .group_by(TokenProxyBinding.primary_proxy_id) + ) + ) + .all() + ) + for pid, c in rows: + primary_counts[int(pid)] = int(c) + + rows2 = ( + ( + await session.execute( + sa.select(TokenProxyBinding.override_proxy_id, sa.func.count()) + .where(TokenProxyBinding.override_proxy_id.is_not(None)) + .where(TokenProxyBinding.override_proxy_id.in_(endpoint_ids)) + .group_by(TokenProxyBinding.override_proxy_id) + ) + ) + .all() + ) + for pid, c in rows2: + if pid is None: + continue + override_counts[int(pid)] = int(c) + + now_iso = iso_utc_ms() + invalid_hosts = {"0.0.0.0", "127.0.0.1", "localhost", "::", "::1", "[::]", "[::1]"} + + items = [ + { + "id": str(p.id), + "enabled": bool(p.enabled), + "source": str(p.source or "manual"), + "source_ref": _sanitize_source_ref(p.source_ref), + "scheme": str(p.scheme), + "host": str(p.host), + "port": int(p.port), + "invalid_host": str(p.host or "").strip().lower() in invalid_hosts, + "uri_masked": _mask_proxy_uri( + scheme=str(p.scheme), + host=str(p.host), + port=int(p.port), + username=str(p.username or ""), + password_set=bool(str(p.password_enc or "").strip()), + ), + "latency_ms": float(p.last_latency_ms) if p.last_latency_ms is not None else None, + "status": ( + "blacklisted" + if (p.blacklisted_until and str(p.blacklisted_until) > str(now_iso)) + else "ok" + if p.last_ok_at and (not p.last_fail_at or str(p.last_ok_at) >= str(p.last_fail_at)) + else "fail" + if p.last_fail_at + else "unknown" + ), + "blacklisted_until": p.blacklisted_until, + "last_error": p.last_error, + "success_count": int(p.success_count or 0), + "failure_count": int(p.failure_count or 0), + "last_ok_at": p.last_ok_at, + "last_fail_at": p.last_fail_at, + "pools": pools_by_eid.get(int(p.id), []), + "bindings": { + "primary_count": int(primary_counts.get(int(p.id), 0)), + "override_count": int(override_counts.get(int(p.id), 0)), + }, + } + for p in endpoints + ] + + return {"ok": True, "items": items, "request_id": rid} + + +def _parse_conflict_policy(value: Any) -> str: + v = str(value or "").strip().lower() or "skip" + if v not in {"skip", "overwrite"}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported conflict_policy", status_code=400) + return v + + +def _parse_easy_conflict_policy(value: Any) -> str: + v = str(value or "").strip().lower() or "skip_non_easy_proxies" + if v not in {"skip_non_easy_proxies", "skip", "overwrite"}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported conflict_policy", status_code=400) + return v + + +def _parse_bool_strict(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return None + + +async def _load_import_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + text = str(data.get("text") or "") + if not text.strip(): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing text", status_code=400) + + source = str(data.get("source") or "manual").strip() or "manual" + conflict_policy = _parse_conflict_policy(data.get("conflict_policy")) + + return {"text": text, "source": source, "conflict_policy": conflict_policy} + + +async def _load_update_endpoint_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + if "enabled" not in data: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing enabled", status_code=400) + + enabled = _parse_bool_strict(data.get("enabled")) + if enabled is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported enabled", status_code=400) + + return {"enabled": bool(enabled)} + + +async def _load_easy_import_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + base_url = str(data.get("base_url") or "").strip() + password = str(data.get("password") or "").strip() + if not base_url: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing base_url", status_code=400) + + host_override_raw = data.get("host_override") + host_override = str(host_override_raw).strip() if host_override_raw is not None else "" + host_override = host_override if host_override else None + if host_override is not None and len(host_override) > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid host_override", status_code=400) + + attach_pool_id: int | None = None + if "attach_pool_id" in data: + raw = data.get("attach_pool_id") + if raw is None or (isinstance(raw, str) and not raw.strip()): + attach_pool_id = None + else: + try: + attach_pool_id = int(raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid attach_pool_id", status_code=400) from exc + if attach_pool_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid attach_pool_id", status_code=400) + + attach_weight = 1 + if "attach_weight" in data: + try: + attach_weight = int(data.get("attach_weight")) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid attach_weight", status_code=400) from exc + if attach_weight < 0 or attach_weight > 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid attach_weight", status_code=400) + + recompute_bindings = False + if "recompute_bindings" in data: + v = _parse_bool_strict(data.get("recompute_bindings")) + if v is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid recompute_bindings", status_code=400) + recompute_bindings = bool(v) + + max_tokens_per_proxy = 2 + if "max_tokens_per_proxy" in data: + try: + max_tokens_per_proxy = int(data.get("max_tokens_per_proxy")) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid max_tokens_per_proxy", status_code=400) from exc + if max_tokens_per_proxy <= 0 or max_tokens_per_proxy > 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid max_tokens_per_proxy", status_code=400) + + strict = True + if "strict" in data: + v = _parse_bool_strict(data.get("strict")) + if v is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid strict", status_code=400) + strict = bool(v) + + conflict_policy = _parse_easy_conflict_policy(data.get("conflict_policy")) + if recompute_bindings and attach_pool_id is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="recompute_bindings requires attach_pool_id", status_code=400) + return { + "base_url": base_url, + "password": password, + "conflict_policy": conflict_policy, + "host_override": host_override, + "attach_pool_id": attach_pool_id, + "attach_weight": int(attach_weight), + "recompute_bindings": bool(recompute_bindings), + "max_tokens_per_proxy": int(max_tokens_per_proxy), + "strict": bool(strict), + } + + +async def _load_probe_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception: + return {} + + if data is None: + return {} + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + out: dict[str, Any] = {} + + if "probe_url" in data: + probe_url = str(data.get("probe_url") or "").strip() + if probe_url: + if len(probe_url) > 2000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid probe_url", status_code=400) + if "://" not in probe_url: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid probe_url", status_code=400) + out["probe_url"] = probe_url + + if "timeout_ms" in data: + raw = data.get("timeout_ms") + try: + timeout_ms = int(raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid timeout_ms", status_code=400) from exc + if timeout_ms <= 0 or timeout_ms > 600_000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid timeout_ms", status_code=400) + out["timeout_ms"] = int(timeout_ms) + + if "concurrency" in data: + raw = data.get("concurrency") + try: + concurrency = int(raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid concurrency", status_code=400) from exc + if concurrency < 1 or concurrency > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid concurrency", status_code=400) + out["concurrency"] = int(concurrency) + + return out + + +async def _load_cleanup_invalid_hosts_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception: + return {} + + if data is None: + return {} + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + out: dict[str, Any] = {} + + for key in ("dry_run", "delete_orphans", "recompute_bindings", "strict"): + if key not in data: + continue + v = _parse_bool_strict(data.get(key)) + if v is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message=f"Invalid {key}", status_code=400) + out[key] = bool(v) + + if "max_tokens_per_proxy" in data: + try: + n = int(data.get("max_tokens_per_proxy")) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid max_tokens_per_proxy", status_code=400) from exc + if n <= 0 or n > 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid max_tokens_per_proxy", status_code=400) + out["max_tokens_per_proxy"] = int(n) + + return out + + +@router.post("/proxies/endpoints/import") +async def import_proxy_endpoints( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + + body = await _load_import_json(request) + + text = str(body["text"]) + source = str(body["source"]) + conflict_policy = str(body["conflict_policy"]) + + created = 0 + updated = 0 + skipped = 0 + errors: list[dict[str, Any]] = [] + + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + settings = request.app.state.settings + encryptor: FieldEncryptor | None = None + + async def _op() -> None: + nonlocal created, updated, skipped, errors, encryptor + async with Session() as session: + for line_no, raw in enumerate(text.splitlines(), start=1): + uri = raw.strip() + if not uri: + continue + try: + parsed = parse_proxy_uri(uri) + except Exception: + errors.append({"line": line_no, "code": "invalid_proxy_uri", "message": "invalid_proxy_uri"}) + continue + + username = (parsed.username or "").strip() + password = parsed.password + password_enc = "" + if password: + if encryptor is None: + try: + encryptor = FieldEncryptor.from_key(settings.field_encryption_key) + except Exception: + errors.append( + { + "line": line_no, + "code": "encryption_not_configured", + "message": "代理包含密码，但未配置加密密钥（FIELD_ENCRYPTION_KEY）", + } + ) + continue + password_enc = encryptor.encrypt_text(password) + + existing = ( + ( + await session.execute( + sa.select(ProxyEndpoint).where( + ProxyEndpoint.scheme == parsed.scheme, + ProxyEndpoint.host == parsed.host, + ProxyEndpoint.port == int(parsed.port), + ProxyEndpoint.username == username, + ) + ) + ) + .scalars() + .first() + ) + + if existing is None: + session.add( + ProxyEndpoint( + scheme=parsed.scheme, + host=parsed.host, + port=int(parsed.port), + username=username, + password_enc=password_enc, + enabled=1, + source=source, + source_ref=None, + updated_at=now, + ) + ) + created += 1 + continue + + if conflict_policy == "overwrite": + existing.password_enc = password_enc + existing.enabled = 1 + existing.source = source + existing.updated_at = now + updated += 1 + else: + skipped += 1 + + await session.commit() + + await with_sqlite_busy_retry(_op) + + return { + "ok": True, + "created": created, + "updated": updated, + "skipped": skipped, + "errors": errors[:200], + "request_id": rid, + } + + +@router.put("/proxies/endpoints/{endpoint_id}") +async def update_proxy_endpoint( + endpoint_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if endpoint_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid endpoint id", status_code=400) + + rid = get_or_create_request_id(request) + body = await _load_update_endpoint_json(request) + enabled = bool(body["enabled"]) + + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + async with Session() as session: + row = await session.get(ProxyEndpoint, endpoint_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Proxy endpoint not found", status_code=404) + + row.enabled = 1 if enabled else 0 + row.updated_at = now + await session.commit() + + return {"ok": True, "endpoint_id": str(endpoint_id), "enabled": enabled, "request_id": rid} + + +@router.post("/proxies/endpoints/{endpoint_id}/reset-failures") +async def reset_proxy_failures( + endpoint_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if endpoint_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid endpoint id", status_code=400) + + rid = get_or_create_request_id(request) + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + row = await session.get(ProxyEndpoint, endpoint_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Proxy endpoint not found", status_code=404) + + row.failure_count = 0 + row.blacklisted_until = None + row.last_fail_at = None + row.last_error = None + row.updated_at = now + + await session.commit() + + return {"ok": True, "endpoint_id": str(endpoint_id), "request_id": rid} + + +@router.post("/proxies/endpoints/cleanup-invalid-hosts") +async def cleanup_invalid_hosts( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + + body = await _load_cleanup_invalid_hosts_json(request) + dry_run = bool(body.get("dry_run", False)) + delete_orphans = bool(body.get("delete_orphans", False)) + recompute_bindings = bool(body.get("recompute_bindings", True)) + max_tokens_per_proxy = int(body.get("max_tokens_per_proxy", 2)) + strict = bool(body.get("strict", False)) + + invalid_hosts = {"0.0.0.0", "127.0.0.1", "localhost", "::", "::1", "[::]", "[::1]"} + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + now = iso_utc_ms() + + async with Session() as session: + endpoint_rows = ( + ( + await session.execute( + sa.select(ProxyEndpoint.id, ProxyEndpoint.host) + .where(sa.func.lower(sa.func.trim(ProxyEndpoint.host)).in_(sorted(invalid_hosts))) + .order_by(ProxyEndpoint.id.asc()) + ) + ) + .all() + ) + endpoint_ids = [int(r[0]) for r in endpoint_rows] + + affected_pool_ids: list[int] = [] + if endpoint_ids: + pool_rows = ( + ( + await session.execute( + sa.select(sa.distinct(ProxyPoolEndpoint.pool_id)) + .where(ProxyPoolEndpoint.endpoint_id.in_(endpoint_ids)) + .order_by(ProxyPoolEndpoint.pool_id.asc()) + ) + ) + .all() + ) + affected_pool_ids = [int(r[0]) for r in pool_rows] + + if dry_run: + return { + "ok": True, + "dry_run": True, + "invalid_hosts": sorted(invalid_hosts), + "matched": len(endpoint_ids), + "endpoint_ids": [str(x) for x in endpoint_ids[:200]], + "affected_pool_ids": [str(x) for x in affected_pool_ids], + "request_id": rid, + } + + disabled = 0 + memberships_removed = 0 + overrides_cleared = 0 + deleted = 0 + binding_results: list[dict[str, Any]] = [] + warnings: list[str] = [] + + if endpoint_ids: + disabled = int( + ( + await session.execute( + sa.update(ProxyEndpoint) + .where(ProxyEndpoint.id.in_(endpoint_ids)) + .values(enabled=0, updated_at=now) + ) + ).rowcount + or 0 + ) + + memberships_removed = int( + ( + await session.execute( + sa.delete(ProxyPoolEndpoint).where(ProxyPoolEndpoint.endpoint_id.in_(endpoint_ids)) + ) + ).rowcount + or 0 + ) + + overrides_cleared = int( + ( + await session.execute( + sa.update(TokenProxyBinding) + .where(TokenProxyBinding.override_proxy_id.is_not(None)) + .where(TokenProxyBinding.override_proxy_id.in_(endpoint_ids)) + .values(override_proxy_id=None, override_expires_at=None, updated_at=now) + ) + ).rowcount + or 0 + ) + + if recompute_bindings and affected_pool_ids: + for pid in affected_pool_ids: + try: + result = await recompute_token_proxy_bindings( + session, + pool_id=int(pid), + now=now, + max_tokens_per_proxy=int(max_tokens_per_proxy), + strict=bool(strict), + ) + except ApiError as exc: + warnings.append(f"pool#{pid} 重算绑定失败: {exc.message}") + continue + binding_results.append({"pool_id": str(pid), **result}) + + if delete_orphans and endpoint_ids: + refs_primary = ( + ( + await session.execute( + sa.select(sa.distinct(TokenProxyBinding.primary_proxy_id)).where( + TokenProxyBinding.primary_proxy_id.in_(endpoint_ids) + ) + ) + ) + .scalars() + .all() + ) + refs_override = ( + ( + await session.execute( + sa.select(sa.distinct(TokenProxyBinding.override_proxy_id)) + .where(TokenProxyBinding.override_proxy_id.is_not(None)) + .where(TokenProxyBinding.override_proxy_id.in_(endpoint_ids)) + ) + ) + .scalars() + .all() + ) + referenced: set[int] = {int(x) for x in refs_primary if x is not None} | {int(x) for x in refs_override if x is not None} + deletable = [int(eid) for eid in endpoint_ids if int(eid) not in referenced] + if deletable: + deleted = int( + (await session.execute(sa.delete(ProxyEndpoint).where(ProxyEndpoint.id.in_(deletable)))).rowcount or 0 + ) + + await session.commit() + + return { + "ok": True, + "dry_run": False, + "invalid_hosts": sorted(invalid_hosts), + "matched": len(endpoint_ids), + "disabled": int(disabled), + "memberships_removed": int(memberships_removed), + "overrides_cleared": int(overrides_cleared), + "deleted": int(deleted), + "affected_pool_ids": [str(x) for x in affected_pool_ids], + "bindings": binding_results, + "warnings": warnings, + "request_id": rid, + } + + +@router.post("/proxies/easy-proxies/import") +async def import_easy_proxies( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + body = await _load_easy_import_json(request) + + base_url_raw = str(body["base_url"]) + base_url = _sanitize_source_ref(base_url_raw) or base_url_raw + password = str(body["password"]) + conflict_policy = str(body["conflict_policy"]) + host_override = body.get("host_override") + attach_pool_id = body.get("attach_pool_id") + attach_weight = int(body.get("attach_weight", 1)) + recompute_bindings = bool(body.get("recompute_bindings", False)) + max_tokens_per_proxy = int(body.get("max_tokens_per_proxy", 2)) + strict = bool(body.get("strict", True)) + + export_host = resolve_export_host(base_url=base_url_raw, host_override=host_override) + + transport = getattr(request.app.state, "httpx_transport", None) + + try: + bearer_token: str | None = None + if password.strip(): + auth = await easy_proxies_auth(base_url=base_url_raw, password=password, transport=transport) + bearer_token = auth.token + uris = await easy_proxies_export(base_url=base_url_raw, bearer_token=bearer_token, transport=transport) + except EasyProxiesError as exc: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="easy_proxies import failed", status_code=502) from exc + except Exception as exc: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="easy_proxies import failed", status_code=502) from exc + + warnings: list[str] = [] + raw_count = len(uris) + seen_keys: set[tuple[str, str, int, str]] = set() + deduped: list[str] = [] + invalid_uris: list[str] = [] + duplicates = 0 + invalid = 0 + placeholder_hosts = 0 + rewritten_hosts = 0 + for raw in uris: + uri = (raw or "").strip() + if not uri: + continue + try: + parsed = parse_proxy_uri(uri) + except Exception: + invalid += 1 + invalid_uris.append(uri) + continue + + desired_host, rewrote = normalize_exported_proxy_host(exported_host=str(parsed.host), export_host=export_host) + if str(parsed.host or "").strip().lower() in {"0.0.0.0", "127.0.0.1", "localhost", "::", "::1"}: + placeholder_hosts += 1 + if rewrote: + rewritten_hosts += 1 + + key = (str(parsed.scheme), str(desired_host), int(parsed.port), str((parsed.username or "").strip())) + if key in seen_keys: + duplicates += 1 + continue + seen_keys.add(key) + deduped.append(uri) + + uris = deduped + if duplicates > 0: + warnings.append(f"检测到导出结果包含重复入口（已去重 {duplicates} 条）。") + if placeholder_hosts > 0 and rewritten_hosts <= 0: + warnings.append("检测到 easy_proxies 导出 host 为 0.0.0.0/127.0.0.1/localhost 等占位符，但无法确定可用的替换 host；请检查 base_url 或使用 host_override。") + if rewritten_hosts > 0 and export_host: + warnings.append(f"检测到 easy_proxies 导出 host 为占位符，已自动替换为 {export_host}。") + if raw_count > 1 and len(deduped) <= 1: + warnings.append("当前 easy_proxies 可能处于 pool 模式（所有节点共享同一入口端口），导入后只会得到一个入口。若要每节点独立端口，请在 easy_proxies 启用 multi-port 或 hybrid 模式后再导入。") + if invalid > 0: + warnings.append(f"有 {invalid} 条导出内容不是合法代理 URI，已跳过。") + + created = 0 + updated = 0 + skipped = 0 + errors: list[dict[str, Any]] = [] + attach_created = 0 + attach_updated = 0 + attach_total = 0 + binding_result: dict[str, Any] | None = None + + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + settings = request.app.state.settings + encryptor: FieldEncryptor | None = None + + async def _op() -> None: + nonlocal created, updated, skipped, errors, encryptor, attach_created, attach_updated, attach_total, binding_result + async with Session() as session: + for uri in deduped + invalid_uris: + try: + parsed = parse_proxy_uri(uri) + except Exception: + errors.append({"code": "invalid_proxy_uri", "message": "invalid_proxy_uri"}) + continue + + username = (parsed.username or "").strip() + password_v = parsed.password + password_enc = "" + if password_v: + if encryptor is None: + try: + encryptor = FieldEncryptor.from_key(settings.field_encryption_key) + except Exception: + errors.append( + { + "code": "encryption_not_configured", + "message": "代理包含密码，但未配置加密密钥（FIELD_ENCRYPTION_KEY）", + } + ) + continue + password_enc = encryptor.encrypt_text(password_v) + + desired_host, rewrote = normalize_exported_proxy_host( + exported_host=str(parsed.host), + export_host=export_host, + ) + original_host = str(parsed.host) + + existing = ( + ( + await session.execute( + sa.select(ProxyEndpoint).where( + ProxyEndpoint.scheme == parsed.scheme, + ProxyEndpoint.host == desired_host, + ProxyEndpoint.port == int(parsed.port), + ProxyEndpoint.username == username, + ) + ) + ) + .scalars() + .first() + ) + + moved_from_placeholder = False + if existing is None and rewrote and desired_host and desired_host != original_host: + placeholder = ( + ( + await session.execute( + sa.select(ProxyEndpoint).where( + ProxyEndpoint.scheme == parsed.scheme, + ProxyEndpoint.host == original_host, + ProxyEndpoint.port == int(parsed.port), + ProxyEndpoint.username == username, + ) + ) + ) + .scalars() + .first() + ) + if placeholder is not None: + can_update_placeholder = conflict_policy == "overwrite" or ( + conflict_policy == "skip_non_easy_proxies" and (placeholder.source or "") == "easy_proxies" + ) + if can_update_placeholder: + existing = placeholder + existing.host = desired_host + moved_from_placeholder = True + + if existing is None: + session.add( + ProxyEndpoint( + scheme=parsed.scheme, + host=desired_host or original_host, + port=int(parsed.port), + username=username, + password_enc=password_enc, + enabled=1, + source="easy_proxies", + source_ref=base_url, + updated_at=now, + ) + ) + created += 1 + continue + + if conflict_policy == "skip_non_easy_proxies" and (existing.source or "") != "easy_proxies": + if moved_from_placeholder: + # can't update, but the desired host identity is new; create a new easy_proxies entry + session.add( + ProxyEndpoint( + scheme=parsed.scheme, + host=desired_host or original_host, + port=int(parsed.port), + username=username, + password_enc=password_enc, + enabled=1, + source="easy_proxies", + source_ref=base_url, + updated_at=now, + ) + ) + created += 1 + else: + skipped += 1 + continue + if conflict_policy == "skip": + skipped += 1 + continue + + existing.password_enc = password_enc + existing.enabled = 1 + existing.source = "easy_proxies" + existing.source_ref = base_url + # easy-proxies export only contains currently-healthy endpoints; clear local blacklist so they can be used immediately. + existing.blacklisted_until = None + existing.last_error = None + existing.updated_at = now + updated += 1 + + if attach_pool_id is not None: + pool = await session.get(ProxyPool, int(attach_pool_id)) + if pool is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Proxy pool not found", status_code=404) + + endpoint_ids = ( + ( + await session.execute( + sa.select(ProxyEndpoint.id) + .where(ProxyEndpoint.source == "easy_proxies") + .where(ProxyEndpoint.source_ref == base_url) + .where(ProxyEndpoint.enabled == 1) + .order_by(ProxyEndpoint.id.asc()) + ) + ) + .scalars() + .all() + ) + attach_total = len(endpoint_ids) + if endpoint_ids: + existing_members = ( + ( + await session.execute( + sa.select(ProxyPoolEndpoint.endpoint_id) + .where(ProxyPoolEndpoint.pool_id == int(attach_pool_id)) + .where(ProxyPoolEndpoint.endpoint_id.in_([int(x) for x in endpoint_ids])) + ) + ) + .scalars() + .all() + ) + existing_set = {int(x) for x in existing_members} + attach_created = len([x for x in endpoint_ids if int(x) not in existing_set]) + attach_updated = len([x for x in endpoint_ids if int(x) in existing_set]) + + for endpoint_id in endpoint_ids: + stmt = sqlite_insert(ProxyPoolEndpoint).values( + pool_id=int(attach_pool_id), + endpoint_id=int(endpoint_id), + enabled=1, + weight=int(attach_weight), + updated_at=now, + ) + stmt = stmt.on_conflict_do_update( + index_elements=[ProxyPoolEndpoint.pool_id, ProxyPoolEndpoint.endpoint_id], + set_={"enabled": 1, "weight": int(attach_weight), "updated_at": now}, + ) + await session.execute(stmt) + + if recompute_bindings: + binding_result = await recompute_token_proxy_bindings( + session, + pool_id=int(attach_pool_id), + now=str(now), + max_tokens_per_proxy=int(max_tokens_per_proxy), + strict=bool(strict), + ) + + await session.commit() + + await with_sqlite_busy_retry(_op) + + resp: dict[str, Any] = { + "ok": True, + "created": created, + "updated": updated, + "skipped": skipped, + "errors": errors[:200], + "warnings": warnings[:50], + "request_id": rid, + } + if attach_pool_id is not None: + resp["attach"] = { + "pool_id": str(attach_pool_id), + "endpoints_total": int(attach_total), + "created": int(attach_created), + "updated": int(attach_updated), + } + if binding_result is not None: + resp["bindings"] = {"pool_id": str(attach_pool_id), **binding_result} + return resp + + +@router.post("/proxies/probe") +async def probe_proxies( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + opts = await _load_probe_json(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> int: + async with Session() as session: + job = JobRow( + type="proxy_probe", + status="pending", + payload_json=json.dumps({"scope": "all", **opts}, ensure_ascii=False), + ref_type="proxy_probe", + ref_id="all", + ) + session.add(job) + await session.flush() + await session.commit() + return int(job.id) + + job_id = await with_sqlite_busy_retry(_op) + + return {"ok": True, "job_id": str(job_id), "request_id": rid} diff --git a/backend/app/api/admin/proxy_pools.py b/backend/app/api/admin/proxy_pools.py new file mode 100644 index 0000000..500fa30 --- /dev/null +++ b/backend/app/api/admin/proxy_pools.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +from typing import Any + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Request +from sqlalchemy.exc import IntegrityError + +from app.api.admin.deps import get_admin_claims +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.session import create_sessionmaker + +router = APIRouter() + + +def _parse_bool(value: Any, *, default: bool) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return default + + +async def _load_create_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + name = str(data.get("name") or "").strip() + if not name: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing name", status_code=400) + if len(name) > 100: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported name", status_code=400) + + desc_raw = data.get("description") + description = str(desc_raw).strip() if desc_raw is not None else None + description = description if description else None + + enabled = _parse_bool(data.get("enabled"), default=True) + return {"name": name, "description": description, "enabled": enabled} + + +async def _load_update_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + out: dict[str, Any] = {} + + if "name" in data: + name = str(data.get("name") or "").strip() + if not name: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported name", status_code=400) + if len(name) > 100: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported name", status_code=400) + out["name"] = name + + if "description" in data: + desc_raw = data.get("description") + description = str(desc_raw).strip() if desc_raw is not None else None + out["description"] = description if description else None + + if "enabled" in data: + out["enabled"] = _parse_bool(data.get("enabled"), default=True) + + if not out: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing fields", status_code=400) + + return out + + +async def _load_set_endpoints_json(request: Request) -> list[dict[str, Any]]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + raw_items = data.get("items") + if raw_items is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing items", status_code=400) + if not isinstance(raw_items, list): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid items", status_code=400) + + out: list[dict[str, Any]] = [] + seen: set[int] = set() + + for raw in raw_items: + if not isinstance(raw, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid items", status_code=400) + try: + endpoint_id = int(raw.get("endpoint_id")) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid endpoint_id", status_code=400) from exc + if endpoint_id <= 0 or endpoint_id in seen: + continue + seen.add(endpoint_id) + + enabled = _parse_bool(raw.get("enabled"), default=True) + + weight_raw = raw.get("weight", 1) + try: + weight = int(weight_raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid weight", status_code=400) from exc + if weight < 0 or weight > 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid weight", status_code=400) + + out.append({"endpoint_id": endpoint_id, "enabled": enabled, "weight": weight}) + + return out + + +@router.get("/proxy-pools") +async def list_proxy_pools( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + async with Session() as session: + pools = ( + (await session.execute(sa.select(ProxyPool).order_by(ProxyPool.id.desc()))) + .scalars() + .all() + ) + + items = [ + { + "id": str(p.id), + "name": p.name, + "description": p.description, + "enabled": bool(p.enabled), + } + for p in pools + ] + + return {"ok": True, "items": items, "request_id": rid} + + +@router.post("/proxy-pools") +async def create_proxy_pool( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + body = await _load_create_json(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + row = ProxyPool( + name=str(body["name"]), + description=body["description"], + enabled=1 if bool(body["enabled"]) else 0, + ) + session.add(row) + try: + await session.commit() + except IntegrityError as exc: + await session.rollback() + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Proxy pool name exists", status_code=400) from exc + await session.refresh(row) + + return {"ok": True, "pool_id": str(row.id), "request_id": rid} + + +@router.put("/proxy-pools/{pool_id}") +async def update_proxy_pool( + pool_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if pool_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid pool id", status_code=400) + + rid = get_or_create_request_id(request) + body = await _load_update_json(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + async with Session() as session: + row = await session.get(ProxyPool, pool_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Proxy pool not found", status_code=404) + + if "name" in body: + row.name = str(body["name"]) + if "description" in body: + row.description = body["description"] + if "enabled" in body: + row.enabled = 1 if bool(body["enabled"]) else 0 + + try: + await session.commit() + except IntegrityError as exc: + await session.rollback() + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Proxy pool name exists", status_code=400) from exc + + return {"ok": True, "pool_id": str(pool_id), "request_id": rid} + + +@router.post("/proxy-pools/{pool_id}/endpoints") +async def set_proxy_pool_endpoints( + pool_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if pool_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid pool id", status_code=400) + + rid = get_or_create_request_id(request) + items = await _load_set_endpoints_json(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + created = 0 + updated = 0 + removed = 0 + + async with Session() as session: + pool = await session.get(ProxyPool, pool_id) + if pool is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Proxy pool not found", status_code=404) + + endpoint_ids = [int(x["endpoint_id"]) for x in items] + if endpoint_ids: + existing = ( + ( + await session.execute( + sa.select(ProxyEndpoint.id).where(ProxyEndpoint.id.in_(endpoint_ids)) + ) + ) + .scalars() + .all() + ) + existing_set = {int(x) for x in existing} + missing = [eid for eid in endpoint_ids if eid not in existing_set] + if missing: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unknown endpoint_id", status_code=400) + + current_rows = ( + ( + await session.execute( + sa.select(ProxyPoolEndpoint).where(ProxyPoolEndpoint.pool_id == pool_id) + ) + ) + .scalars() + .all() + ) + current_by_eid = {int(r.endpoint_id): r for r in current_rows} + + keep: set[int] = set() + for item in items: + eid = int(item["endpoint_id"]) + keep.add(eid) + row = current_by_eid.get(eid) + if row is None: + session.add( + ProxyPoolEndpoint( + pool_id=pool_id, + endpoint_id=eid, + enabled=1 if bool(item["enabled"]) else 0, + weight=int(item["weight"]), + ) + ) + created += 1 + else: + row.enabled = 1 if bool(item["enabled"]) else 0 + row.weight = int(item["weight"]) + updated += 1 + + for eid, row in current_by_eid.items(): + if eid in keep: + continue + await session.delete(row) + removed += 1 + + await session.commit() + + return { + "ok": True, + "pool_id": str(pool_id), + "created": created, + "updated": updated, + "removed": removed, + "request_id": rid, + } diff --git a/backend/app/api/admin/router.py b/backend/app/api/admin/router.py new file mode 100644 index 0000000..ac5b041 --- /dev/null +++ b/backend/app/api/admin/router.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from fastapi import APIRouter + +from app.api.admin.auth import router as auth_router +from app.api.admin.api_keys import router as api_keys_router +from app.api.admin.audit import router as audit_router +from app.api.admin.bindings import router as bindings_router +from app.api.admin.hydration_runs import router as hydration_runs_router +from app.api.admin.images import router as images_router +from app.api.admin.imports import router as imports_router +from app.api.admin.jobs import router as jobs_router +from app.api.admin.maintenance import router as maintenance_router +from app.api.admin.proxies import router as proxies_router +from app.api.admin.proxy_pools import router as proxy_pools_router +from app.api.admin.settings import router as settings_router +from app.api.admin.stats import router as stats_router +from app.api.admin.summary import router as summary_router +from app.api.admin.tokens import router as tokens_router + +router = APIRouter(prefix="/admin/api") +router.include_router(auth_router) +router.include_router(api_keys_router) +router.include_router(audit_router) +router.include_router(bindings_router) +router.include_router(hydration_runs_router) +router.include_router(images_router) +router.include_router(imports_router) +router.include_router(jobs_router) +router.include_router(maintenance_router) +router.include_router(proxies_router) +router.include_router(proxy_pools_router) +router.include_router(settings_router) +router.include_router(stats_router) +router.include_router(summary_router) +router.include_router(tokens_router) diff --git a/backend/app/api/admin/settings.py b/backend/app/api/admin/settings.py new file mode 100644 index 0000000..e1d5cd0 --- /dev/null +++ b/backend/app/api/admin/settings.py @@ -0,0 +1,444 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, Request + +from app.api.admin.deps import get_admin_claims +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.core.runtime_settings import ( + fetch_runtime_settings, + runtime_config_from_values, + set_runtime_setting, +) + +router = APIRouter() + +_DEFAULT_SCORE_WEIGHTS: dict[str, float] = { + "bookmark": 4.0, + "view": 0.5, + "comment": 2.0, + "pixels": 1.0, + "bookmark_rate": 3.0, +} + +_DEFAULT_RECOMMENDATION: dict[str, Any] = { + "pick_mode": "weighted", + "temperature": 1.0, + "score_weights": dict(_DEFAULT_SCORE_WEIGHTS), + "multipliers": { + "ai": 1.0, + "non_ai": 1.0, + "unknown_ai": 1.0, + "illust": 1.0, + "manga": 1.0, + "ugoira": 1.0, + "unknown_illust_type": 1.0, + }, +} + +_DEFAULT_SETTINGS = { + "random": { + "default_attempts": 3, + "default_r18_strict": True, + "fail_cooldown_ms": 600_000, + "strategy": "quality", + "quality_samples": 5, + "recommendation": dict(_DEFAULT_RECOMMENDATION), + }, + "proxy": {"allowlist_domains": []}, +} + + +def _as_str_list(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + out: list[str] = [] + seen: set[str] = set() + for item in value: + if not isinstance(item, str): + continue + v = item.strip() + if not v or v in seen: + continue + seen.add(v) + out.append(v) + return out + + +def _as_bool(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"true", "1", "yes", "y", "on"}: + return True + if v in {"false", "0", "no", "n", "off"}: + return False + return None + + +def _as_float(value: Any) -> float | None: + if value is None: + return None + if isinstance(value, bool): + return None + try: + return float(value) + except Exception: + return None + + +def _normalize_recommendation(value: Any, *, strict: bool) -> dict[str, Any]: + if value is None: + return dict(_DEFAULT_RECOMMENDATION) + if not isinstance(value, dict): + if strict: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid random.recommendation", status_code=400) + return dict(_DEFAULT_RECOMMENDATION) + + pick_mode_default = str(_DEFAULT_RECOMMENDATION["pick_mode"]) + pick_mode_raw = value.get("pick_mode") + pick_mode = pick_mode_default + if pick_mode_raw is not None: + candidate = str(pick_mode_raw or "").strip().lower() + if candidate not in {"best", "weighted"}: + if strict: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid random.recommendation.pick_mode", status_code=400) + else: + pick_mode = candidate + + temperature_default = float(_DEFAULT_RECOMMENDATION["temperature"]) + temperature = temperature_default + if "temperature" in value: + v = _as_float(value.get("temperature")) + if v is None: + if strict: + raise ApiError( + code=ErrorCode.BAD_REQUEST, + message="Invalid random.recommendation.temperature", + status_code=400, + ) + else: + temperature = float(max(0.05, min(float(v), 100.0))) + + score_weights_default = _DEFAULT_RECOMMENDATION["score_weights"] + score_weights_raw = value.get("score_weights") + if score_weights_raw is None: + score_weights_obj: dict[str, Any] = {} + elif not isinstance(score_weights_raw, dict): + if strict: + raise ApiError( + code=ErrorCode.BAD_REQUEST, + message="Invalid random.recommendation.score_weights", + status_code=400, + ) + score_weights_obj = {} + else: + score_weights_obj = score_weights_raw + if strict: + for k in score_weights_obj.keys(): + if str(k) not in _DEFAULT_SCORE_WEIGHTS: + raise ApiError( + code=ErrorCode.BAD_REQUEST, + message="Invalid random.recommendation.score_weights", + status_code=400, + ) + + score_weights: dict[str, float] = {} + for key, default_value in _DEFAULT_SCORE_WEIGHTS.items(): + if key in score_weights_obj: + v = _as_float(score_weights_obj.get(key)) + if v is None: + if strict: + raise ApiError( + code=ErrorCode.BAD_REQUEST, + message="Invalid random.recommendation.score_weights", + status_code=400, + ) + v = float(default_value) + score_weights[key] = float(max(-100.0, min(float(v), 100.0))) + else: + score_weights[key] = float(default_value) + + multipliers_default = _DEFAULT_RECOMMENDATION["multipliers"] + multipliers_raw = value.get("multipliers") + if multipliers_raw is None: + multipliers_obj: dict[str, Any] = {} + elif not isinstance(multipliers_raw, dict): + if strict: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid random.recommendation.multipliers", status_code=400) + multipliers_obj = {} + else: + multipliers_obj = multipliers_raw + if strict: + for k in multipliers_obj.keys(): + if str(k) not in multipliers_default: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid random.recommendation.multipliers", status_code=400) + + multipliers: dict[str, float] = {} + for key, default_value in multipliers_default.items(): + if key in multipliers_obj: + v = _as_float(multipliers_obj.get(key)) + if v is None: + if strict: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid random.recommendation.multipliers", status_code=400) + v = float(default_value) + multipliers[key] = float(max(0.0, min(float(v), 100.0))) + else: + multipliers[key] = float(default_value) + + return { + "pick_mode": pick_mode, + "temperature": temperature, + "score_weights": score_weights, + "multipliers": multipliers, + } + + +async def _load_settings_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + settings = data.get("settings") if "settings" in data else data + if not isinstance(settings, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid settings", status_code=400) + if not settings: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing fields", status_code=400) + + return settings + + +@router.get("/settings") +async def get_settings( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + values = await fetch_runtime_settings(engine) + runtime = runtime_config_from_values(values) + + random_defaults = dict(_DEFAULT_SETTINGS["random"]) + if isinstance(runtime.random_defaults, dict): + for k in list(random_defaults.keys()): + if k == "recommendation": + continue + if k in runtime.random_defaults: + random_defaults[k] = runtime.random_defaults[k] + random_defaults["recommendation"] = _normalize_recommendation(runtime.random_defaults.get("recommendation"), strict=False) + else: + random_defaults["recommendation"] = _normalize_recommendation(None, strict=False) + + return { + "ok": True, + "settings": { + "proxy": { + "enabled": bool(runtime.proxy_enabled), + "fail_closed": bool(runtime.proxy_fail_closed), + "route_mode": runtime.proxy_route_mode, + "allowlist_domains": list(runtime.proxy_allowlist_domains), + "default_pool_id": str(runtime.proxy_default_pool_id) if runtime.proxy_default_pool_id is not None else "", + "route_pools": {k: str(v) for k, v in runtime.proxy_route_pools.items()}, + }, + "random": random_defaults, + "security": {"hide_origin_url_in_public_json": bool(runtime.hide_origin_url_in_public_json)}, + "rate_limit": dict(runtime.rate_limit), + }, + "request_id": rid, + } + + +@router.put("/settings") +async def update_settings( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + rid = get_or_create_request_id(request) + body = await _load_settings_json(request) + + actor = str(_claims.get("sub") or "admin").strip() or "admin" + updated_by = f"admin:{actor}" + + updates: list[tuple[str, Any]] = [] + proxy_enabled_override: bool | None = None + proxy_fail_closed_override: bool | None = None + + proxy = body.get("proxy") + if proxy is not None: + if not isinstance(proxy, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy", status_code=400) + + if "enabled" in proxy: + v = _as_bool(proxy.get("enabled")) + if v is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.enabled", status_code=400) + proxy_enabled_override = bool(v) + updates.append(("proxy.enabled", bool(v))) + + if "fail_closed" in proxy: + v = _as_bool(proxy.get("fail_closed")) + if v is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.fail_closed", status_code=400) + proxy_fail_closed_override = bool(v) + updates.append(("proxy.fail_closed", bool(v))) + + if "route_mode" in proxy: + route_mode = str(proxy.get("route_mode") or "").strip().lower() + if route_mode not in {"pixiv_only", "all", "allowlist", "off"}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.route_mode", status_code=400) + updates.append(("proxy.route_mode", route_mode)) + + if "allowlist_domains" in proxy: + domains = _as_str_list(proxy.get("allowlist_domains")) + if len(domains) > 200 or any(len(d) > 200 for d in domains): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.allowlist_domains", status_code=400) + updates.append(("proxy.allowlist_domains", domains)) + + if "default_pool_id" in proxy: + raw = proxy.get("default_pool_id") + if raw is None or raw == "": + updates.append(("proxy.default_pool_id", None)) + else: + try: + pool_id = int(raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.default_pool_id", status_code=400) from exc + if pool_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.default_pool_id", status_code=400) + updates.append(("proxy.default_pool_id", int(pool_id))) + + if "route_pools" in proxy: + raw = proxy.get("route_pools") + if raw is None: + updates.append(("proxy.route_pools", {})) + else: + if not isinstance(raw, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.route_pools", status_code=400) + route_pools: dict[str, int] = {} + if len(raw) > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.route_pools", status_code=400) + for k, v in raw.items(): + key = str(k or "").strip().lower().strip(".") + if not key or len(key) > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.route_pools", status_code=400) + try: + pool_id = int(v) + except Exception as exc: + raise ApiError( + code=ErrorCode.BAD_REQUEST, + message="Invalid proxy.route_pools", + status_code=400, + ) from exc + if pool_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid proxy.route_pools", status_code=400) + route_pools[key] = int(pool_id) + updates.append(("proxy.route_pools", route_pools)) + + random = body.get("random") + if random is not None: + if not isinstance(random, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid random", status_code=400) + + defaults: dict[str, Any] = {} + for key in ("default_attempts", "default_r18_strict", "fail_cooldown_ms", "strategy", "quality_samples"): + if key not in random: + continue + if key in {"default_attempts", "fail_cooldown_ms", "quality_samples"}: + try: + n = int(random.get(key)) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message=f"Invalid random.{key}", status_code=400) from exc + if key == "quality_samples": + if n < 1 or n > 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message=f"Invalid random.{key}", status_code=400) + elif n < 0 or n > 10_000_000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message=f"Invalid random.{key}", status_code=400) + defaults[key] = n + elif key == "strategy": + s = str(random.get(key) or "").strip().lower() + if s not in {"quality", "random"}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid random.strategy", status_code=400) + defaults[key] = s + else: + v = _as_bool(random.get(key)) + if v is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message=f"Invalid random.{key}", status_code=400) + defaults[key] = bool(v) + + if "recommendation" in random: + defaults["recommendation"] = _normalize_recommendation(random.get("recommendation"), strict=True) + + if defaults: + values = await fetch_runtime_settings(request.app.state.engine) + runtime = runtime_config_from_values(values) + existing_raw = runtime.random_defaults if isinstance(runtime.random_defaults, dict) else {} + if existing_raw: + merged = dict(existing_raw) + merged.update(defaults) + defaults = merged + updates.append(("random.defaults", defaults)) + + security = body.get("security") + if security is not None: + if not isinstance(security, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid security", status_code=400) + + if "hide_origin_url_in_public_json" in security: + v = _as_bool(security.get("hide_origin_url_in_public_json")) + if v is None: + raise ApiError( + code=ErrorCode.BAD_REQUEST, + message="Invalid security.hide_origin_url_in_public_json", + status_code=400, + ) + updates.append(("security.hide_origin_url_in_public_json", bool(v))) + + rate_limit = body.get("rate_limit") + if rate_limit is not None: + if not isinstance(rate_limit, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid rate_limit", status_code=400) + + for key, value in rate_limit.items(): + k = str(key or "").strip() + if not k or len(k) > 100: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid rate_limit key", status_code=400) + updates.append((f"rate_limit.{k}", value)) + + if not updates: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing fields", status_code=400) + + engine = request.app.state.engine + + if proxy_enabled_override is not None or proxy_fail_closed_override is not None: + values = await fetch_runtime_settings(engine) + runtime = runtime_config_from_values(values) + desired_enabled = proxy_enabled_override if proxy_enabled_override is not None else bool(runtime.proxy_enabled) + desired_fail_closed = ( + proxy_fail_closed_override if proxy_fail_closed_override is not None else bool(runtime.proxy_fail_closed) + ) + if desired_enabled and desired_fail_closed: + async with engine.connect() as conn: + result = await conn.exec_driver_sql("SELECT COUNT(*) FROM proxy_endpoints WHERE enabled=1;") + enabled_proxy_count = int(result.scalar_one()) + if enabled_proxy_count <= 0: + raise ApiError( + code=ErrorCode.PROXY_REQUIRED, + message="Proxy required (fail-closed) but no enabled proxies", + status_code=400, + ) + + for key, value in updates: + await set_runtime_setting(engine, key=key, value=value, updated_by=updated_by) + + return {"ok": True, "updated": len(updates), "request_id": rid} diff --git a/backend/app/api/admin/stats.py b/backend/app/api/admin/stats.py new file mode 100644 index 0000000..79327c2 --- /dev/null +++ b/backend/app/api/admin/stats.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from dataclasses import asdict +from typing import Any + +from fastapi import APIRouter, Depends, Request + +from app.api.admin.deps import get_admin_claims +from app.core.request_id import get_or_create_request_id + +router = APIRouter() + + +@router.get("/stats/random") +async def get_random_stats( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + + stats = getattr(request.app.state, "random_request_stats", None) + if stats is None: + snapshot = { + "total_requests": 0, + "total_ok": 0, + "total_error": 0, + "in_flight": 0, + "window_seconds": 60, + "last_window_requests": 0, + "last_window_ok": 0, + "last_window_error": 0, + "last_window_success_rate": 0.0, + } + else: + snap = await stats.snapshot() + snapshot = asdict(snap) + + return {"ok": True, "stats": snapshot, "request_id": rid} + diff --git a/backend/app/api/admin/summary.py b/backend/app/api/admin/summary.py new file mode 100644 index 0000000..a9915e2 --- /dev/null +++ b/backend/app/api/admin/summary.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import json +from typing import Any + +from fastapi import APIRouter, Depends, Request + +from app.api.admin.deps import get_admin_claims +from app.core.request_id import get_or_create_request_id +from app.db.session import with_sqlite_busy_retry + +router = APIRouter() + + +def _worker_last_seen_from_value_json(value_json: str | None) -> str | None: + if value_json is None: + return None + raw = str(value_json or "").strip() + if not raw: + return None + try: + data = json.loads(raw) + except Exception: + return None + if isinstance(data, dict): + at = data.get("at") + return str(at) if isinstance(at, str) and at.strip() else None + if isinstance(data, str): + return data.strip() or None + return None + + +@router.get("/summary") +async def get_summary( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + + async def _op() -> dict[str, Any]: + async with engine.connect() as conn: + images_total = int((await conn.exec_driver_sql("SELECT COUNT(*) FROM images;")).scalar_one()) + images_enabled = int( + (await conn.exec_driver_sql("SELECT COUNT(*) FROM images WHERE status=1;")).scalar_one() + ) + + missing_tags = int( + ( + await conn.exec_driver_sql( + """ +SELECT COUNT(*) +FROM images +WHERE status=1 + AND id NOT IN (SELECT DISTINCT image_id FROM image_tags); +""".strip() + ) + ).scalar_one() + ) + missing_geometry = int( + ( + await conn.exec_driver_sql( + "SELECT COUNT(*) FROM images WHERE status=1 AND (width IS NULL OR height IS NULL);" + ) + ).scalar_one() + ) + missing_r18 = int( + (await conn.exec_driver_sql("SELECT COUNT(*) FROM images WHERE status=1 AND x_restrict IS NULL;")).scalar_one() + ) + missing_ai = int( + (await conn.exec_driver_sql("SELECT COUNT(*) FROM images WHERE status=1 AND ai_type IS NULL;")).scalar_one() + ) + missing_illust_type = int( + (await conn.exec_driver_sql("SELECT COUNT(*) FROM images WHERE status=1 AND illust_type IS NULL;")).scalar_one() + ) + missing_user = int( + (await conn.exec_driver_sql("SELECT COUNT(*) FROM images WHERE status=1 AND user_id IS NULL;")).scalar_one() + ) + missing_title = int( + ( + await conn.exec_driver_sql( + "SELECT COUNT(*) FROM images WHERE status=1 AND (title IS NULL OR TRIM(title)='');" + ) + ).scalar_one() + ) + missing_created_at = int( + ( + await conn.exec_driver_sql( + "SELECT COUNT(*) FROM images WHERE status=1 AND (created_at_pixiv IS NULL OR TRIM(created_at_pixiv)='');" + ) + ).scalar_one() + ) + missing_popularity = int( + ( + await conn.exec_driver_sql( + "SELECT COUNT(*) FROM images " + "WHERE status=1 AND (bookmark_count IS NULL OR view_count IS NULL OR comment_count IS NULL);" + ) + ).scalar_one() + ) + + tokens_total = int((await conn.exec_driver_sql("SELECT COUNT(*) FROM pixiv_tokens;")).scalar_one()) + tokens_enabled = int( + (await conn.exec_driver_sql("SELECT COUNT(*) FROM pixiv_tokens WHERE enabled=1;")).scalar_one() + ) + + proxies_total = int((await conn.exec_driver_sql("SELECT COUNT(*) FROM proxy_endpoints;")).scalar_one()) + proxies_enabled = int( + (await conn.exec_driver_sql("SELECT COUNT(*) FROM proxy_endpoints WHERE enabled=1;")).scalar_one() + ) + + pools_total = int((await conn.exec_driver_sql("SELECT COUNT(*) FROM proxy_pools;")).scalar_one()) + pools_enabled = int( + (await conn.exec_driver_sql("SELECT COUNT(*) FROM proxy_pools WHERE enabled=1;")).scalar_one() + ) + + bindings_total = int((await conn.exec_driver_sql("SELECT COUNT(*) FROM token_proxy_bindings;")).scalar_one()) + + jobs_counts: dict[str, int] = {} + rows = (await conn.exec_driver_sql("SELECT status, COUNT(*) AS c FROM jobs GROUP BY status;")).fetchall() + for status, count in rows: + jobs_counts[str(status)] = int(count) + + worker_last_seen_json = ( + await conn.exec_driver_sql("SELECT value_json FROM runtime_settings WHERE key = ?;", ("worker.last_seen_at",)) + ).scalar_one_or_none() + worker_last_seen_at = _worker_last_seen_from_value_json(str(worker_last_seen_json) if worker_last_seen_json is not None else None) + + return { + "images": {"total": images_total, "enabled": images_enabled}, + "hydration": { + "enabled_images_total": images_enabled, + "missing": { + "tags": missing_tags, + "geometry": missing_geometry, + "r18": missing_r18, + "ai": missing_ai, + "illust_type": missing_illust_type, + "user": missing_user, + "title": missing_title, + "created_at": missing_created_at, + "popularity": missing_popularity, + }, + }, + "tokens": {"total": tokens_total, "enabled": tokens_enabled}, + "proxies": {"endpoints_total": proxies_total, "endpoints_enabled": proxies_enabled}, + "proxy_pools": {"total": pools_total, "enabled": pools_enabled}, + "bindings": {"total": bindings_total}, + "jobs": {"counts": jobs_counts}, + "worker": {"last_seen_at": worker_last_seen_at}, + } + + counts = await with_sqlite_busy_retry(_op) + return {"ok": True, "counts": counts, "request_id": rid} diff --git a/backend/app/api/admin/tokens.py b/backend/app/api/admin/tokens.py new file mode 100644 index 0000000..3df2031 --- /dev/null +++ b/backend/app/api/admin/tokens.py @@ -0,0 +1,450 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Request + +from app.api.admin.deps import get_admin_claims +from app.core.crypto import FieldEncryptor, mask_secret +from app.core.errors import ApiError, ErrorCode +from app.core.proxy_routing import select_proxy_uri_for_url +from app.core.request_id import get_or_create_request_id +from app.core.runtime_settings import load_runtime_config +from app.core.time import iso_utc_ms +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker, with_sqlite_busy_retry +from app.pixiv.oauth import OAUTH_TOKEN_PATH, PixivOauthConfig, PixivOauthError, refresh_access_token +from app.pixiv.refresh_backoff import refresh_backoff_seconds + +router = APIRouter() + + +def _parse_bool(value: Any, *, default: bool) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return default + + +def _parse_bool_strict(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return None + + +async def _load_create_token_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + + refresh_token = str(data.get("refresh_token") or "").strip() + if not refresh_token: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing refresh_token", status_code=400) + if len(refresh_token) > 2048: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported refresh_token", status_code=400) + + label_raw = data.get("label") + label = str(label_raw).strip() if label_raw is not None else None + label = label if label else None + + enabled = _parse_bool(data.get("enabled"), default=True) + + weight_raw = data.get("weight", 1.0) + try: + weight = float(weight_raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported weight", status_code=400) from exc + if weight < 0.0 or weight > 100.0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported weight", status_code=400) + + return { + "label": label, + "enabled": enabled, + "weight": weight, + "refresh_token": refresh_token, + } + + +async def _load_update_token_json(request: Request) -> dict[str, Any]: + try: + data = await request.json() + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) from exc + + if not isinstance(data, dict): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid JSON body", status_code=400) + if not data: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing fields", status_code=400) + + out: dict[str, Any] = {} + + if "label" in data: + label_raw = data.get("label") + if label_raw is None: + label = None + else: + label = str(label_raw).strip() or None + if label is not None and len(label) > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported label", status_code=400) + out["label"] = label + + if "enabled" in data: + enabled = _parse_bool_strict(data.get("enabled")) + if enabled is None: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported enabled", status_code=400) + out["enabled"] = bool(enabled) + + if "weight" in data: + weight_raw = data.get("weight") + try: + weight = float(weight_raw) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported weight", status_code=400) from exc + if weight < 0.0 or weight > 100.0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported weight", status_code=400) + out["weight"] = float(weight) + + if not out: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Missing fields", status_code=400) + + return out + + +@router.get("/tokens") +async def list_tokens( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + tokens = ( + ( + await session.execute( + sa.select(PixivToken).order_by(PixivToken.id.desc()) + ) + ) + .scalars() + .all() + ) + + items = [ + { + "id": str(t.id), + "label": t.label, + "enabled": bool(t.enabled), + "refresh_token_masked": t.refresh_token_masked, + "weight": float(t.weight), + "error_count": int(t.error_count or 0), + "backoff_until": t.backoff_until, + "last_ok_at": t.last_ok_at, + "last_fail_at": t.last_fail_at, + "last_error_code": t.last_error_code, + "last_error_msg": t.last_error_msg, + } + for t in tokens + ] + + return {"ok": True, "items": items, "request_id": rid} + + +@router.post("/tokens") +async def create_token( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + rid = get_or_create_request_id(request) + body = await _load_create_token_json(request) + + settings = request.app.state.settings + try: + encryptor = FieldEncryptor.from_key(settings.field_encryption_key) + except Exception as exc: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="Encryption not configured", status_code=500) from exc + + refresh_token = str(body["refresh_token"]) + refresh_token_enc = encryptor.encrypt_text(refresh_token) + refresh_token_masked = mask_secret(refresh_token) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + async with Session() as session: + row = PixivToken( + label=body["label"], + enabled=1 if bool(body["enabled"]) else 0, + refresh_token_enc=refresh_token_enc, + refresh_token_masked=refresh_token_masked, + weight=float(body["weight"]), + ) + session.add(row) + await session.commit() + await session.refresh(row) + + return {"ok": True, "token_id": str(row.id), "request_id": rid} + + +@router.put("/tokens/{token_id}") +async def update_token( + token_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if token_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid token id", status_code=400) + + rid = get_or_create_request_id(request) + body = await _load_update_token_json(request) + + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + row = await session.get(PixivToken, token_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Token not found", status_code=404) + + if "label" in body: + row.label = body["label"] + if "enabled" in body: + row.enabled = 1 if bool(body["enabled"]) else 0 + if "weight" in body: + row.weight = float(body["weight"]) + + row.updated_at = now + + await session.commit() + + return {"ok": True, "token_id": str(token_id), "request_id": rid} + + +@router.delete("/tokens/{token_id}") +async def delete_token( + token_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if token_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid token id", status_code=400) + + rid = get_or_create_request_id(request) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + row = await session.get(PixivToken, token_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Token not found", status_code=404) + + await session.execute(sa.delete(TokenProxyBinding).where(TokenProxyBinding.token_id == int(token_id))) + await session.delete(row) + await session.commit() + + return {"ok": True, "token_id": str(token_id), "request_id": rid} + + return await with_sqlite_busy_retry(_op) + + +@router.post("/tokens/{token_id}/test-refresh") +async def test_refresh_token( + token_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if token_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid token id", status_code=400) + + rid = get_or_create_request_id(request) + + settings = request.app.state.settings + try: + encryptor = FieldEncryptor.from_key(settings.field_encryption_key) + except Exception as exc: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="Encryption not configured", status_code=500) from exc + + client_id = (settings.pixiv_oauth_client_id or "").strip() + client_secret = (settings.pixiv_oauth_client_secret or "").strip() + if not client_id or not client_secret: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="Pixiv OAuth not configured", status_code=500) + + config = PixivOauthConfig( + client_id=client_id, + client_secret=client_secret, + hash_secret=(settings.pixiv_oauth_hash_secret or "").strip() or None, + ) + + transport = getattr(request.app.state, "httpx_transport", None) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + row = await session.get(PixivToken, token_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Token not found", status_code=404) + + try: + refresh_token = encryptor.decrypt_text(row.refresh_token_enc) + except Exception as exc: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="Invalid stored token", status_code=500) from exc + + now = iso_utc_ms() + + try: + runtime = await load_runtime_config(engine) + oauth_url = config.base_url.rstrip("/") + OAUTH_TOKEN_PATH + picked_proxy = await select_proxy_uri_for_url( + engine, + settings, + runtime, + url=oauth_url, + token_id=int(token_id), + ) + proxy_uri = picked_proxy.uri if picked_proxy is not None else None + + token = await refresh_access_token( + refresh_token=refresh_token, + config=config, + transport=transport, + proxy=proxy_uri, + ) + except PixivOauthError as exc: + new_error_count = int(row.error_count or 0) + 1 + backoff_s = refresh_backoff_seconds(attempt=new_error_count, status_code=exc.status_code) + backoff_until = ( + iso_utc_ms(datetime.now(timezone.utc) + timedelta(seconds=backoff_s)) if backoff_s > 0 else None + ) + + row.error_count = new_error_count + row.backoff_until = backoff_until + row.last_fail_at = now + row.last_error_code = ErrorCode.TOKEN_REFRESH_FAILED.value + row.last_error_msg = str(exc)[:500] + row.updated_at = now + + await session.commit() + + raise ApiError( + code=ErrorCode.TOKEN_REFRESH_FAILED, + message="Token refresh failed", + status_code=502, + details={"upstream_status": exc.status_code or 0, "backoff_until": backoff_until or ""}, + ) from exc + except Exception as exc: + new_error_count = int(row.error_count or 0) + 1 + backoff_s = refresh_backoff_seconds(attempt=new_error_count, status_code=None) + backoff_until = ( + iso_utc_ms(datetime.now(timezone.utc) + timedelta(seconds=backoff_s)) if backoff_s > 0 else None + ) + + row.error_count = new_error_count + row.backoff_until = backoff_until + row.last_fail_at = now + row.last_error_code = ErrorCode.TOKEN_REFRESH_FAILED.value + row.last_error_msg = "exception" + row.updated_at = now + + await session.commit() + + raise ApiError( + code=ErrorCode.TOKEN_REFRESH_FAILED, + message="Token refresh failed", + status_code=502, + details={"backoff_until": backoff_until or ""}, + ) from exc + + row.error_count = 0 + row.backoff_until = None + row.last_ok_at = now + row.last_fail_at = None + row.last_error_code = None + row.last_error_msg = None + row.updated_at = now + + rotated = token.refresh_token + if rotated: + row.refresh_token_enc = encryptor.encrypt_text(rotated) + row.refresh_token_masked = mask_secret(rotated) + + await session.commit() + + proxy_details: dict[str, Any] | None = None + if picked_proxy is not None: + proxy_details = {"endpoint_id": str(picked_proxy.endpoint_id), "pool_id": str(picked_proxy.pool_id)} + + return { + "ok": True, + "expires_in": int(token.expires_in), + "user_id": token.user_id, + "proxy": proxy_details, + "request_id": rid, + } + + +@router.post("/tokens/{token_id}/reset-failures") +async def reset_failures( + token_id: int, + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> dict[str, Any]: + _ = _claims + if token_id <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid token id", status_code=400) + + rid = get_or_create_request_id(request) + + now = iso_utc_ms() + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + row = await session.get(PixivToken, token_id) + if row is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Token not found", status_code=404) + + row.error_count = 0 + row.backoff_until = None + row.last_fail_at = None + row.last_error_code = None + row.last_error_msg = None + row.updated_at = now + + await session.commit() + + return {"ok": True, "token_id": str(token_id), "request_id": rid} diff --git a/backend/app/api/metrics.py b/backend/app/api/metrics.py new file mode 100644 index 0000000..56ff6cc --- /dev/null +++ b/backend/app/api/metrics.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import Response +from prometheus_client.exposition import CONTENT_TYPE_LATEST, generate_latest +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.api.admin.deps import get_admin_claims +from app.core.metrics import ( + METRICS_LAST_SCRAPE_SUCCESS, + METRICS_SCRAPE_ERRORS_TOTAL, + ensure_known_keys, + set_jobs_status_counts, + set_proxy_state_counts, +) +from app.core.time import iso_utc_ms + +router = APIRouter() + + +async def _query_job_status_counts(engine: AsyncEngine) -> dict[str, int]: + sql = "SELECT status, COUNT(*) AS c FROM jobs GROUP BY status" + async with engine.connect() as conn: + result = await conn.exec_driver_sql(sql) + counts: dict[str, int] = {} + for row in result.fetchall(): + status = str(row[0]) + counts[status] = int(row[1]) + return counts + + +async def _query_proxy_state_counts(engine: AsyncEngine) -> dict[str, int]: + now = iso_utc_ms() + sql = """ +SELECT + COUNT(*) AS total, + SUM(CASE WHEN enabled = 1 THEN 1 ELSE 0 END) AS enabled, + SUM(CASE WHEN enabled = 1 AND blacklisted_until IS NOT NULL AND blacklisted_until > :now THEN 1 ELSE 0 END) AS blacklisted, + SUM(CASE WHEN enabled = 1 + AND (blacklisted_until IS NULL OR blacklisted_until <= :now) + AND last_ok_at IS NOT NULL + AND (last_fail_at IS NULL OR last_ok_at >= last_fail_at) + THEN 1 ELSE 0 END) AS healthy, + SUM(CASE WHEN enabled = 1 + AND (blacklisted_until IS NULL OR blacklisted_until <= :now) + AND (last_ok_at IS NULL OR (last_fail_at IS NOT NULL AND last_fail_at > last_ok_at)) + THEN 1 ELSE 0 END) AS unhealthy +FROM proxy_endpoints; +""".strip() + async with engine.connect() as conn: + result = await conn.exec_driver_sql(sql, {"now": now}) + row = result.fetchone() + if row is None: + return {"total": 0, "enabled": 0, "healthy": 0, "unhealthy": 0, "blacklisted": 0} + return { + "total": int(row[0] or 0), + "enabled": int(row[1] or 0), + "blacklisted": int(row[2] or 0), + "healthy": int(row[3] or 0), + "unhealthy": int(row[4] or 0), + } + + +@router.get("/metrics") +async def metrics( + request: Request, + _claims: dict[str, Any] = Depends(get_admin_claims), +) -> Response: + engine: AsyncEngine | None = getattr(request.app.state, "engine", None) + if engine is not None: + try: + job_counts = await _query_job_status_counts(engine) + set_jobs_status_counts(job_counts) + + proxy_counts = await _query_proxy_state_counts(engine) + proxy_counts = ensure_known_keys(["total", "enabled", "healthy", "unhealthy", "blacklisted"], proxy_counts) + set_proxy_state_counts(proxy_counts) + + METRICS_LAST_SCRAPE_SUCCESS.set(1) + except Exception: + METRICS_SCRAPE_ERRORS_TOTAL.inc() + METRICS_LAST_SCRAPE_SUCCESS.set(0) + + content = generate_latest() + return Response(content=content, media_type=CONTENT_TYPE_LATEST) + diff --git a/backend/app/api/public/authors.py b/backend/app/api/public/authors.py new file mode 100644 index 0000000..185a8eb --- /dev/null +++ b/backend/app/api/public/authors.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.db.authors_list import list_authors as db_list_authors +from app.db.session import create_sessionmaker + +router = APIRouter() + + +@router.get("/authors") +async def list_authors( + request: Request, + q: str | None = None, + limit: int = 50, + cursor: str | None = None, +) -> Any: + if limit < 1 or limit > 100: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported limit", status_code=400) + + q_norm = (q or "").strip() + if q_norm and len(q_norm) > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported q", status_code=400) + + cursor_i: int | None = None + cursor_raw = (cursor or "").strip() + if cursor_raw: + if not cursor_raw.isdigit(): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + cursor_i = int(cursor_raw) + if cursor_i <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + async with Session() as session: + items, next_cursor = await db_list_authors(session, limit=limit, cursor=cursor_i, q=q_norm or None) + + rid = get_or_create_request_id(request) + resp = JSONResponse( + status_code=200, + content={ + "ok": True, + "items": [ + { + "user_id": str(item.user_id), + "user_name": item.user_name, + "count_images": item.count_images, + } + for item in items + ], + "next_cursor": str(next_cursor) if next_cursor is not None else "", + "request_id": rid, + }, + ) + return resp + diff --git a/backend/app/api/public/docs_page.py b/backend/app/api/public/docs_page.py new file mode 100644 index 0000000..75f525d --- /dev/null +++ b/backend/app/api/public/docs_page.py @@ -0,0 +1,361 @@ +from __future__ import annotations + +from fastapi import APIRouter, Request +from fastapi.responses import HTMLResponse + +router = APIRouter() + + +def _build_docs_html(*, base_url: str) -> str: + base = (base_url or "").rstrip("/") + if not base: + base = "" + + def u(path: str) -> str: + path_norm = (path or "").strip() + if not path_norm.startswith("/"): + path_norm = "/" + path_norm + return f"{base}{path_norm}" + + examples = { + "img_default": u("/random"), + "img_redirect": u("/random?redirect=1"), + "json_full": u("/random?format=json"), + "json_simple": u("/random?format=simple_json"), + "r18_only": u("/random?r18=1"), + "safe_only": u("/random?r18=0"), + "any_r18": u("/random?r18=2"), + "portrait": u("/random?orientation=portrait"), + "landscape": u("/random?orientation=landscape"), + "adaptive": u("/random?adaptive=1"), + "tag_loli": u("/random?included_tags=loli"), + "pure_random": u("/random?strategy=random"), + "quality_strong": u( + "/random?strategy=quality&quality_samples=200&min_pixels=2000000&min_bookmarks=100&illust_type=illust&ai_type=0" + ), + "complex": u( + "/random?format=json&r18=1&illust_type=illust&orientation=portrait&min_pixels=2500000&min_bookmarks=25&min_comments=50&included_tags=loli" + ), + "tags_api": u("/tags"), + "authors_api": u("/authors"), + "swagger": u("/api/docs"), + "openapi": u("/openapi.json"), + } + + return f""" + + + + + Random Mage Docs + + + +
+
+

Random Mage · 随机图片 API

+

质量优先随机 + 强力筛选，支持标签/热度/分辨率/R18/AI/作品类型等组合查询。

+
+ /random 直接出图 + format=json 返回 JSON + strategy=random 纯随机 + /api/docs Swagger +
+
+ +
+
+

1) 最常用（直接出图）

+

默认返回图片（不是 JSON）。想要稳定 URL 可加 redirect=1 跳转到本站缓存/代理路径。

+ 
{examples["img_default"]}
+{examples["img_redirect"]}
+
+ +
+

2) JSON / 调试信息

+

format=json 返回完整数据（含 tags / debug）；format=simple_json 更轻量。

+ 
{examples["json_full"]}
+{examples["json_simple"]}
+
+ +
+

3) R18 / 作品类型 / AI

+

R18 用 r18 控制：0=仅全年龄（默认），1=仅R18，2=都可。

+ 
{examples["safe_only"]}
+{examples["r18_only"]}
+{examples["any_r18"]}
+
+ 提示：如果你库里有不少作品还没补全 x_restrict，且你想“更容易命中”，可用 + r18=0&r18_strict=0（允许未知的 x_restrict）。 +
+
+ +
+

4) 横竖图 / 自适应

+

orientation 支持：any / portrait / landscape / square。自适应用 adaptive=1：移动端默认竖图+更低像素门槛，PC 默认横图+更高门槛（不会覆盖你显式传入的过滤条件）。

+ 
{examples["portrait"]}
+{examples["landscape"]}
+{examples["adaptive"]}
+
+
+ +
+

筛选语法速记

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
参数说明
strategy + 随机策略：quality（默认，质量优先） / random（纯随机）。
+ quality 会先抽样 N 张候选再按热度/分辨率打分选图。 +
quality_samples + 质量抽样数量（1–1000）。越大越“挑剔”，但 DB 开销也更高。
+ 示例：{examples["quality_strong"]} +
included_tags + 必须包含的标签（AND）。可重复传参：included_tags=a&included_tags=b。
+ 也支持用 | 分隔：included_tags=a|b 等价于同时包含 a 与 b。
+ 示例：{examples["tag_loli"]} +
excluded_tags必须不包含的标签（OR）。可重复传参。示例：excluded_tags=AI生成
min_width
min_height
min_pixels		分辨率门槛：宽/高/像素数（例如 2000000 ≈ 2MP）。
min_bookmarks
min_views
min_comments		热度门槛：收藏/浏览/评论（需要补全元数据后更准确）。
ai_typeany / 0（非AI） / 1（AI）。
illust_typeany / illust / manga / ugoira。
user_id只返回某个作者的作品（Pixiv 用户 ID）。
created_from
created_to		按发布时间筛选（ISO8601，例如 2024-01-01T00:00:00Z）。
seed固定随机种子（同参数 + 同 seed 更容易复现）。
attempts上游失败时重试次数（1–10）。
+
+ 其它接口：/tags（标签列表）、/authors（作者列表）。
+ OpenAPI：/openapi.json，Swagger：/api/docs。
+
+
+ +
+

复杂示例（组合查询）

+

示例：R18 + 插画 + 竖图 + 250万像素以上 + 收藏≥25 + 评论≥50 + 标签包含 loli，并返回 JSON。

+ 
{examples["complex"]}
+
+
+ + +""" + + +@router.get("/docs", include_in_schema=False) +async def docs_page(request: Request) -> HTMLResponse: + html = _build_docs_html(base_url=str(getattr(request, "base_url", "") or "").rstrip("/")) + return HTMLResponse(content=html, status_code=200, headers={"Cache-Control": "no-store"}) + diff --git a/backend/app/api/public/healthz.py b/backend/app/api/public/healthz.py new file mode 100644 index 0000000..7585987 --- /dev/null +++ b/backend/app/api/public/healthz.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import json +import os +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.errors import ErrorCode, error_body +from app.core.request_id import get_or_create_request_id, set_request_id_header, set_request_id_on_state +from app.db.session import with_sqlite_busy_retry + +router = APIRouter() + +_JOB_STATUSES: tuple[str, ...] = ( + "pending", + "running", + "paused", + "canceled", + "completed", + "failed", + "dlq", +) + + +def _ensure_sqlite_dir(engine: AsyncEngine) -> None: + url = engine.url + if url.get_backend_name() != "sqlite": + return + db_path = url.database + if not db_path or db_path == ":memory:": + return + Path(db_path).expanduser().resolve().parent.mkdir(parents=True, exist_ok=True) + + +async def _check_db(engine: AsyncEngine) -> bool: + try: + _ensure_sqlite_dir(engine) + async with engine.connect() as conn: + await conn.exec_driver_sql("SELECT 1") + return True + except Exception: + return False + + +def _parse_iso_utc(value: str) -> datetime | None: + raw = (value or "").strip() + if not raw: + return None + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(raw) + except Exception: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +async def _query_worker_last_seen(engine: AsyncEngine) -> tuple[str | None, str]: + sql = "SELECT value_json FROM runtime_settings WHERE key = ?" + + async def _op() -> str | None: + async with engine.connect() as conn: + result = await conn.exec_driver_sql(sql, ("worker.last_seen_at",)) + row = result.fetchone() + return str(row[0]) if row is not None else None + + try: + raw = await with_sqlite_busy_retry(_op) + except Exception: + return None, "runtime_settings_unavailable" + + if raw is None: + return None, "no_heartbeat" + + try: + value = json.loads(raw) + except Exception: + return None, "invalid_json" + + if isinstance(value, dict): + at = value.get("at") + if isinstance(at, str): + return at, "ok" + return None, "invalid_value" + + if isinstance(value, str): + return value, "ok" + + return None, "invalid_value" + + +async def _query_queue_status_counts(engine: AsyncEngine) -> tuple[dict[str, int] | None, str]: + sql = "SELECT status, COUNT(*) AS c FROM jobs GROUP BY status" + + async def _op() -> dict[str, int]: + async with engine.connect() as conn: + result = await conn.exec_driver_sql(sql) + counts: dict[str, int] = {} + for row in result.fetchall(): + counts[str(row[0])] = int(row[1]) + return counts + + try: + counts = await with_sqlite_busy_retry(_op) + except Exception: + return None, "jobs_unavailable" + + for status in _JOB_STATUSES: + counts.setdefault(status, 0) + return counts, "ok" + + +@router.get("/healthz") +async def healthz(request: Request) -> Any: + rid = get_or_create_request_id(request) + set_request_id_on_state(request, rid) + + engine: AsyncEngine | None = getattr(request.app.state, "engine", None) + db_ok = await _check_db(engine) if engine is not None else False + + if db_ok: + try: + stale_after_s = int((os.environ.get("WORKER_HEARTBEAT_STALE_SECONDS") or "60").strip() or "60") + except Exception: + stale_after_s = 60 + stale_after_s = max(1, min(int(stale_after_s), 24 * 60 * 60)) + + worker_last_seen_at, worker_reason = await _query_worker_last_seen(engine) # type: ignore[arg-type] + worker_ok = False + if worker_last_seen_at is not None: + last_seen_dt = _parse_iso_utc(worker_last_seen_at) + if last_seen_dt is None: + worker_reason = "invalid_timestamp" + else: + worker_ok = datetime.now(timezone.utc) - last_seen_dt <= timedelta(seconds=stale_after_s) + if not worker_ok: + worker_reason = "stale" + + queue_counts, queue_reason = await _query_queue_status_counts(engine) # type: ignore[arg-type] + queue_ok = queue_counts is not None + + resp = JSONResponse( + status_code=200, + content={ + "ok": True, + "db_ok": True, + "worker_ok": worker_ok, + "queue_ok": queue_ok, + "worker": { + "last_seen_at": worker_last_seen_at, + "stale_after_s": stale_after_s, + "reason": worker_reason, + }, + "queue": { + "counts": queue_counts or {}, + "reason": queue_reason, + }, + "request_id": rid, + }, + ) + else: + resp = JSONResponse( + status_code=503, + content=error_body( + code=ErrorCode.INTERNAL_ERROR, + message="Database unavailable", + request_id=rid, + details={"db_ok": False}, + ), + ) + + set_request_id_header(resp, rid) + return resp diff --git a/backend/app/api/public/images.py b/backend/app/api/public/images.py new file mode 100644 index 0000000..6e66198 --- /dev/null +++ b/backend/app/api/public/images.py @@ -0,0 +1,333 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from fastapi import APIRouter, Query, Request +from fastapi.responses import JSONResponse + +from app.core.errors import ApiError, ErrorCode +from app.core.http_stream import stream_url +import sqlalchemy as sa + +from app.core.pixiv_urls import ALLOWED_IMAGE_EXTS +from app.core.request_id import get_or_create_request_id, set_request_id_header, set_request_id_on_state +from app.core.runtime_settings import load_runtime_config +from app.core.time import iso_utc_ms +from app.core.proxy_routing import select_proxy_uri_for_url +from app.db.images_get import get_image_by_id +from app.db.images_list import list_images as db_list_images +from app.db.images_mark import mark_image_failure, mark_image_ok +from app.db.models.image_tags import ImageTag +from app.db.session import create_sessionmaker +from app.db.tags_get import get_tag_names_for_image +from app.jobs.enqueue import enqueue_opportunistic_hydrate_metadata + +router = APIRouter() + +_MAX_TAG_FILTERS = 50 + + +@router.get("/images") +async def list_images( + request: Request, + limit: int = 50, + cursor: str | None = None, + r18: int = 0, + r18_strict: int = 1, + ai_type: str = "any", + orientation: str = "any", + min_width: int = 0, + min_height: int = 0, + min_pixels: int = 0, + included_tags: list[str] | None = Query(default=None), + excluded_tags: list[str] | None = Query(default=None), + user_id: int | None = None, + illust_id: int | None = None, + created_from: str | None = None, + created_to: str | None = None, +) -> Any: + if limit < 1 or limit > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported limit", status_code=400) + + cursor_i: int | None = None + cursor_raw = (cursor or "").strip() + if cursor_raw: + if not cursor_raw.isdigit(): + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + cursor_i = int(cursor_raw) + if cursor_i <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported cursor", status_code=400) + + if r18 not in {0, 1, 2}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported r18", status_code=400) + if r18_strict not in {0, 1}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported r18_strict", status_code=400) + + ai_type_raw = (ai_type or "any").strip().lower() + ai_type_i: int | None = None + if ai_type_raw in {"", "any"}: + ai_type_i = None + elif ai_type_raw in {"0", "1"}: + ai_type_i = int(ai_type_raw) + else: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported ai_type", status_code=400) + + orientation = (orientation or "").strip().lower() + orientation_map = {"any": None, "portrait": 1, "landscape": 2, "square": 3} + if orientation not in orientation_map: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported orientation", status_code=400) + + if min_width < 0 or min_height < 0 or min_pixels < 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported min_*", status_code=400) + + def _parse_tags(values: list[str] | None) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for raw in values or []: + for part in str(raw).split("|"): + name = part.strip() + if not name or name in seen: + continue + seen.add(name) + out.append(name) + return out + + included = _parse_tags(included_tags) + excluded = _parse_tags(excluded_tags) + if len(included) > _MAX_TAG_FILTERS or len(excluded) > _MAX_TAG_FILTERS: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Too many tag filters", status_code=400) + + if user_id is not None and int(user_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported user_id", status_code=400) + if illust_id is not None and int(illust_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported illust_id", status_code=400) + + def _normalize_iso_utc(value: str) -> str: + raw = (value or "").strip() + if not raw: + raise ValueError("empty datetime") + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + dt = datetime.fromisoformat(raw) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + dt = dt.astimezone(timezone.utc).replace(microsecond=0) + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + created_from_norm: str | None = None + created_to_norm: str | None = None + try: + if created_from is not None: + created_from_norm = _normalize_iso_utc(created_from) + if created_to is not None: + created_to_norm = _normalize_iso_utc(created_to) + except Exception: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported created_*", status_code=400) + + if created_from_norm is not None and created_to_norm is not None: + if created_from_norm > created_to_norm: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="created_from > created_to", status_code=400) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + async with Session() as session: + images, next_cursor = await db_list_images( + session, + limit=limit, + cursor=cursor_i, + r18=r18, + r18_strict=bool(r18_strict), + orientation=orientation_map[orientation], + ai_type=ai_type_i, + min_width=min_width, + min_height=min_height, + min_pixels=min_pixels, + included_tags=included, + excluded_tags=excluded, + user_id=user_id, + illust_id=illust_id, + created_from=created_from_norm, + created_to=created_to_norm, + ) + + rid = get_or_create_request_id(request) + set_request_id_on_state(request, rid) + + items = [ + { + "id": str(img.id), + "illust_id": str(img.illust_id), + "page_index": img.page_index, + "ext": img.ext, + "width": img.width, + "height": img.height, + "x_restrict": img.x_restrict, + "ai_type": img.ai_type, + "bookmark_count": getattr(img, "bookmark_count", None), + "view_count": getattr(img, "view_count", None), + "comment_count": getattr(img, "comment_count", None), + "user": { + "id": str(img.user_id) if img.user_id is not None else None, + "name": img.user_name, + }, + "title": img.title, + "created_at_pixiv": img.created_at_pixiv, + } + for img in images + ] + + resp = JSONResponse( + status_code=200, + content={ + "ok": True, + "items": items, + "next_cursor": str(next_cursor) if next_cursor is not None else "", + "request_id": rid, + }, + ) + set_request_id_header(resp, rid) + return resp + + +@router.get("/images/{image_id}") +async def get_image( + request: Request, + image_id: int, +) -> Any: + if int(image_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported image_id", status_code=400) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + image = await get_image_by_id(session, image_id=image_id) + if image is None: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Image not found", status_code=404) + tags = await get_tag_names_for_image(session, image_id=image.id) + + rid = get_or_create_request_id(request) + set_request_id_on_state(request, rid) + + resp = JSONResponse( + status_code=200, + content={ + "ok": True, + "item": { + "image": { + "id": str(image.id), + "illust_id": str(image.illust_id), + "page_index": image.page_index, + "ext": image.ext, + "width": image.width, + "height": image.height, + "x_restrict": image.x_restrict, + "ai_type": image.ai_type, + "bookmark_count": getattr(image, "bookmark_count", None), + "view_count": getattr(image, "view_count", None), + "comment_count": getattr(image, "comment_count", None), + "user": { + "id": str(image.user_id) if image.user_id is not None else None, + "name": image.user_name, + }, + "title": image.title, + "created_at_pixiv": image.created_at_pixiv, + }, + "tags": tags, + }, + "request_id": rid, + }, + ) + set_request_id_header(resp, rid) + return resp + + +@router.get("/i/{image_id}.{ext}") +async def proxy_image( + request: Request, + image_id: int, + ext: str, +): + ext = (ext or "").lower() + if ext not in ALLOWED_IMAGE_EXTS: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported ext", status_code=400) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + needs_hydrate = False + should_mark_ok = False + async with Session() as session: + image = await get_image_by_id(session, image_id=image_id) + if image is None or (image.ext or "").lower() != ext: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Image not found", status_code=404) + should_mark_ok = image.last_ok_at is None or image.last_error_code is not None + + missing_basic_fields = ( + getattr(image, "width", None) is None + or getattr(image, "height", None) is None + or getattr(image, "x_restrict", None) is None + or getattr(image, "ai_type", None) is None + or getattr(image, "user_id", None) is None + or not str(getattr(image, "user_name", "") or "").strip() + or not str(getattr(image, "title", "") or "").strip() + or not str(getattr(image, "created_at_pixiv", "") or "").strip() + or getattr(image, "bookmark_count", None) is None + or getattr(image, "view_count", None) is None + or getattr(image, "comment_count", None) is None + ) + if missing_basic_fields: + needs_hydrate = True + else: + tag_row = ( + await session.execute( + sa.select(ImageTag.image_id).where(ImageTag.image_id == int(image.id)).limit(1) + ) + ).scalar_one_or_none() + needs_hydrate = tag_row is None + + runtime = await load_runtime_config(engine) + proxy_uri = None + picked = await select_proxy_uri_for_url( + engine, + request.app.state.settings, + runtime, + url=str(image.original_url), + ) + if picked is not None: + proxy_uri = picked.uri + + transport = getattr(request.app.state, "httpx_transport", None) + now = iso_utc_ms() + try: + resp = await stream_url( + image.original_url, + transport=transport, + proxy=proxy_uri, + cache_control="public, max-age=31536000, immutable", + range_header=request.headers.get("Range"), + ) + if should_mark_ok: + await mark_image_ok(engine, image_id=int(image.id), now=now) + if needs_hydrate: + try: + await enqueue_opportunistic_hydrate_metadata(engine, illust_id=int(image.illust_id), reason="image_proxy") + except Exception: + pass + return resp + except ApiError as exc: + if exc.code in { + ErrorCode.UPSTREAM_STREAM_ERROR, + ErrorCode.UPSTREAM_403, + ErrorCode.UPSTREAM_404, + ErrorCode.UPSTREAM_RATE_LIMIT, + }: + await mark_image_failure( + engine, + image_id=int(image.id), + now=now, + error_code=exc.code.value, + error_message=exc.message, + ) + raise diff --git a/backend/app/api/public/legacy.py b/backend/app/api/public/legacy.py new file mode 100644 index 0000000..4742165 --- /dev/null +++ b/backend/app/api/public/legacy.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from fastapi import APIRouter, Request + +from app.core.errors import ApiError, ErrorCode +from app.core.http_stream import stream_url +from app.core.pixiv_urls import ALLOWED_IMAGE_EXTS +from app.core.proxy_routing import select_proxy_uri_for_url +from app.core.runtime_settings import load_runtime_config +from app.db.images_get_by_illust import get_image_by_illust_page +from app.db.session import create_sessionmaker + +router = APIRouter() + + +@router.get("/{illust_id}-{page}.{ext}") +async def legacy_multi( + request: Request, + illust_id: int, + page: int, + ext: str, +): + if int(illust_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported illust_id", status_code=400) + if int(page) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported page", status_code=400) + + ext = (ext or "").lower() + if ext not in ALLOWED_IMAGE_EXTS: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported ext", status_code=400) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + image = await get_image_by_illust_page(session, illust_id=illust_id, page_index=int(page) - 1) + if image is None or (image.ext or "").lower() != ext: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Image not found", status_code=404) + + runtime = await load_runtime_config(engine) + proxy_uri = None + picked = await select_proxy_uri_for_url( + engine, + request.app.state.settings, + runtime, + url=str(image.original_url), + ) + if picked is not None: + proxy_uri = picked.uri + + transport = getattr(request.app.state, "httpx_transport", None) + return await stream_url( + image.original_url, + transport=transport, + proxy=proxy_uri, + cache_control="public, max-age=31536000, immutable", + range_header=request.headers.get("Range"), + ) + + +@router.get("/{illust_id}.{ext}") +async def legacy_single( + request: Request, + illust_id: int, + ext: str, +): + if int(illust_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported illust_id", status_code=400) + + ext = (ext or "").lower() + if ext not in ALLOWED_IMAGE_EXTS: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported ext", status_code=400) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async with Session() as session: + image = await get_image_by_illust_page(session, illust_id=illust_id, page_index=0) + if image is None or (image.ext or "").lower() != ext: + raise ApiError(code=ErrorCode.NOT_FOUND, message="Image not found", status_code=404) + + runtime = await load_runtime_config(engine) + proxy_uri = None + picked = await select_proxy_uri_for_url( + engine, + request.app.state.settings, + runtime, + url=str(image.original_url), + ) + if picked is not None: + proxy_uri = picked.uri + + transport = getattr(request.app.state, "httpx_transport", None) + return await stream_url( + image.original_url, + transport=transport, + proxy=proxy_uri, + cache_control="public, max-age=31536000, immutable", + range_header=request.headers.get("Range"), + ) diff --git a/backend/app/api/public/random.py b/backend/app/api/public/random.py new file mode 100644 index 0000000..9a3b3ce --- /dev/null +++ b/backend/app/api/public/random.py @@ -0,0 +1,861 @@ +from __future__ import annotations + +import random +import os +import math +from datetime import datetime, timedelta, timezone +from typing import Any + +from fastapi import APIRouter, Query, Request +from fastapi.responses import RedirectResponse + +from app.core.errors import ApiError, ErrorCode +from app.core.http_stream import stream_url +from app.core.imgproxy import build_signed_processing_url, load_imgproxy_config_from_settings +from app.core.proxy_routing import select_proxy_uri_for_url +from app.core.runtime_settings import load_runtime_config +from app.core.time import iso_utc_ms +from app.db.images_mark import mark_image_failure, mark_image_ok +from app.db.tags_get import get_tag_names_for_image +from app.db.random_pick import pick_random_image, pick_random_images +from app.db.session import create_sessionmaker +from app.jobs.enqueue import enqueue_opportunistic_hydrate_metadata + +router = APIRouter() + +_MAX_TAG_FILTERS = 50 + + +def _as_nonneg_int(value: Any) -> int: + if value is None: + return 0 + if isinstance(value, bool): + return 0 + try: + i = int(value) + except Exception: + return 0 + return i if i > 0 else 0 + + +_DEFAULT_SCORE_WEIGHTS: dict[str, float] = { + "bookmark": 4.0, + "view": 0.5, + "comment": 2.0, + "pixels": 1.0, + "bookmark_rate": 3.0, +} + +_DEFAULT_RECOMMENDATION: dict[str, Any] = { + "pick_mode": "weighted", + "temperature": 1.0, + "score_weights": dict(_DEFAULT_SCORE_WEIGHTS), + "multipliers": { + "ai": 1.0, + "non_ai": 1.0, + "unknown_ai": 1.0, + "illust": 1.0, + "manga": 1.0, + "ugoira": 1.0, + "unknown_illust_type": 1.0, + }, +} + + +def _as_float(value: Any, *, default: float) -> float: + if value is None: + return float(default) + if isinstance(value, bool): + return float(default) + try: + return float(value) + except Exception: + return float(default) + + +def _quality_score(image: Any, *, weights: dict[str, float] | None = None) -> float: + w = weights or _DEFAULT_SCORE_WEIGHTS + w_bookmark = float(w.get("bookmark", _DEFAULT_SCORE_WEIGHTS["bookmark"])) + w_view = float(w.get("view", _DEFAULT_SCORE_WEIGHTS["view"])) + w_comment = float(w.get("comment", _DEFAULT_SCORE_WEIGHTS["comment"])) + w_pixels = float(w.get("pixels", _DEFAULT_SCORE_WEIGHTS["pixels"])) + w_bookmark_rate = float(w.get("bookmark_rate", _DEFAULT_SCORE_WEIGHTS["bookmark_rate"])) + + bookmark_count = _as_nonneg_int(getattr(image, "bookmark_count", None)) + view_count = _as_nonneg_int(getattr(image, "view_count", None)) + comment_count = _as_nonneg_int(getattr(image, "comment_count", None)) + + width = _as_nonneg_int(getattr(image, "width", None)) + height = _as_nonneg_int(getattr(image, "height", None)) + pixels = width * height if width > 0 and height > 0 else 0 + + rate_term = 0.0 + if view_count > 0: + bookmark_rate_per_mille = (float(bookmark_count) / float(view_count)) * 1000.0 + rate_term = math.log1p(max(0.0, bookmark_rate_per_mille)) + + score = ( + float(w_bookmark) * math.log1p(bookmark_count) + + float(w_view) * math.log1p(view_count) + + float(w_comment) * math.log1p(comment_count) + + float(w_pixels) * math.log1p(float(pixels) / 1_000_000.0) + + float(w_bookmark_rate) * rate_term + ) + return float(score) + + +@router.get("/random") +async def random_image( + request: Request, + format: str = "image", + redirect: int = 0, + attempts: int | None = None, + seed: str | None = None, + strategy: str | None = None, + quality_samples: int | None = None, + r18: int = 0, + r18_strict: int | None = None, + ai_type: str = "any", + illust_type: str = "any", + orientation: str = "any", + layout: str | None = None, + adaptive: int = 0, + min_width: int = 0, + min_height: int = 0, + min_pixels: int = 0, + min_bookmarks: int = 0, + min_views: int = 0, + min_comments: int = 0, + included_tags: list[str] | None = Query(default=None), + excluded_tags: list[str] | None = Query(default=None), + user_id: int | None = None, + illust_id: int | None = None, + created_from: str | None = None, + created_to: str | None = None, +) -> Any: + if format not in {"image", "json", "simple_json"}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported format", status_code=400) + if redirect not in {0, 1}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported redirect", status_code=400) + seed_norm = (seed or "").strip() + if seed is not None and not seed_norm: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported seed", status_code=400) + if seed_norm and len(seed_norm) > 128: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported seed", status_code=400) + + ai_type_raw = (ai_type or "any").strip().lower() + ai_type_i: int | None = None + if ai_type_raw in {"", "any"}: + ai_type_i = None + elif ai_type_raw in {"0", "1"}: + ai_type_i = int(ai_type_raw) + else: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported ai_type", status_code=400) + + illust_type_raw = (illust_type or "any").strip().lower() + illust_type_i: int | None = None + if illust_type_raw in {"", "any"}: + illust_type_i = None + elif illust_type_raw in {"0", "illust", "illustration"}: + illust_type_i = 0 + elif illust_type_raw in {"1", "manga"}: + illust_type_i = 1 + elif illust_type_raw in {"2", "ugoira"}: + illust_type_i = 2 + else: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported illust_type", status_code=400) + + if r18 not in {0, 1, 2}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported r18", status_code=400) + + if adaptive not in {0, 1}: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported adaptive", status_code=400) + + layout_source = "orientation" + raw_layout = orientation + if layout is not None: + layout_source = "layout" + raw_layout = layout + + layout_norm = (raw_layout or "").strip().lower() + alias_map = {"vertical": "portrait", "horizontal": "landscape"} + layout_norm = alias_map.get(layout_norm, layout_norm) + orientation_map = {"any": None, "portrait": 1, "landscape": 2, "square": 3} + if layout_norm not in orientation_map: + raise ApiError( + code=ErrorCode.BAD_REQUEST, + message="Unsupported layout" if layout_source == "layout" else "Unsupported orientation", + status_code=400, + ) + + min_width_i = int(min_width) + min_height_i = int(min_height) + min_pixels_i = int(min_pixels) + min_bookmarks_i = int(min_bookmarks) + min_views_i = int(min_views) + min_comments_i = int(min_comments) + if min_width_i < 0 or min_height_i < 0 or min_pixels_i < 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported min_*", status_code=400) + if min_bookmarks_i < 0 or min_views_i < 0 or min_comments_i < 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported min_*", status_code=400) + + # 自适应：在用户没有显式指定的情况下，根据设备类型设置默认的方向/分辨率门槛。 + # 注意：不会覆盖用户显式传入的 orientation/layout/min_*。 + if int(adaptive) == 1: + qp = request.query_params + orientation_explicit = ("layout" in qp) or ("orientation" in qp) + min_explicit = ("min_width" in qp) or ("min_height" in qp) or ("min_pixels" in qp) + + ch_mobile = (request.headers.get("sec-ch-ua-mobile") or request.headers.get("Sec-CH-UA-Mobile") or "").strip() + if ch_mobile == "?1": + is_mobile = True + elif ch_mobile == "?0": + is_mobile = False + else: + ua = (request.headers.get("user-agent") or request.headers.get("User-Agent") or "").lower() + is_mobile = any(x in ua for x in ("mobi", "android", "iphone", "ipad", "ipod")) + + if not orientation_explicit and layout_norm == "any": + layout_norm = "portrait" if is_mobile else "landscape" + + if not min_explicit and min_width_i == 0 and min_height_i == 0 and min_pixels_i == 0: + min_pixels_i = 1_000_000 if is_mobile else 2_000_000 + + def _parse_tags(values: list[str] | None) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for raw in values or []: + for part in str(raw).split("|"): + name = part.strip() + if not name or name in seen: + continue + seen.add(name) + out.append(name) + return out + + included = _parse_tags(included_tags) + excluded = _parse_tags(excluded_tags) + if len(included) > _MAX_TAG_FILTERS or len(excluded) > _MAX_TAG_FILTERS: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Too many tag filters", status_code=400) + if user_id is not None and int(user_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported user_id", status_code=400) + if illust_id is not None and int(illust_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported illust_id", status_code=400) + + def _normalize_iso_utc(value: str) -> str: + raw = (value or "").strip() + if not raw: + raise ValueError("empty datetime") + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + dt = datetime.fromisoformat(raw) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + dt = dt.astimezone(timezone.utc).replace(microsecond=0) + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + created_from_norm: str | None = None + created_to_norm: str | None = None + try: + if created_from is not None: + created_from_norm = _normalize_iso_utc(created_from) + if created_to is not None: + created_to_norm = _normalize_iso_utc(created_to) + except Exception: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported created_*", status_code=400) + + if created_from_norm is not None and created_to_norm is not None: + if created_from_norm > created_to_norm: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="created_from > created_to", status_code=400) + + def _no_match_error() -> ApiError: + applied_filters: dict[str, Any] = { + "r18": r18, + "r18_strict": int(r18_strict), + "ai_type": ai_type_raw, + "illust_type": illust_type_raw, + "adaptive": int(adaptive), + "orientation": layout_norm, + "min_width": int(min_width_i), + "min_height": int(min_height_i), + "min_pixels": int(min_pixels_i), + "min_bookmarks": int(min_bookmarks_i), + "min_views": int(min_views_i), + "min_comments": int(min_comments_i), + } + if included: + applied_filters["included_tags"] = included + if excluded: + applied_filters["excluded_tags"] = excluded + if user_id is not None: + applied_filters["user_id"] = int(user_id) + if illust_id is not None: + applied_filters["illust_id"] = int(illust_id) + if created_from_norm is not None: + applied_filters["created_from"] = created_from_norm + if created_to_norm is not None: + applied_filters["created_to"] = created_to_norm + + suggestions: list[str] = ["运行元数据补全任务以提升元数据覆盖率"] + if r18 == 0 and int(r18_strict) == 1: + suggestions.append("将 r18_strict=0 以允许未知 x_restrict（冷启动阶段更容易命中）") + if layout_norm != "any": + suggestions.append("将 orientation=any（取消方向限制）") + if int(min_width_i) > 0 or int(min_height_i) > 0 or int(min_pixels_i) > 0: + suggestions.append("降低 min_width/min_height/min_pixels（放宽分辨率门槛）") + if int(min_bookmarks_i) > 0 or int(min_views_i) > 0 or int(min_comments_i) > 0: + suggestions.append("降低 min_bookmarks/min_views/min_comments（放宽热度门槛）") + if included: + suggestions.append("放宽 included_tags（减少必须包含的标签）") + if excluded: + suggestions.append("放宽 excluded_tags（减少必须排除的标签）") + if user_id is not None: + suggestions.append("移除 user_id 过滤") + if illust_id is not None: + suggestions.append("移除 illust_id 过滤") + if ai_type_i is not None: + suggestions.append("将 ai_type=any（取消 AI 限制）") + if illust_type_i is not None: + suggestions.append("将 illust_type=any（取消作品类型限制）") + if created_from_norm is not None or created_to_norm is not None: + suggestions.append("扩大 created_from/created_to 时间范围") + if int(adaptive) == 1: + suggestions.append("若自适应导致过滤过严，可尝试 adaptive=0 或显式设置 min_*") + + return ApiError( + code=ErrorCode.NO_MATCH, + message="没有匹配的图片。", + status_code=404, + details={"hints": {"applied_filters": applied_filters, "suggestions": suggestions}}, + ) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + runtime = await load_runtime_config(engine) + + random_defaults = runtime.random_defaults if isinstance(runtime.random_defaults, dict) else {} + + attempts_source = "query" + attempts_i = 3 + if attempts is not None: + try: + attempts_i = int(attempts) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported attempts", status_code=400) from exc + else: + raw = random_defaults.get("default_attempts") + if raw is None: + attempts_source = "fallback" + attempts_i = 3 + else: + attempts_source = "runtime" + try: + attempts_i = int(raw) + except Exception: + attempts_i = 3 + if attempts_i < 1 or attempts_i > 10: + if attempts_source == "query": + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported attempts", status_code=400) + attempts_source = "fallback" + attempts_i = 3 + attempts = int(attempts_i) + + r18_strict_source = "query" + r18_strict_i = 1 + if r18_strict is not None: + try: + r18_strict_i = int(r18_strict) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported r18_strict", status_code=400) from exc + else: + raw = random_defaults.get("default_r18_strict") + if raw is None: + r18_strict_source = "fallback" + r18_strict_i = 1 + elif isinstance(raw, bool): + r18_strict_source = "runtime" + r18_strict_i = 1 if raw else 0 + else: + r18_strict_source = "runtime" + try: + r18_strict_i = int(raw) + except Exception: + r18_strict_i = 1 + if r18_strict_i not in {0, 1}: + if r18_strict_source == "query": + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported r18_strict", status_code=400) + r18_strict_source = "fallback" + r18_strict_i = 1 + r18_strict = int(r18_strict_i) + + fail_cooldown_source = "runtime" + fail_cooldown_ms = random_defaults.get("fail_cooldown_ms") + try: + fail_cooldown_ms_i = int(fail_cooldown_ms) if fail_cooldown_ms is not None else None + except Exception: + fail_cooldown_ms_i = None + + if fail_cooldown_ms_i is None: + fail_cooldown_source = "fallback" + cooldown_s_raw = (os.environ.get("RANDOM_FAIL_COOLDOWN_SECONDS") or "600").strip() + try: + cooldown_s = int(cooldown_s_raw) + except Exception: + cooldown_s = 600 + cooldown_s = max(0, min(int(cooldown_s), 24 * 60 * 60)) + fail_cooldown_ms_i = int(cooldown_s) * 1000 + fail_cooldown_ms_i = max(0, min(int(fail_cooldown_ms_i), 24 * 60 * 60 * 1000)) + + request_now = datetime.now(timezone.utc) + fail_cooldown_before = ( + iso_utc_ms(request_now - timedelta(milliseconds=int(fail_cooldown_ms_i))) + if int(fail_cooldown_ms_i) > 0 + else None + ) + + pick_kwargs: dict[str, Any] = { + "r18": r18, + "r18_strict": bool(r18_strict), + "ai_type": ai_type_i, + "illust_type": illust_type_i, + "orientation": orientation_map[layout_norm], + "min_width": int(min_width_i), + "min_height": int(min_height_i), + "min_pixels": int(min_pixels_i), + "min_bookmarks": int(min_bookmarks_i), + "min_views": int(min_views_i), + "min_comments": int(min_comments_i), + "included_tags": included, + "excluded_tags": excluded, + "user_id": user_id, + "illust_id": illust_id, + "created_from": created_from_norm, + "created_to": created_to_norm, + "fail_cooldown_before": fail_cooldown_before, + } + + rng = random.Random(seed_norm) if seed_norm else random + + strategy_raw = (strategy or "").strip().lower() + strategy_source = "query" + if not strategy_raw: + strategy_source = "runtime" + strategy_raw = str(random_defaults.get("strategy") or "").strip().lower() + if not strategy_raw: + strategy_source = "fallback" + strategy_raw = "quality" + if strategy_raw not in {"quality", "random"}: + if strategy_source == "query": + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported strategy", status_code=400) + strategy_source = "fallback" + strategy_raw = "quality" + + strategy_norm = strategy_raw + + quality_samples_i: int + quality_samples_source = "query" + if quality_samples is not None: + try: + quality_samples_i = int(quality_samples) + except Exception as exc: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported quality_samples", status_code=400) from exc + else: + raw = random_defaults.get("quality_samples") + if raw is None: + quality_samples_source = "fallback" + quality_samples_i = 5 + else: + quality_samples_source = "runtime" + try: + quality_samples_i = int(raw) + except Exception: + quality_samples_i = 5 + if quality_samples_i < 1 or quality_samples_i > 1000: + if quality_samples_source == "query": + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported quality_samples", status_code=400) + quality_samples_source = "fallback" + quality_samples_i = 5 + + recommendation_raw = random_defaults.get("recommendation") + recommendation_source = "fallback" + recommendation_obj: dict[str, Any] = {} + if isinstance(recommendation_raw, dict): + recommendation_source = "runtime" + recommendation_obj = dict(recommendation_raw) + + pick_mode_raw = str(recommendation_obj.get("pick_mode") or _DEFAULT_RECOMMENDATION["pick_mode"]).strip().lower() + if pick_mode_raw not in {"best", "weighted"}: + pick_mode_raw = str(_DEFAULT_RECOMMENDATION["pick_mode"]) + + temperature_raw = _as_float(recommendation_obj.get("temperature"), default=float(_DEFAULT_RECOMMENDATION["temperature"])) + temperature = float(max(0.05, min(float(temperature_raw), 100.0))) + + score_weights_raw = recommendation_obj.get("score_weights") + score_weights_obj = score_weights_raw if isinstance(score_weights_raw, dict) else {} + score_weights: dict[str, float] = {} + for key, default_value in _DEFAULT_SCORE_WEIGHTS.items(): + v = _as_float(score_weights_obj.get(key), default=float(default_value)) + score_weights[key] = float(max(-100.0, min(float(v), 100.0))) + + multipliers_default = _DEFAULT_RECOMMENDATION["multipliers"] + multipliers_raw = recommendation_obj.get("multipliers") + multipliers_obj = multipliers_raw if isinstance(multipliers_raw, dict) else {} + multipliers: dict[str, float] = {} + for key, default_value in multipliers_default.items(): + v = _as_float(multipliers_obj.get(key), default=float(default_value)) + multipliers[key] = float(max(0.0, min(float(v), 100.0))) + + debug_base = { + "attempts": int(attempts_i), + "attempts_source": attempts_source, + "r18_strict": int(r18_strict_i), + "r18_strict_source": r18_strict_source, + "fail_cooldown_ms": int(fail_cooldown_ms_i), + "fail_cooldown_source": fail_cooldown_source, + "strategy": strategy_norm, + "strategy_source": strategy_source, + "quality_samples": int(quality_samples_i), + "quality_samples_source": quality_samples_source, + "recommendation_source": recommendation_source, + } + + async def _pick_with_strategy( + *, + session: Any, + exclude_image_ids: list[int] | None = None, + ) -> tuple[Any, dict[str, Any]] | tuple[None, dict[str, Any]]: + if strategy_norm == "random": + image = await pick_random_image(session, r=rng.random(), exclude_image_ids=exclude_image_ids, **pick_kwargs) + if image is None: + return None, {**debug_base, "attempts_used": 1, "picked_by": "random_key"} + return image, {**debug_base, "attempts_used": 1, "picked_by": "random_key"} + + def _multiplier_for_image(image: Any) -> float: + m = 1.0 + + ai = getattr(image, "ai_type", None) + if ai == 1: + m *= float(multipliers.get("ai", 1.0)) + elif ai == 0: + m *= float(multipliers.get("non_ai", 1.0)) + else: + m *= float(multipliers.get("unknown_ai", 1.0)) + + it = getattr(image, "illust_type", None) + if it == 0: + m *= float(multipliers.get("illust", 1.0)) + elif it == 1: + m *= float(multipliers.get("manga", 1.0)) + elif it == 2: + m *= float(multipliers.get("ugoira", 1.0)) + else: + m *= float(multipliers.get("unknown_illust_type", 1.0)) + + if not math.isfinite(float(m)) or float(m) <= 0.0: + return 0.0 + return float(m) + + exclude_set: set[int] = set(int(x) for x in exclude_image_ids or []) + candidates: list[tuple[Any, float, float, float]] = [] + + # 批量抽样：一次性取 N 个候选（必要时 wrap-around 再取一次），避免 N 次 DB 循环查询。 + # 若用户把某些类别倍率设为 0（例如 manga=0），直接在 SQL 抽样阶段剔除，减少无效候选。 + ai_allowed: set[int | None] = set() + if float(multipliers.get("ai", 1.0)) > 0.0: + ai_allowed.add(1) + if float(multipliers.get("non_ai", 1.0)) > 0.0: + ai_allowed.add(0) + if float(multipliers.get("unknown_ai", 1.0)) > 0.0: + ai_allowed.add(None) + + illust_allowed: set[int | None] = set() + if float(multipliers.get("illust", 1.0)) > 0.0: + illust_allowed.add(0) + if float(multipliers.get("manga", 1.0)) > 0.0: + illust_allowed.add(1) + if float(multipliers.get("ugoira", 1.0)) > 0.0: + illust_allowed.add(2) + if float(multipliers.get("unknown_illust_type", 1.0)) > 0.0: + illust_allowed.add(None) + + if not ai_allowed or not illust_allowed: + return None, { + **debug_base, + "attempts_used": 1, + "picked_by": "quality_weighted" if pick_mode_raw == "weighted" else "quality_best", + "candidates_drawn": 0, + "candidates_accepted": 0, + "quality_pick_mode": pick_mode_raw, + "quality_temperature": float(temperature), + } + + images = await pick_random_images( + session, + r=rng.random(), + limit=int(quality_samples_i), + exclude_image_ids=list(exclude_set), + ai_type_allowed=ai_allowed, + illust_type_allowed=illust_allowed, + **pick_kwargs, + ) + + drawn = int(len(images)) + accepted = 0 + for image in images: + score = _quality_score(image, weights=score_weights) + multiplier = _multiplier_for_image(image) + if multiplier <= 0.0: + continue + + logit = float(score) / float(temperature) + math.log(float(multiplier)) + candidates.append((image, float(score), float(multiplier), float(logit))) + accepted += 1 + + if not candidates: + return None, { + **debug_base, + "attempts_used": 1, + "picked_by": "quality_weighted" if pick_mode_raw == "weighted" else "quality_best", + "candidates_drawn": int(drawn), + "candidates_accepted": int(accepted), + "quality_pick_mode": pick_mode_raw, + "quality_temperature": float(temperature), + } + + if pick_mode_raw == "best": + picked = max(candidates, key=lambda x: x[3]) + picked_by = "quality_best" + else: + max_logit = max(x[3] for x in candidates) + weights = [math.exp(float(x[3]) - float(max_logit)) for x in candidates] + total = float(sum(weights)) + if not math.isfinite(total) or total <= 0.0: + picked = max(candidates, key=lambda x: x[3]) + picked_by = "quality_best" + else: + r = float(rng.random()) * total + idx = 0 + for i, w in enumerate(weights): + r -= float(w) + if r <= 0: + idx = i + break + picked = candidates[int(max(0, min(idx, len(candidates) - 1)))] + picked_by = "quality_weighted" + + best_image, best_score, best_multiplier, _best_logit = picked + + return ( + best_image, + { + **debug_base, + "attempts_used": 1, + "picked_by": picked_by, + "candidates_drawn": int(drawn), + "candidates_accepted": int(accepted), + "quality_pick_mode": pick_mode_raw, + "quality_temperature": float(temperature), + "quality_score": float(best_score), + "quality_multiplier": float(best_multiplier), + }, + ) + + def _needs_opportunistic_hydrate(image: Any) -> bool: + return ( + getattr(image, "width", None) is None + or getattr(image, "height", None) is None + or getattr(image, "x_restrict", None) is None + or getattr(image, "ai_type", None) is None + or getattr(image, "illust_type", None) is None + or getattr(image, "user_id", None) is None + or getattr(image, "bookmark_count", None) is None + or getattr(image, "view_count", None) is None + or getattr(image, "comment_count", None) is None + ) + + if format in {"json", "simple_json"} or (format == "image" and redirect == 1): + tags: list[str] = [] + debug: dict[str, Any] = {} + async with Session() as session: + image, debug = await _pick_with_strategy(session=session) + if image is None: + raise _no_match_error() + if format == "json": + tags = await get_tag_names_for_image(session, image_id=image.id) + + if _needs_opportunistic_hydrate(image): + try: + await enqueue_opportunistic_hydrate_metadata(engine, illust_id=int(image.illust_id), reason="random") + except Exception: + pass + + if format == "image" and redirect == 1: + return RedirectResponse( + url=f"/i/{image.id}.{image.ext}", + status_code=302, + headers={"Cache-Control": "no-store"}, + ) + + origin_url = None if runtime.hide_origin_url_in_public_json else image.original_url + + imgproxy_url = None + try: + cfg = load_imgproxy_config_from_settings(request.app.state.settings) + except Exception: + cfg = None + if cfg is not None: + try: + if runtime.hide_origin_url_in_public_json: + base = str(getattr(request, "base_url", "") or "").rstrip("/") + source_url = f"{base}/i/{image.id}.{image.ext}" + else: + source_url = str(image.original_url) + imgproxy_url = build_signed_processing_url(cfg, source_url=source_url, extension=str(image.ext)) + except Exception: + imgproxy_url = None + + if format == "simple_json": + return { + "ok": True, + "code": "OK", + "request_id": getattr(getattr(request, "state", None), "request_id", None) or "req_unknown", + "data": { + "image": { + "id": str(image.id), + "illust_id": str(image.illust_id), + "page_index": image.page_index, + "ext": image.ext, + "width": image.width, + "height": image.height, + "x_restrict": image.x_restrict, + "ai_type": image.ai_type, + "illust_type": getattr(image, "illust_type", None), + "bookmark_count": getattr(image, "bookmark_count", None), + "view_count": getattr(image, "view_count", None), + "comment_count": getattr(image, "comment_count", None), + }, + "urls": { + "proxy": f"/i/{image.id}.{image.ext}", + "origin": origin_url, + "imgproxy": imgproxy_url, + }, + "debug": { + **debug, + }, + }, + } + + return { + "ok": True, + "code": "OK", + "request_id": getattr(getattr(request, "state", None), "request_id", None) or "req_unknown", + "data": { + "image": { + "id": str(image.id), + "illust_id": str(image.illust_id), + "page_index": image.page_index, + "ext": image.ext, + "width": image.width, + "height": image.height, + "x_restrict": image.x_restrict, + "ai_type": image.ai_type, + "illust_type": getattr(image, "illust_type", None), + "bookmark_count": getattr(image, "bookmark_count", None), + "view_count": getattr(image, "view_count", None), + "comment_count": getattr(image, "comment_count", None), + "user": { + "id": str(image.user_id) if image.user_id is not None else None, + "name": image.user_name, + }, + "title": image.title, + "created_at_pixiv": image.created_at_pixiv, + }, + "tags": tags, + "urls": { + "proxy": f"/i/{image.id}.{image.ext}", + "origin": origin_url, + "imgproxy": imgproxy_url, + "legacy_single": f"/{image.illust_id}.{image.ext}", + "legacy_multi": f"/{image.illust_id}-{image.page_index + 1}.{image.ext}", + }, + "debug": { + **debug, + }, + }, + } + + tried_ids: set[int] = set() + last_error: ApiError | None = None + attempts_i = int(attempts) + runtime_stream = runtime + + for _ in range(attempts_i): + async with Session() as session: + image, _debug = await _pick_with_strategy(session=session, exclude_image_ids=list(tried_ids)) + if image is None: + break + image_id = int(image.id) + origin_url = str(image.original_url) + illust_id_for_hydrate = int(image.illust_id) + needs_hydrate = _needs_opportunistic_hydrate(image) + + transport = getattr(request.app.state, "httpx_transport", None) + proxy_uri = None + picked = await select_proxy_uri_for_url( + engine, + request.app.state.settings, + runtime_stream, + url=origin_url, + ) + if picked is not None: + proxy_uri = picked.uri + try: + resp = await stream_url( + origin_url, + transport=transport, + proxy=proxy_uri, + cache_control="no-store", + range_header=request.headers.get("Range"), + ) + await mark_image_ok(engine, image_id=image_id, now=iso_utc_ms()) + if needs_hydrate: + try: + await enqueue_opportunistic_hydrate_metadata( + engine, + illust_id=illust_id_for_hydrate, + reason="random", + ) + except Exception: + pass + return resp + except ApiError as exc: + if exc.code in { + ErrorCode.UPSTREAM_STREAM_ERROR, + ErrorCode.UPSTREAM_403, + ErrorCode.UPSTREAM_404, + ErrorCode.UPSTREAM_RATE_LIMIT, + }: + await mark_image_failure( + engine, + image_id=image_id, + now=iso_utc_ms(), + error_code=exc.code.value, + error_message=exc.message, + ) + tried_ids.add(image_id) + last_error = exc + continue + raise + + if last_error is None: + raise _no_match_error() + + raise ApiError( + code=ErrorCode.UPSTREAM_STREAM_ERROR, + message="多次尝试后上游请求仍失败。", + status_code=502, + details={ + "attempts_used": len(tried_ids), + "last_upstream_code": last_error.code.value, + }, + ) diff --git a/backend/app/api/public/tags.py b/backend/app/api/public/tags.py new file mode 100644 index 0000000..1ee303d --- /dev/null +++ b/backend/app/api/public/tags.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +from app.core.errors import ApiError, ErrorCode +from app.core.request_id import get_or_create_request_id +from app.db.session import create_sessionmaker +from app.db.tags_list import list_tags as db_list_tags + +router = APIRouter() + + +@router.get("/tags") +async def list_tags( + request: Request, + q: str | None = None, + limit: int = 50, + cursor: str | None = None, +) -> Any: + if limit < 1 or limit > 100: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported limit", status_code=400) + + q_norm = (q or "").strip() + if q_norm and len(q_norm) > 200: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Unsupported q", status_code=400) + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + async with Session() as session: + items, next_cursor = await db_list_tags(session, limit=limit, cursor=cursor, q=q_norm or None) + + rid = get_or_create_request_id(request) + resp = JSONResponse( + status_code=200, + content={ + "ok": True, + "items": [ + { + "id": str(item.id), + "name": item.name, + "translated_name": item.translated_name, + "count_images": item.count_images, + } + for item in items + ], + "next_cursor": next_cursor or "", + "request_id": rid, + }, + ) + return resp + diff --git a/backend/app/api/public/version.py b/backend/app/api/public/version.py new file mode 100644 index 0000000..d838d89 --- /dev/null +++ b/backend/app/api/public/version.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import os +from typing import Any + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +from app.core.request_id import get_or_create_request_id, set_request_id_header, set_request_id_on_state + +router = APIRouter() + + +def _get_env(key: str, default: str) -> str: + value = os.environ.get(key, default) + return str(value).strip() + + +@router.get("/version") +async def version(request: Request) -> Any: + rid = get_or_create_request_id(request) + set_request_id_on_state(request, rid) + + body = { + "ok": True, + "version": _get_env("APP_VERSION", "dev"), + "build_time": _get_env("APP_BUILD_TIME", ""), + "git_commit": _get_env("APP_COMMIT", ""), + "request_id": rid, + } + + resp = JSONResponse(status_code=200, content=body) + set_request_id_header(resp, rid) + return resp + diff --git a/backend/app/core/api_keys.py b/backend/app/core/api_keys.py new file mode 100644 index 0000000..cc7aff8 --- /dev/null +++ b/backend/app/core/api_keys.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import hashlib +import hmac +import time +from dataclasses import dataclass, field +from collections.abc import Mapping + +import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.errors import ApiError, ErrorCode +from app.db.models.api_keys import ApiKey +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + + +def _coerce_int(value: int, *, default: int = 0) -> int: + try: + return int(value) + except Exception: + return default + + +def _normalize_api_key(value: str) -> str: + v = (value or "").strip() + return v + + +def hmac_sha256_hex(*, secret_key: str, message: str) -> str: + secret_key = (secret_key or "").strip() + if not secret_key: + raise ValueError("SECRET_KEY is required") + mac = hmac.new(secret_key.encode("utf-8"), message.encode("utf-8"), digestmod=hashlib.sha256) + return mac.hexdigest() + + +def api_key_hint(api_key: str) -> str: + api_key = _normalize_api_key(api_key) + if not api_key: + return "" + digest = hashlib.sha256(api_key.encode("utf-8")).hexdigest() + return digest[:8] + + +@dataclass(frozen=True, slots=True) +class ApiKeyAuthConfig: + required: bool + rpm: int + burst: int + secret_key: str + + +@dataclass(slots=True) +class _CacheEntry: + api_key_id: int + enabled: bool + expires_at_m: float + + +@dataclass(slots=True) +class ApiKeyAuthenticator: + engine: AsyncEngine + cfg: ApiKeyAuthConfig + cache_ttl_s: float = 5.0 + _cache: dict[str, _CacheEntry] = field(default_factory=dict) + + async def _lookup(self, key_hash: str, *, now_m: float) -> _CacheEntry | None: + Session = create_sessionmaker(self.engine) + + async def _op() -> _CacheEntry | None: + async with Session() as session: + row = ( + ( + await session.execute( + sa.select(ApiKey.id, ApiKey.enabled).where(ApiKey.key_hash == key_hash).limit(1) + ) + ) + .first() + ) + if row is None: + return None + return _CacheEntry( + api_key_id=int(row[0]), + enabled=bool(int(row[1] or 0)), + expires_at_m=float(now_m) + float(self.cache_ttl_s), + ) + + return await with_sqlite_busy_retry(_op) + + async def authenticate(self, api_key: str) -> int | None: + api_key = _normalize_api_key(api_key) + if not api_key: + return None + + key_hash = hmac_sha256_hex(secret_key=self.cfg.secret_key, message=api_key) + now_m = time.monotonic() + + cached = self._cache.get(key_hash) + if cached is not None and float(cached.expires_at_m) > float(now_m): + return int(cached.api_key_id) if bool(cached.enabled) else None + + entry = await self._lookup(key_hash, now_m=now_m) + if entry is None: + self._cache[key_hash] = _CacheEntry(api_key_id=0, enabled=False, expires_at_m=float(now_m) + 2.0) + return None + + self._cache[key_hash] = entry + + if len(self._cache) > 10_000: + self._cache = {k: v for k, v in self._cache.items() if float(v.expires_at_m) > float(now_m)} + + return int(entry.api_key_id) if bool(entry.enabled) else None + + +@dataclass(slots=True) +class _Bucket: + tokens: float + updated_at_m: float + + +@dataclass(slots=True) +class ApiKeyRateLimiter: + rpm: int + burst: int + _buckets: dict[int, _Bucket] = field(default_factory=dict) + + def _params(self) -> tuple[float, float]: + rpm = max(0, _coerce_int(self.rpm)) + if rpm <= 0: + return 0.0, 0.0 + capacity = max(1.0, float(_coerce_int(self.burst)) or float(rpm)) + refill_per_s = float(rpm) / 60.0 + return capacity, refill_per_s + + def allow(self, api_key_id: int) -> bool: + api_key_id_i = _coerce_int(api_key_id) + if api_key_id_i <= 0: + return False + + capacity, refill_per_s = self._params() + if capacity <= 0.0 or refill_per_s <= 0.0: + return True + + now_m = time.monotonic() + b = self._buckets.get(api_key_id_i) + if b is None: + self._buckets[api_key_id_i] = _Bucket(tokens=float(capacity) - 1.0, updated_at_m=now_m) + return True + + elapsed = max(0.0, float(now_m) - float(b.updated_at_m)) + b.tokens = min(float(capacity), float(b.tokens) + elapsed * float(refill_per_s)) + b.updated_at_m = now_m + if float(b.tokens) >= 1.0: + b.tokens -= 1.0 + return True + return False + + +def extract_api_key(headers: Mapping[str, str] | None) -> str | None: + if not headers: + return None + return (headers.get("X-API-Key") or headers.get("x-api-key") or "").strip() or None + + +async def require_public_api_key( + authenticator: ApiKeyAuthenticator, + limiter: ApiKeyRateLimiter, + *, + headers: Mapping[str, str] | None, +) -> int: + api_key = extract_api_key(headers) + if not api_key: + raise ApiError(code=ErrorCode.UNAUTHORIZED, message="Missing API key", status_code=401) + + api_key_id = await authenticator.authenticate(api_key) + if api_key_id is None: + raise ApiError(code=ErrorCode.UNAUTHORIZED, message="Invalid API key", status_code=401) + + if not limiter.allow(api_key_id): + raise ApiError(code=ErrorCode.RATE_LIMITED, message="Rate limited", status_code=429) + + return int(api_key_id) diff --git a/backend/app/core/bindings_recompute.py b/backend/app/core/bindings_recompute.py new file mode 100644 index 0000000..229a440 --- /dev/null +++ b/backend/app/core/bindings_recompute.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +from typing import Any + +import sqlalchemy as sa +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.errors import ApiError, ErrorCode +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.token_proxy_bindings import TokenProxyBinding + + +def _fnv1a64(text: str) -> int: + h = 14695981039346656037 + prime = 1099511628211 + for b in text.encode("utf-8"): + h ^= b + h = (h * prime) & 0xFFFFFFFFFFFFFFFF + return h + + +def _rendezvous_proxy_order(*, token_id: int, proxy_ids: list[int], salt: str) -> list[int]: + scored = [(_fnv1a64(f"{token_id}|{pid}|{salt}"), pid) for pid in proxy_ids] + scored.sort(key=lambda x: (-x[0], x[1])) + return [pid for _, pid in scored] + + +def _compute_primary_assignments( + *, + token_ids: list[int], + proxy_ids: list[int], + capacity_by_proxy_id: dict[int, int], + salt: str, +) -> dict[int, int]: + remaining = {pid: int(capacity_by_proxy_id.get(int(pid), 0)) for pid in proxy_ids} + out: dict[int, int] = {} + for token_id in token_ids: + for pid in _rendezvous_proxy_order(token_id=token_id, proxy_ids=proxy_ids, salt=salt): + if remaining.get(pid, 0) > 0: + out[token_id] = pid + remaining[pid] -= 1 + break + return out + + +def _compute_primary_assignments_soft( + *, + token_ids: list[int], + proxy_ids: list[int], + capacity_by_proxy_id: dict[int, int], + salt: str, +) -> tuple[dict[int, int], int]: + out = _compute_primary_assignments( + token_ids=token_ids, + proxy_ids=proxy_ids, + capacity_by_proxy_id=capacity_by_proxy_id, + salt=salt, + ) + + over_capacity = 0 + for token_id in token_ids: + if token_id in out: + continue + order = _rendezvous_proxy_order(token_id=token_id, proxy_ids=proxy_ids, salt=salt) + if not order: + continue + out[token_id] = int(order[0]) + over_capacity += 1 + + return out, int(over_capacity) + + +async def recompute_token_proxy_bindings( + session: AsyncSession, + *, + pool_id: int, + now: str, + max_tokens_per_proxy: int, + strict: bool, +) -> dict[str, Any]: + if int(pool_id) <= 0: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid pool_id", status_code=400) + if int(max_tokens_per_proxy) <= 0 or int(max_tokens_per_proxy) > 1000: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="Invalid max_tokens_per_proxy", status_code=400) + + proxy_rows = ( + ( + await session.execute( + sa.select(ProxyPoolEndpoint.endpoint_id, ProxyPoolEndpoint.weight) + .join(ProxyEndpoint, ProxyEndpoint.id == ProxyPoolEndpoint.endpoint_id) + .where(ProxyPoolEndpoint.pool_id == int(pool_id)) + .where(ProxyPoolEndpoint.enabled == 1) + .where(ProxyEndpoint.enabled == 1) + .order_by(ProxyPoolEndpoint.endpoint_id.asc()) + ) + ) + .all() + ) + + proxies: list[tuple[int, int]] = [] + for endpoint_id, weight in proxy_rows: + try: + eid = int(endpoint_id) + except Exception: + continue + try: + w = int(weight or 0) + except Exception: + w = 0 + if eid <= 0: + continue + proxies.append((eid, w)) + + capacity_by_proxy_id = {pid: int(max_tokens_per_proxy) * max(0, int(w)) for pid, w in proxies} + proxy_ids_norm = [pid for pid, _w in proxies if capacity_by_proxy_id.get(pid, 0) > 0] + weight_sum = sum(max(0, int(w)) for pid, w in proxies if int(pid) in set(proxy_ids_norm)) + + if not proxy_ids_norm: + raise ApiError(code=ErrorCode.BAD_REQUEST, message="No enabled proxies in pool", status_code=400) + + token_ids = ( + ( + await session.execute( + sa.select(PixivToken.id) + .where(PixivToken.enabled == 1) + .where(PixivToken.weight > 0) + .order_by(PixivToken.id.asc()) + ) + ) + .scalars() + .all() + ) + if not token_ids: + return {"recomputed": 0} + + capacity = sum(int(capacity_by_proxy_id.get(int(pid), 0)) for pid in proxy_ids_norm) + if strict and len(token_ids) > capacity: + raise ApiError( + code=ErrorCode.BAD_REQUEST, + message="代理容量不足（请增加节点或调高单代理最多绑定令牌数）", + status_code=400, + details={ + "token_count": len(token_ids), + "proxy_count": len(proxy_ids_norm), + "max_tokens_per_proxy": int(max_tokens_per_proxy), + "weight_sum": int(weight_sum), + "capacity": int(capacity), + }, + ) + + salt = f"pool:{pool_id}" + over_capacity_assigned = 0 + if strict: + assignments = _compute_primary_assignments( + token_ids=token_ids, + proxy_ids=proxy_ids_norm, + capacity_by_proxy_id=capacity_by_proxy_id, + salt=salt, + ) + else: + assignments, over_capacity_assigned = _compute_primary_assignments_soft( + token_ids=token_ids, + proxy_ids=proxy_ids_norm, + capacity_by_proxy_id=capacity_by_proxy_id, + salt=salt, + ) + + for token_id in token_ids: + primary_proxy_id = assignments.get(int(token_id)) + if primary_proxy_id is None: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="Binding recompute failed", status_code=500) + + stmt = sqlite_insert(TokenProxyBinding).values( + token_id=int(token_id), + pool_id=int(pool_id), + primary_proxy_id=int(primary_proxy_id), + override_proxy_id=None, + override_expires_at=None, + updated_at=now, + ) + stmt = stmt.on_conflict_do_update( + index_elements=[TokenProxyBinding.token_id, TokenProxyBinding.pool_id], + set_={"primary_proxy_id": int(primary_proxy_id), "updated_at": now}, + ) + await session.execute(stmt) + + resp: dict[str, Any] = {"recomputed": len(token_ids)} + if not strict: + resp.update( + { + "strict": False, + "over_capacity_assigned": int(over_capacity_assigned), + "capacity": int(capacity), + "token_count": len(token_ids), + "proxy_count": len(proxy_ids_norm), + "max_tokens_per_proxy": int(max_tokens_per_proxy), + "weight_sum": int(weight_sum), + } + ) + return resp + diff --git a/backend/app/core/config.py b/backend/app/core/config.py new file mode 100644 index 0000000..4d704a0 --- /dev/null +++ b/backend/app/core/config.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping + +from cryptography.fernet import Fernet + +from app.core.crypto import FieldEncryptor +from app.core.logging import get_logger + +log = get_logger(__name__) + + +@dataclass(frozen=True, slots=True) +class Settings: + app_env: str + database_url: str + secret_key: str + field_encryption_key: str + admin_username: str + admin_password: str + pixiv_oauth_client_id: str + pixiv_oauth_client_secret: str + pixiv_oauth_hash_secret: str + imgproxy_base_url: str + imgproxy_key: str + imgproxy_salt: str + imgproxy_max_dim: int + imgproxy_default_options: str + imgproxy_url_chunk_size: int + public_api_key_required: bool + public_api_key_rpm: int + public_api_key_burst: int + + @property + def is_prod(self) -> bool: + return self.app_env in {"prod", "production"} + + +def _get(env: Mapping[str, str], key: str, default: str) -> str: + value = env.get(key, default) + return value.strip() + + +def _get_bool(env: Mapping[str, str], key: str, default: bool) -> bool: + raw = _get(env, key, "1" if default else "0").lower() + if raw in {"1", "true", "yes", "y", "on"}: + return True + if raw in {"0", "false", "no", "n", "off"}: + return False + return default + + +def _read_key_file(path: Path) -> str | None: + try: + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: + return None + except Exception as exc: + log.warning("field_encryption_key_read_failed path=%s err=%s", str(path), type(exc).__name__) + return None + + value = raw.strip() + return value or None + + +def _atomic_write(path: Path, *, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(content, encoding="utf-8") + try: + os.chmod(tmp, 0o600) + except Exception: + pass + os.replace(tmp, path) + + +def _ensure_field_encryption_key(env: Mapping[str, str], *, app_env: str) -> str: + key = _get(env, "FIELD_ENCRYPTION_KEY", "") + if key: + FieldEncryptor.from_key(key) + return key + + file_raw = _get(env, "FIELD_ENCRYPTION_KEY_FILE", "") + key_file = Path(file_raw) if file_raw else Path("./data/field_encryption_key") + + from_file = _read_key_file(key_file) + if from_file is not None: + FieldEncryptor.from_key(from_file) + return from_file + + if app_env in {"prod", "production"}: + return "" + + generated = Fernet.generate_key().decode("utf-8") + try: + _atomic_write(key_file, content=generated + "\n") + log.info("field_encryption_key_generated path=%s", str(key_file)) + except Exception as exc: + log.warning( + "field_encryption_key_generated_not_persisted path=%s err=%s", + str(key_file), + type(exc).__name__, + ) + return generated + + +def _load_pixiv_oauth_config(env: Mapping[str, str]) -> tuple[str, str, str]: + # NOTE: We intentionally do NOT ship any embedded Pixiv OAuth credentials. + # If you want token refresh / metadata hydration jobs, configure these via env. + client_id = _get(env, "PIXIV_OAUTH_CLIENT_ID", "") + client_secret = _get(env, "PIXIV_OAUTH_CLIENT_SECRET", "") + hash_secret = _get(env, "PIXIV_OAUTH_HASH_SECRET", "") + return client_id, client_secret, hash_secret + + +def load_settings(env: Mapping[str, str] | None = None) -> Settings: + env = env or os.environ + + app_env = _get(env, "APP_ENV", "dev").lower() + database_url = _get(env, "DATABASE_URL", "sqlite+aiosqlite:///./data/app.db") + secret_key = _get(env, "SECRET_KEY", "dev-secret-key" if app_env != "prod" else "") + field_encryption_key = _ensure_field_encryption_key(env, app_env=app_env) + + admin_username = _get(env, "ADMIN_USERNAME", "admin") + admin_password = _get(env, "ADMIN_PASSWORD", "admin" if app_env != "prod" else "") + + pixiv_oauth_client_id, pixiv_oauth_client_secret, pixiv_oauth_hash_secret = _load_pixiv_oauth_config(env) + + imgproxy_base_url = _get(env, "IMGPROXY_BASE_URL", "") + imgproxy_key = _get(env, "IMGPROXY_KEY", "") + imgproxy_salt = _get(env, "IMGPROXY_SALT", "") + try: + imgproxy_max_dim = int(_get(env, "IMGPROXY_MAX_DIM", "2048") or "2048") + except Exception: + imgproxy_max_dim = 2048 + imgproxy_max_dim = max(16, min(int(imgproxy_max_dim), 20_000)) + + imgproxy_default_options = _get(env, "IMGPROXY_DEFAULT_OPTIONS", "") + + try: + imgproxy_url_chunk_size = int(_get(env, "IMGPROXY_URL_CHUNK_SIZE", "16") or "16") + except Exception: + imgproxy_url_chunk_size = 16 + imgproxy_url_chunk_size = max(0, min(int(imgproxy_url_chunk_size), 128)) + + public_api_key_required = _get_bool(env, "PUBLIC_API_KEY_REQUIRED", False) + try: + public_api_key_rpm = int(_get(env, "PUBLIC_API_KEY_RPM", "0") or "0") + except Exception: + public_api_key_rpm = 0 + public_api_key_rpm = max(0, min(int(public_api_key_rpm), 10_000_000)) + try: + public_api_key_burst = int(_get(env, "PUBLIC_API_KEY_BURST", "0") or "0") + except Exception: + public_api_key_burst = 0 + public_api_key_burst = max(0, min(int(public_api_key_burst), 10_000_000)) + + settings = Settings( + app_env=app_env, + database_url=database_url, + secret_key=secret_key, + field_encryption_key=field_encryption_key, + admin_username=admin_username, + admin_password=admin_password, + pixiv_oauth_client_id=pixiv_oauth_client_id, + pixiv_oauth_client_secret=pixiv_oauth_client_secret, + pixiv_oauth_hash_secret=pixiv_oauth_hash_secret, + imgproxy_base_url=imgproxy_base_url, + imgproxy_key=imgproxy_key, + imgproxy_salt=imgproxy_salt, + imgproxy_max_dim=imgproxy_max_dim, + imgproxy_default_options=imgproxy_default_options, + imgproxy_url_chunk_size=imgproxy_url_chunk_size, + public_api_key_required=public_api_key_required, + public_api_key_rpm=public_api_key_rpm, + public_api_key_burst=public_api_key_burst, + ) + + if settings.is_prod: + missing: list[str] = [] + if not settings.secret_key: + missing.append("SECRET_KEY") + if not settings.field_encryption_key: + missing.append("FIELD_ENCRYPTION_KEY") + if not settings.admin_password: + missing.append("ADMIN_PASSWORD") + if settings.imgproxy_base_url and (not settings.imgproxy_key or not settings.imgproxy_salt): + missing.append("IMGPROXY_KEY/IMGPROXY_SALT") + if missing: + raise ValueError(f"Missing required env vars for prod: {', '.join(missing)}") + + return settings diff --git a/backend/app/core/crypto.py b/backend/app/core/crypto.py new file mode 100644 index 0000000..beb2a64 --- /dev/null +++ b/backend/app/core/crypto.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from cryptography.fernet import Fernet, InvalidToken + + +MASKED = "***" + + +def mask_secret(value: str | None) -> str: + value = (value or "").strip() + return MASKED if value else "" + + +@dataclass(frozen=True, slots=True) +class FieldEncryptor: + _fernet: Fernet + + @classmethod + def from_key(cls, key: str) -> "FieldEncryptor": + key = (key or "").strip() + if not key: + raise ValueError("FIELD_ENCRYPTION_KEY is required") + try: + fernet = Fernet(key.encode("utf-8")) + except Exception as exc: + raise ValueError("Invalid FIELD_ENCRYPTION_KEY") from exc + return cls(_fernet=fernet) + + def encrypt_text(self, plaintext: str) -> str: + if not isinstance(plaintext, str): + raise TypeError("plaintext must be str") + token = self._fernet.encrypt(plaintext.encode("utf-8")) + return token.decode("utf-8") + + def decrypt_text(self, token: str) -> str: + if not isinstance(token, str): + raise TypeError("token must be str") + try: + plaintext = self._fernet.decrypt(token.encode("utf-8")) + except InvalidToken as exc: + raise ValueError("Invalid encrypted value") from exc + return plaintext.decode("utf-8") diff --git a/backend/app/core/data_files.py b/backend/app/core/data_files.py new file mode 100644 index 0000000..06a603e --- /dev/null +++ b/backend/app/core/data_files.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from pathlib import Path + +from sqlalchemy.engine.url import make_url + + +def get_sqlite_db_dir(database_url: str) -> Path: + """ + Returns the directory containing the SQLite database file. + + For non-SQLite or in-memory DBs, falls back to ./data relative to CWD. + """ + + fallback = Path("./data").resolve() + + try: + url = make_url(database_url) + except Exception: + return fallback + + backend = (url.get_backend_name() or "").lower() + if backend != "sqlite": + return fallback + + db = str(url.database or "").strip() + if not db or db == ":memory:": + return fallback + + p = Path(db) + if not p.is_absolute(): + p = Path.cwd() / p + return p.resolve().parent + + +def make_file_ref(path: Path, *, base_dir: Path) -> str: + base = base_dir.resolve() + target = path.resolve() + return target.relative_to(base).as_posix() + + +def resolve_file_ref(file_ref: str, *, base_dir: Path) -> Path: + raw = (file_ref or "").strip() + if not raw: + raise ValueError("file_ref is required") + + rel = Path(raw) + if rel.is_absolute(): + raise ValueError("file_ref must be relative") + if any(part == ".." for part in rel.parts): + raise ValueError("file_ref must not contain '..'") + + base = base_dir.resolve() + target = (base / rel).resolve() + target.relative_to(base) # raises if escaped + return target + diff --git a/backend/app/core/effective_settings.py b/backend/app/core/effective_settings.py new file mode 100644 index 0000000..7b3248e --- /dev/null +++ b/backend/app/core/effective_settings.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Mapping + +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.config import Settings, load_settings +from app.core.runtime_settings import RuntimeConfig, load_runtime_config + + +@dataclass(frozen=True, slots=True) +class EffectiveSettings: + env: Settings + runtime: RuntimeConfig + + +async def load_effective_settings(engine: AsyncEngine, *, env: Mapping[str, str] | None = None) -> EffectiveSettings: + settings = load_settings(env) + runtime = await load_runtime_config(engine) + return EffectiveSettings(env=settings, runtime=runtime) + diff --git a/backend/app/core/errors.py b/backend/app/core/errors.py new file mode 100644 index 0000000..411d2ef --- /dev/null +++ b/backend/app/core/errors.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class ErrorCode(str, Enum): + # 4.1 通用 + BAD_REQUEST = "BAD_REQUEST" + UNAUTHORIZED = "UNAUTHORIZED" + FORBIDDEN = "FORBIDDEN" + NOT_FOUND = "NOT_FOUND" + RATE_LIMITED = "RATE_LIMITED" + INTERNAL_ERROR = "INTERNAL_ERROR" + # 4.2 Random + NO_MATCH = "NO_MATCH" + UPSTREAM_STREAM_ERROR = "UPSTREAM_STREAM_ERROR" + UPSTREAM_403 = "UPSTREAM_403" + UPSTREAM_404 = "UPSTREAM_404" + UPSTREAM_RATE_LIMIT = "UPSTREAM_RATE_LIMIT" + # 4.3 Import + INVALID_UPLOAD_TYPE = "INVALID_UPLOAD_TYPE" + PAYLOAD_TOO_LARGE = "PAYLOAD_TOO_LARGE" + UNSUPPORTED_URL = "UNSUPPORTED_URL" + # 4.4 Pixiv/Token + TOKEN_REFRESH_FAILED = "TOKEN_REFRESH_FAILED" + TOKEN_BACKOFF = "TOKEN_BACKOFF" + NO_TOKEN_AVAILABLE = "NO_TOKEN_AVAILABLE" + # 4.5 Proxy + PROXY_REQUIRED = "PROXY_REQUIRED" + PROXY_AUTH_FAILED = "PROXY_AUTH_FAILED" + PROXY_CONNECT_FAILED = "PROXY_CONNECT_FAILED" + + +UNKNOWN_REQUEST_ID = "req_unknown" + +_DEFAULT_ZH_MESSAGE_BY_CODE: dict[ErrorCode, str] = { + ErrorCode.BAD_REQUEST: "请求参数错误", + ErrorCode.UNAUTHORIZED: "未授权", + ErrorCode.FORBIDDEN: "禁止访问", + ErrorCode.NOT_FOUND: "资源不存在", + ErrorCode.RATE_LIMITED: "请求过于频繁", + ErrorCode.INTERNAL_ERROR: "服务器内部错误", + ErrorCode.NO_MATCH: "没有匹配的图片", + ErrorCode.UPSTREAM_STREAM_ERROR: "上游请求失败", + ErrorCode.UPSTREAM_403: "上游拒绝访问（403）", + ErrorCode.UPSTREAM_404: "上游资源不存在（404）", + ErrorCode.UPSTREAM_RATE_LIMIT: "上游触发限流（429）", + ErrorCode.INVALID_UPLOAD_TYPE: "上传类型不支持", + ErrorCode.PAYLOAD_TOO_LARGE: "请求体过大", + ErrorCode.UNSUPPORTED_URL: "链接格式不支持", + ErrorCode.TOKEN_REFRESH_FAILED: "令牌刷新失败", + ErrorCode.TOKEN_BACKOFF: "令牌已进入退避", + ErrorCode.NO_TOKEN_AVAILABLE: "没有可用令牌", + ErrorCode.PROXY_REQUIRED: "需要代理", + ErrorCode.PROXY_AUTH_FAILED: "代理认证失败", + ErrorCode.PROXY_CONNECT_FAILED: "代理连接失败", +} + + +def _is_ascii_only(text: str) -> bool: + return all(ord(ch) < 128 for ch in text) + + +_ASCII_MESSAGE_MAP: dict[str, str] = { + "missing api key": "缺少 API Key", + "invalid api key": "API Key 无效", + "rate limited": "请求过于频繁", + "database unavailable": "数据库不可用", + "invalid json body": "JSON 请求体无效", + "unsupported limit": "limit 参数不支持", + "unsupported cursor": "cursor 参数不支持", + "unsupported status": "status 参数不支持", + "unsupported type": "type 参数不支持", + "unsupported format": "format 参数不支持", + "unsupported redirect": "redirect 参数不支持", + "unsupported seed": "seed 参数不支持", + "unsupported ai_type": "ai_type 参数不支持", + "unsupported orientation": "orientation 参数不支持", + "unsupported r18": "r18 参数不支持", + "unsupported min_*": "min_* 参数不支持", + "too many tag filters": "标签筛选条件过多", + "pixiv oauth not configured": "未配置 Pixiv OAuth（client_id/client_secret）", + "encryption not configured": "未配置加密密钥（FIELD_ENCRYPTION_KEY）", + "invalid stored token": "令牌密文无效（无法解密）", + "token not found": "令牌不存在", + "job not found": "任务不存在", + "proxy endpoint not found": "代理节点不存在", + "proxy pool not found": "代理池不存在", + "job is running": "任务正在运行中", + "proxy authentication failed": "代理认证失败", + "proxy connect failed": "代理连接失败", + "upstream request failed": "上游请求失败", + "upstream forbidden": "上游拒绝访问（403）", + "upstream not found": "上游资源不存在（404）", + "upstream rate limited": "上游触发限流（429）", + "upstream error": "上游错误", + "upstream request failed after attempts.": "多次尝试后上游请求仍失败", + "invalid proxy endpoint": "代理节点配置无效", + "unsupported conflict_policy": "conflict_policy 参数不支持", +} + + +def normalize_error_message(*, code: ErrorCode, message: str) -> str: + msg = str(message or "").strip() + if not msg: + return _DEFAULT_ZH_MESSAGE_BY_CODE.get(code, "请求失败") + + if not _is_ascii_only(msg): + return msg + + key = msg.lower() + if key in _ASCII_MESSAGE_MAP: + return _ASCII_MESSAGE_MAP[key] + + if key.startswith("unsupported "): + rest = msg[len("Unsupported ") :].strip() + return f"参数不支持：{rest}" if rest else _DEFAULT_ZH_MESSAGE_BY_CODE.get(code, "请求失败") + + if key.startswith("invalid "): + rest = msg[len("Invalid ") :].strip() + return f"参数无效：{rest}" if rest else _DEFAULT_ZH_MESSAGE_BY_CODE.get(code, "请求失败") + + if key.startswith("missing "): + rest = msg[len("Missing ") :].strip() + return f"缺少字段：{rest}" if rest else _DEFAULT_ZH_MESSAGE_BY_CODE.get(code, "请求失败") + + return _DEFAULT_ZH_MESSAGE_BY_CODE.get(code, "请求失败") + + +@dataclass(frozen=True, slots=True) +class ApiError(Exception): + code: ErrorCode + message: str + status_code: int = 400 + details: dict[str, Any] | None = None + + +def _coerce_request_id(request_id: str | None) -> str: + request_id = (request_id or "").strip() + return request_id if request_id else UNKNOWN_REQUEST_ID + + +def _request_id_from_request(request: Any | None) -> str | None: + if request is None: + return None + request_id = getattr(getattr(request, "state", None), "request_id", None) + if request_id: + return str(request_id) + header = request.headers.get("X-Request-Id") + return header.strip() if header else None + + +def error_body( + *, + code: ErrorCode, + message: str, + request_id: str | None = None, + details: dict[str, Any] | None = None, +) -> dict[str, Any]: + return { + "ok": False, + "code": code.value, + "message": normalize_error_message(code=code, message=message), + "request_id": _coerce_request_id(request_id), + "details": details or {}, + } + + +def json_error_response( + *, + code: ErrorCode, + message: str, + status_code: int, + request: Any | None = None, + request_id: str | None = None, + details: dict[str, Any] | None = None, +) -> Any: + if request_id is None: + request_id = _request_id_from_request(request) + from fastapi.responses import JSONResponse + + return JSONResponse( + status_code=status_code, + content=error_body(code=code, message=message, request_id=request_id, details=details), + ) diff --git a/backend/app/core/failover.py b/backend/app/core/failover.py new file mode 100644 index 0000000..1873ee7 --- /dev/null +++ b/backend/app/core/failover.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from enum import Enum + +import httpx + + +class OutboundErrorKind(str, Enum): + PROXY_CONNECT = "proxy_connect" + PROXY_AUTH = "proxy_auth" + PIXIV_RATE_LIMIT = "pixiv_rate_limit" + + +def classify_httpx_exception(exc: BaseException) -> OutboundErrorKind | None: + if isinstance(exc, httpx.ProxyError): + msg = str(exc).lower() + if "407" in msg or "proxy authentication" in msg: + return OutboundErrorKind.PROXY_AUTH + return OutboundErrorKind.PROXY_CONNECT + return None + + +def classify_pixiv_rate_limit(*, status_code: int, body_text: str | None) -> OutboundErrorKind | None: + if int(status_code) != 403: + return None + text = (body_text or "").strip().lower() + if "rate limit" in text: + return OutboundErrorKind.PIXIV_RATE_LIMIT + return None + + +def should_override_proxy(kind: OutboundErrorKind) -> bool: + return kind in {OutboundErrorKind.PROXY_CONNECT, OutboundErrorKind.PROXY_AUTH} + + +def should_backoff_token(kind: OutboundErrorKind) -> bool: + return kind == OutboundErrorKind.PIXIV_RATE_LIMIT + + +def proxy_override_ttl_seconds(*, attempt: int) -> int: + attempt_i = int(attempt) + if attempt_i <= 0: + return 0 + + schedule = { + 1: 20 * 60, + 2: 60 * 60, + 3: 6 * 60 * 60, + } + if attempt_i in schedule: + return schedule[attempt_i] + + base = 6 * 60 * 60 + seconds = base * (2 ** (attempt_i - 3)) + return min(seconds, 24 * 60 * 60) + + +def pixiv_rate_limit_backoff_seconds(*, attempt: int) -> int: + attempt_i = int(attempt) + if attempt_i <= 0: + return 0 + + schedule = { + 1: 60, + 2: 5 * 60, + 3: 15 * 60, + 4: 60 * 60, + 5: 6 * 60 * 60, + } + if attempt_i in schedule: + return schedule[attempt_i] + + base = 6 * 60 * 60 + seconds = base * (2 ** (attempt_i - 5)) + return min(seconds, 24 * 60 * 60) + diff --git a/backend/app/core/http_stream.py b/backend/app/core/http_stream.py new file mode 100644 index 0000000..12cd755 --- /dev/null +++ b/backend/app/core/http_stream.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from typing import Any + +import httpx +from starlette.responses import StreamingResponse + +from app.core.errors import ApiError, ErrorCode +from app.core.metrics import UPSTREAM_STREAM_ERRORS_TOTAL + +PIXIV_REFERER = "https://www.pixiv.net/" + + +async def stream_url( + url: str, + *, + transport: httpx.BaseTransport | None = None, + proxy: str | None = None, + cache_control: str, + referer: str = PIXIV_REFERER, + timeout_s: float = 30.0, + range_header: str | None = None, +) -> StreamingResponse: + client = httpx.AsyncClient( + transport=transport, + proxy=proxy, + follow_redirects=True, + timeout=httpx.Timeout(timeout_s, connect=10.0), + ) + + request_headers: dict[str, str] = {} + if referer: + request_headers["Referer"] = referer + if range_header: + request_headers["Range"] = range_header + request = client.build_request("GET", url, headers=request_headers) + + try: + upstream = await client.send(request, stream=True) + except httpx.ProxyError as exc: + UPSTREAM_STREAM_ERRORS_TOTAL.inc() + await client.aclose() + msg = str(exc).lower() + if "407" in msg or "proxy authentication" in msg: + raise ApiError( + code=ErrorCode.PROXY_AUTH_FAILED, + message="代理认证失败", + status_code=502, + ) from exc + raise ApiError( + code=ErrorCode.PROXY_CONNECT_FAILED, + message="代理连接失败", + status_code=502, + ) from exc + except Exception as exc: + UPSTREAM_STREAM_ERRORS_TOTAL.inc() + await client.aclose() + raise ApiError(code=ErrorCode.UPSTREAM_STREAM_ERROR, message="上游请求失败", status_code=502) from exc + + if upstream.status_code not in {200, 206}: + status = upstream.status_code + UPSTREAM_STREAM_ERRORS_TOTAL.inc() + await upstream.aclose() + await client.aclose() + if status == 403: + raise ApiError(code=ErrorCode.UPSTREAM_403, message="上游拒绝访问（403）", status_code=502) + if status == 404: + raise ApiError(code=ErrorCode.UPSTREAM_404, message="上游资源不存在（404）", status_code=502) + if status == 429: + raise ApiError(code=ErrorCode.UPSTREAM_RATE_LIMIT, message="上游触发限流（429）", status_code=502) + raise ApiError(code=ErrorCode.UPSTREAM_STREAM_ERROR, message="上游错误", status_code=502) + + media_type = upstream.headers.get("content-type") or "application/octet-stream" + content_length = upstream.headers.get("content-length") + + async def _iter_bytes() -> Any: + try: + async for chunk in upstream.aiter_bytes(): + yield chunk + except Exception: + UPSTREAM_STREAM_ERRORS_TOTAL.inc() + raise + finally: + await upstream.aclose() + await client.aclose() + + accept_ranges = upstream.headers.get("accept-ranges") + content_range = upstream.headers.get("content-range") + + resp = StreamingResponse(_iter_bytes(), status_code=upstream.status_code, media_type=media_type) + resp.headers["Cache-Control"] = cache_control + if content_length: + resp.headers["Content-Length"] = content_length + if accept_ranges: + resp.headers["Accept-Ranges"] = accept_ranges + if content_range: + resp.headers["Content-Range"] = content_range + return resp diff --git a/backend/app/core/imgproxy.py b/backend/app/core/imgproxy.py new file mode 100644 index 0000000..bb30ff9 --- /dev/null +++ b/backend/app/core/imgproxy.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import base64 +import hashlib +import hmac +from dataclasses import dataclass +from typing import Mapping + +from app.core.config import Settings + + +@dataclass(frozen=True, slots=True) +class ImgproxyConfig: + base_url: str + key: bytes + salt: bytes + max_dim: int + default_options: str + url_chunk_size: int + + +def _decode_hex(raw: str, *, name: str) -> bytes: + raw = (raw or "").strip() + if not raw: + raise ValueError(f"{name} is required") + try: + return bytes.fromhex(raw) + except Exception as exc: + raise ValueError(f"{name} must be hex") from exc + + +def load_imgproxy_config_from_settings(settings: Settings) -> ImgproxyConfig | None: + base_url = (settings.imgproxy_base_url or "").strip() + if not base_url: + return None + + key = _decode_hex(settings.imgproxy_key, name="IMGPROXY_KEY") + salt = _decode_hex(settings.imgproxy_salt, name="IMGPROXY_SALT") + + default_options = (settings.imgproxy_default_options or "").strip().strip("/") + if not default_options: + default_options = f"rs:fit:{int(settings.imgproxy_max_dim)}:{int(settings.imgproxy_max_dim)}" + + return ImgproxyConfig( + base_url=base_url.rstrip("/"), + key=key, + salt=salt, + max_dim=int(settings.imgproxy_max_dim), + default_options=default_options, + url_chunk_size=int(settings.imgproxy_url_chunk_size), + ) + + +def urlsafe_b64_no_pad(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + + +def encode_source_url(source_url: str, *, chunk_size: int) -> str: + source_url = (source_url or "").strip() + if not source_url: + raise ValueError("source_url is required") + + encoded = urlsafe_b64_no_pad(source_url.encode("utf-8")) + if chunk_size <= 0 or len(encoded) <= chunk_size: + return encoded + return "/".join(encoded[i : i + chunk_size] for i in range(0, len(encoded), chunk_size)) + + +def sign_path(cfg: ImgproxyConfig, path_after_signature: str) -> str: + path = (path_after_signature or "").strip() + if not path.startswith("/"): + raise ValueError("path_after_signature must start with '/'") + + mac = hmac.new(cfg.key, digestmod=hashlib.sha256) + mac.update(cfg.salt) + mac.update(path.encode("utf-8")) + return urlsafe_b64_no_pad(mac.digest()) + + +def build_processing_path( + *, + processing_options: str, + source_url: str, + extension: str, + url_chunk_size: int, +) -> str: + processing_options = (processing_options or "").strip().strip("/") + if not processing_options: + raise ValueError("processing_options is required") + + extension = (extension or "").strip().lower().lstrip(".") + if not extension or len(extension) > 10 or any(c for c in extension if not (c.isalnum() or c == "_")): + raise ValueError("extension is invalid") + + encoded = encode_source_url(source_url, chunk_size=int(url_chunk_size)) + return f"/{processing_options}/{encoded}.{extension}" + + +def build_signed_processing_url( + cfg: ImgproxyConfig, + *, + source_url: str, + extension: str, + processing_options: str | None = None, +) -> str: + path = build_processing_path( + processing_options=processing_options or cfg.default_options, + source_url=source_url, + extension=extension, + url_chunk_size=int(cfg.url_chunk_size), + ) + sig = sign_path(cfg, path) + return f"{cfg.base_url}/{sig}{path}" + + +def load_imgproxy_config(env: Mapping[str, str]) -> ImgproxyConfig | None: + base_url = (env.get("IMGPROXY_BASE_URL") or "").strip() + if not base_url: + return None + + key = _decode_hex(env.get("IMGPROXY_KEY") or "", name="IMGPROXY_KEY") + salt = _decode_hex(env.get("IMGPROXY_SALT") or "", name="IMGPROXY_SALT") + + try: + max_dim = int((env.get("IMGPROXY_MAX_DIM") or "2048").strip() or "2048") + except Exception: + max_dim = 2048 + max_dim = max(16, min(int(max_dim), 20_000)) + + default_options = (env.get("IMGPROXY_DEFAULT_OPTIONS") or "").strip().strip("/") + if not default_options: + default_options = f"rs:fit:{max_dim}:{max_dim}" + + try: + url_chunk_size = int((env.get("IMGPROXY_URL_CHUNK_SIZE") or "16").strip() or "16") + except Exception: + url_chunk_size = 16 + url_chunk_size = max(0, min(int(url_chunk_size), 128)) + + return ImgproxyConfig( + base_url=base_url.rstrip("/"), + key=key, + salt=salt, + max_dim=max_dim, + default_options=default_options, + url_chunk_size=url_chunk_size, + ) + diff --git a/backend/app/core/logging.py b/backend/app/core/logging.py new file mode 100644 index 0000000..b4748de --- /dev/null +++ b/backend/app/core/logging.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import logging +from typing import Any + +from app.core.redact import redact_any + + +class RedactFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + try: + message = record.getMessage() + record.msg = redact_any(message) + record.args = () + for key, value in list(record.__dict__.items()): + if key.startswith("_"): + continue + record.__dict__[key] = redact_any(value) + except Exception: + pass + return True + + +def configure_logging(level: int = logging.INFO) -> None: + logging.basicConfig(level=level, format="%(levelname)s %(name)s %(message)s") + logging.getLogger().addFilter(RedactFilter()) + + +def get_logger(name: str) -> logging.Logger: + return logging.getLogger(name) diff --git a/backend/app/core/metrics.py b/backend/app/core/metrics.py new file mode 100644 index 0000000..482a344 --- /dev/null +++ b/backend/app/core/metrics.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +from collections.abc import Iterable + +from prometheus_client import Counter, Gauge, Histogram + +RANDOM_RESULTS: tuple[str, ...] = ( + "ok", + "no_match", + "upstream_error", + "bad_request", + "error", +) + +JOB_STATUSES: tuple[str, ...] = ( + "pending", + "running", + "paused", + "canceled", + "completed", + "failed", + "dlq", +) + +PROXY_STATES: tuple[str, ...] = ( + "total", + "enabled", + "healthy", + "unhealthy", + "blacklisted", +) + +RANDOM_REQUESTS_TOTAL = Counter( + "new_pixiv_random_requests_total", + "Total /random requests by result.", + ["result"], +) + +RANDOM_NO_MATCH_TOTAL = Counter( + "new_pixiv_random_no_match_total", + "Total /random NO_MATCH responses.", +) + +RANDOM_OPPORTUNISTIC_HYDRATE_ENQUEUED_TOTAL = Counter( + "new_pixiv_random_opportunistic_hydrate_enqueued_total", + "Total opportunistic hydrate_metadata enqueues from /random.", +) + +RANDOM_LATENCY_SECONDS = Histogram( + "new_pixiv_random_latency_seconds", + "Latency for /random endpoint (seconds).", + buckets=( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + ), +) + +UPSTREAM_STREAM_ERRORS_TOTAL = Counter( + "new_pixiv_upstream_stream_errors_total", + "Total upstream stream failures (stream_url).", +) + +JOBS_CLAIM_TOTAL = Counter( + "new_pixiv_jobs_claim_total", + "Total jobs claimed by workers.", +) + +JOBS_FAILED_TOTAL = Counter( + "new_pixiv_jobs_failed_total", + "Total jobs transitioned to failed/dlq status.", +) + +TOKEN_REFRESH_FAIL_TOTAL = Counter( + "new_pixiv_token_refresh_fail_total", + "Total token refresh failures.", +) + +JOBS_STATUS_COUNT = Gauge( + "new_pixiv_jobs_status_count", + "Current jobs count by status (from SQLite).", + ["status"], +) + +PROXY_ENDPOINTS_STATE_COUNT = Gauge( + "new_pixiv_proxy_endpoints_state_count", + "Current proxy endpoints count by state (from SQLite).", + ["state"], +) + +PROXY_PROBE_LATENCY_MS = Histogram( + "new_pixiv_proxy_probe_latency_ms", + "Proxy probe latency (ms).", + buckets=( + 10.0, + 25.0, + 50.0, + 100.0, + 250.0, + 500.0, + 1000.0, + 2000.0, + 5000.0, + 10000.0, + ), +) + +METRICS_SCRAPE_ERRORS_TOTAL = Counter( + "new_pixiv_metrics_scrape_errors_total", + "Total /metrics scrape errors while querying backing dependencies.", +) + +METRICS_LAST_SCRAPE_SUCCESS = Gauge( + "new_pixiv_metrics_last_scrape_success", + "Last /metrics scrape success (1=ok, 0=error).", +) + + +def _init_labelsets() -> None: + for result in RANDOM_RESULTS: + RANDOM_REQUESTS_TOTAL.labels(result=result).inc(0) + RANDOM_NO_MATCH_TOTAL.inc(0) + RANDOM_OPPORTUNISTIC_HYDRATE_ENQUEUED_TOTAL.inc(0) + UPSTREAM_STREAM_ERRORS_TOTAL.inc(0) + JOBS_CLAIM_TOTAL.inc(0) + JOBS_FAILED_TOTAL.inc(0) + TOKEN_REFRESH_FAIL_TOTAL.inc(0) + for status in JOB_STATUSES: + JOBS_STATUS_COUNT.labels(status=status).set(0) + for state in PROXY_STATES: + PROXY_ENDPOINTS_STATE_COUNT.labels(state=state).set(0) + METRICS_LAST_SCRAPE_SUCCESS.set(1) + + +_init_labelsets() + + +def observe_random_result(*, result: str, duration_s: float | None) -> None: + result = (result or "").strip() + if result not in RANDOM_RESULTS: + result = "error" + RANDOM_REQUESTS_TOTAL.labels(result=result).inc() + if result == "no_match": + RANDOM_NO_MATCH_TOTAL.inc() + if duration_s is not None and duration_s >= 0: + RANDOM_LATENCY_SECONDS.observe(duration_s) + + +def set_jobs_status_counts(counts: dict[str, int]) -> None: + for status in JOB_STATUSES: + JOBS_STATUS_COUNT.labels(status=status).set(float(int(counts.get(status, 0) or 0))) + + +def set_proxy_state_counts(counts: dict[str, int]) -> None: + for state in PROXY_STATES: + PROXY_ENDPOINTS_STATE_COUNT.labels(state=state).set(float(int(counts.get(state, 0) or 0))) + + +def ensure_known_keys(keys: Iterable[str], counts: dict[str, int]) -> dict[str, int]: + out = dict(counts) + for k in keys: + out.setdefault(k, 0) + return out diff --git a/backend/app/core/pixiv_urls.py b/backend/app/core/pixiv_urls.py new file mode 100644 index 0000000..9f56f93 --- /dev/null +++ b/backend/app/core/pixiv_urls.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from urllib.parse import urlparse + + +@dataclass(frozen=True, slots=True) +class PixivOriginalUrl: + illust_id: int + page_index: int + ext: str + + +_PIXIV_P_RE = re.compile(r"(?P\d+)_p(?P\d+)(?:_master1200)?\.(?P[A-Za-z0-9]+)$") +_PIXIV_UGOIRA_RE = re.compile(r"(?P\d+)_ugoira(?P\d+)\.(?P[A-Za-z0-9]+)$") + +ALLOWED_IMAGE_EXTS = {"jpg", "jpeg", "png", "gif", "webp"} + + +def parse_pixiv_original_url(url: str) -> PixivOriginalUrl: + url = url.strip() + if not url: + raise ValueError("url is required") + + parsed = urlparse(url) + if parsed.scheme.lower() not in {"http", "https"}: + raise ValueError("unsupported scheme") + + host = (parsed.hostname or "").lower() + if not host.endswith("pximg.net"): + raise ValueError("unsupported host") + + m = _PIXIV_P_RE.search(parsed.path) or _PIXIV_UGOIRA_RE.search(parsed.path) + if not m: + raise ValueError("unsupported pixiv original url") + + illust_id = int(m.group("illust_id")) + page_index = int(m.group("page_index")) + ext = m.group("ext").lower() + if ext not in ALLOWED_IMAGE_EXTS: + raise ValueError("unsupported ext") + + return PixivOriginalUrl(illust_id=illust_id, page_index=page_index, ext=ext) diff --git a/backend/app/core/proxy_routing.py b/backend/app/core/proxy_routing.py new file mode 100644 index 0000000..0cc9ca1 --- /dev/null +++ b/backend/app/core/proxy_routing.py @@ -0,0 +1,539 @@ +from __future__ import annotations + +import random +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlparse +from urllib.parse import quote + +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.config import Settings +from app.core.crypto import FieldEncryptor +from app.core.errors import ApiError, ErrorCode +from app.core.runtime_settings import RuntimeConfig +from app.core.time import iso_utc_ms +from app.db.session import with_sqlite_busy_retry + + +_PIXIV_HOST_SUFFIXES = ( + "pixiv.net", + "pximg.net", + "secure.pixiv.net", +) + + +def _normalize_host(host: str) -> str: + return (host or "").strip().lower().strip(".") + + +def host_from_url(url: str) -> str | None: + url = (url or "").strip() + if not url: + return None + try: + parsed = urlparse(url) + except Exception: + return None + host = _normalize_host(parsed.hostname or "") + return host or None + + +def _suffix_match(*, host: str, suffix: str) -> bool: + host = _normalize_host(host) + suffix = _normalize_host(suffix) + if not host or not suffix: + return False + return host == suffix or host.endswith("." + suffix) + + +def should_use_proxy_for_host(runtime: RuntimeConfig, *, host: str) -> bool: + if not bool(runtime.proxy_enabled): + return False + + host_n = _normalize_host(host) + if not host_n: + return False + + mode = (runtime.proxy_route_mode or "").strip().lower() + if mode in {"off"}: + return False + if mode == "all": + return True + if mode == "allowlist": + return any(_suffix_match(host=host_n, suffix=d) for d in (runtime.proxy_allowlist_domains or [])) + if mode == "pixiv_only": + return any(_suffix_match(host=host_n, suffix=s) for s in _PIXIV_HOST_SUFFIXES) + return False + + +def resolve_pool_id_for_host(runtime: RuntimeConfig, *, host: str) -> int | None: + host_n = _normalize_host(host) + if not host_n: + return None + + best_len = -1 + best: int | None = None + for suffix, pool_id in (runtime.proxy_route_pools or {}).items(): + suf = _normalize_host(str(suffix)) + if not suf: + continue + if not _suffix_match(host=host_n, suffix=suf): + continue + if len(suf) > best_len: + best_len = len(suf) + best = int(pool_id) + + if best is not None and int(best) > 0: + return int(best) + + if runtime.proxy_default_pool_id is not None and int(runtime.proxy_default_pool_id) > 0: + return int(runtime.proxy_default_pool_id) + + return None + + +def _weighted_choice(items: list[tuple[int, int]]) -> int | None: + if not items: + return None + total = sum(max(0, int(w)) for _pid, w in items) + if total <= 0: + items2 = [pid for pid, _w in items] + return int(random.choice(items2)) if items2 else None + r = random.random() * total + for pid, w in items: + w_i = max(0, int(w)) + if w_i <= 0: + continue + if r < w_i: + return int(pid) + r -= w_i + return int(items[-1][0]) + + +@dataclass(frozen=True, slots=True) +class ProxyUri: + uri: str + endpoint_id: int + pool_id: int + + +async def _first_enabled_pool_id(engine: AsyncEngine) -> int | None: + sql = "SELECT id FROM proxy_pools WHERE enabled=1 ORDER BY id ASC LIMIT 1;" + + async def _op() -> int | None: + async with engine.connect() as conn: + result = await conn.exec_driver_sql(sql) + value = result.scalar_one_or_none() + return int(value) if value is not None else None + + return await with_sqlite_busy_retry(_op) + + +async def _list_enabled_pool_ids(engine: AsyncEngine) -> list[int]: + sql = "SELECT id FROM proxy_pools WHERE enabled=1 ORDER BY id ASC;" + + async def _op() -> list[int]: + async with engine.connect() as conn: + rows = (await conn.exec_driver_sql(sql)).fetchall() + out: list[int] = [] + seen: set[int] = set() + for (pid,) in rows: + try: + pool_id = int(pid) + except Exception: + continue + if pool_id <= 0 or pool_id in seen: + continue + seen.add(pool_id) + out.append(pool_id) + return out + + return await with_sqlite_busy_retry(_op) + + +async def _pool_health_stats(engine: AsyncEngine, *, pool_id: int, now_iso: str) -> dict[str, Any]: + sql = """ +SELECT + COUNT(*) AS endpoints_total, + COALESCE(SUM(CASE + WHEN pe.blacklisted_until IS NULL OR pe.blacklisted_until <= :now THEN 1 + ELSE 0 + END), 0) AS endpoints_eligible, + MIN(CASE + WHEN pe.blacklisted_until > :now THEN pe.blacklisted_until + ELSE NULL + END) AS next_available_at +FROM proxy_pools pp +JOIN proxy_pool_endpoints ppe + ON ppe.pool_id = pp.id AND ppe.enabled = 1 +JOIN proxy_endpoints pe + ON pe.id = ppe.endpoint_id AND pe.enabled = 1 +WHERE pp.id = :pool_id AND pp.enabled = 1; +""".strip() + + async def _op() -> dict[str, Any]: + async with engine.connect() as conn: + row = (await conn.exec_driver_sql(sql, {"pool_id": int(pool_id), "now": str(now_iso)})).first() + if row is None: + return {"endpoints_total": 0, "endpoints_eligible": 0, "next_available_at": None} + total = int(row[0] or 0) + eligible = int(row[1] or 0) + next_available_at = str(row[2]).strip() if row[2] is not None else None + return { + "endpoints_total": max(0, total), + "endpoints_eligible": max(0, eligible), + "next_available_at": next_available_at or None, + } + + return await with_sqlite_busy_retry(_op) + + +async def _pick_endpoint_in_pool(engine: AsyncEngine, *, pool_id: int, now_iso: str) -> tuple[int, str, str, int, str, str] | None: + sql = """ +SELECT pe.id, pe.scheme, pe.host, pe.port, pe.username, pe.password_enc, ppe.weight, pe.last_ok_at, pe.last_fail_at +FROM proxy_pools pp +JOIN proxy_pool_endpoints ppe + ON ppe.pool_id = pp.id AND ppe.enabled = 1 +JOIN proxy_endpoints pe + ON pe.id = ppe.endpoint_id AND pe.enabled = 1 +WHERE pp.id = :pool_id AND pp.enabled = 1 + AND (pe.blacklisted_until IS NULL OR pe.blacklisted_until <= :now) +ORDER BY pe.id ASC; +""".strip() + + async def _op() -> tuple[int, str, str, int, str, str] | None: + async with engine.connect() as conn: + result = await conn.exec_driver_sql(sql, {"pool_id": int(pool_id), "now": now_iso}) + rows = result.fetchall() + ok_weighted: list[tuple[int, int]] = [] + unknown_weighted: list[tuple[int, int]] = [] + fail_weighted: list[tuple[int, int]] = [] + + for r in rows: + endpoint_id = int(r[0]) + weight = int(r[6] or 0) + last_ok_at = str(r[7]).strip() if r[7] is not None else "" + last_fail_at = str(r[8]).strip() if r[8] is not None else "" + + if last_ok_at and (not last_fail_at or last_ok_at >= last_fail_at): + ok_weighted.append((endpoint_id, weight)) + elif not last_ok_at and not last_fail_at: + unknown_weighted.append((endpoint_id, weight)) + else: + fail_weighted.append((endpoint_id, weight)) + + weighted = ok_weighted or unknown_weighted or fail_weighted + chosen_id = _weighted_choice(weighted) + if chosen_id is None: + return None + for r in rows: + if int(r[0]) == int(chosen_id): + return ( + int(r[0]), + str(r[1]), + str(r[2]), + int(r[3]), + str(r[4] or ""), + str(r[5] or ""), + ) + return None + + return await with_sqlite_busy_retry(_op) + + +async def _load_token_binding( + engine: AsyncEngine, + *, + token_id: int, + pool_id: int, +) -> tuple[int, int | None, str | None] | None: + sql = """ +SELECT primary_proxy_id, override_proxy_id, override_expires_at +FROM token_proxy_bindings +WHERE token_id=:token_id AND pool_id=:pool_id +LIMIT 1; +""".strip() + + async def _op() -> tuple[int, int | None, str | None] | None: + async with engine.connect() as conn: + result = await conn.exec_driver_sql(sql, {"token_id": int(token_id), "pool_id": int(pool_id)}) + row = result.first() + if row is None: + return None + primary_id = int(row[0]) + override_id = int(row[1]) if row[1] is not None else None + override_expires_at = str(row[2]) if row[2] is not None else None + return primary_id, override_id, override_expires_at + + return await with_sqlite_busy_retry(_op) + + +async def _load_endpoint_in_pool( + engine: AsyncEngine, + *, + pool_id: int, + endpoint_id: int, + now_iso: str, +) -> tuple[int, str, str, int, str, str] | None: + sql = """ +SELECT pe.id, pe.scheme, pe.host, pe.port, pe.username, pe.password_enc +FROM proxy_pools pp +JOIN proxy_pool_endpoints ppe + ON ppe.pool_id = pp.id AND ppe.enabled = 1 +JOIN proxy_endpoints pe + ON pe.id = ppe.endpoint_id AND pe.enabled = 1 +WHERE pp.id = :pool_id AND pp.enabled = 1 + AND pe.id = :endpoint_id + AND (pe.blacklisted_until IS NULL OR pe.blacklisted_until <= :now) +LIMIT 1; +""".strip() + + async def _op() -> tuple[int, str, str, int, str, str] | None: + async with engine.connect() as conn: + result = await conn.exec_driver_sql( + sql, + {"pool_id": int(pool_id), "endpoint_id": int(endpoint_id), "now": now_iso}, + ) + row = result.first() + if row is None: + return None + return ( + int(row[0]), + str(row[1]), + str(row[2]), + int(row[3]), + str(row[4] or ""), + str(row[5] or ""), + ) + + return await with_sqlite_busy_retry(_op) + + +def _build_proxy_uri( + encryptor: FieldEncryptor | None, + *, + scheme: str, + host: str, + port: int, + username: str, + password_enc: str, +) -> str: + scheme = (scheme or "").strip().lower() + host = (host or "").strip() + username = (username or "").strip() + password_enc = (password_enc or "").strip() + if not scheme or not host or int(port) <= 0: + raise ValueError("invalid proxy endpoint") + + password = "" + if password_enc: + if encryptor is None: + raise ValueError("FIELD_ENCRYPTION_KEY is required to decrypt proxy password") + password = encryptor.decrypt_text(password_enc) if password_enc else "" + + host_part = host + if ":" in host_part and not host_part.startswith("["): + host_part = f"[{host_part}]" + + auth = "" + if username: + user_q = quote(username, safe="") + pass_q = quote(password or "", safe="") + auth = f"{user_q}:{pass_q}@" + + return f"{scheme}://{auth}{host_part}:{int(port)}" + + +def _proxy_uri_from_endpoint_row( + settings: Settings, + *, + endpoint_id: int, + pool_id: int, + scheme: str, + host: str, + port: int, + username: str, + password_enc: str, +) -> ProxyUri: + encryptor = None + if str(password_enc or "").strip(): + try: + encryptor = FieldEncryptor.from_key(settings.field_encryption_key) + except Exception as exc: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="Encryption not configured", status_code=500) from exc + + try: + uri = _build_proxy_uri( + encryptor, + scheme=scheme, + host=host, + port=int(port), + username=username, + password_enc=password_enc, + ) + except ApiError: + raise + except Exception as exc: + raise ApiError(code=ErrorCode.INTERNAL_ERROR, message="Invalid proxy endpoint", status_code=500) from exc + + return ProxyUri(uri=uri, endpoint_id=int(endpoint_id), pool_id=int(pool_id)) + + +async def select_proxy_uri_for_url( + engine: AsyncEngine, + settings: Settings, + runtime: RuntimeConfig, + *, + url: str, + now_iso: str | None = None, + token_id: int | None = None, +) -> ProxyUri | None: + host = host_from_url(url) + if host is None: + return None + + if not should_use_proxy_for_host(runtime, host=host): + return None + + now_iso = now_iso or iso_utc_ms() + + preferred_pool_id = resolve_pool_id_for_host(runtime, host=host) + enabled_pools = await _list_enabled_pool_ids(engine) + enabled_pool_set = {int(pid) for pid in enabled_pools} + + pool_candidates: list[int] = [] + if preferred_pool_id is not None and int(preferred_pool_id) > 0: + preferred_pool_id_i = int(preferred_pool_id) + if preferred_pool_id_i in enabled_pool_set: + pool_candidates.append(preferred_pool_id_i) + + for pid in enabled_pools: + if pid not in pool_candidates: + pool_candidates.append(int(pid)) + + if not pool_candidates: + if bool(runtime.proxy_fail_closed): + raise ApiError( + code=ErrorCode.PROXY_REQUIRED, + message="需要代理，但未配置代理池", + status_code=502, + details={ + "reason": "no_proxy_pool_configured", + "host": host, + "url": url, + }, + ) + return None + + async def _select_in_pool(pool_id: int) -> ProxyUri | None: + if int(pool_id) <= 0: + return None + + if token_id is not None and int(token_id) > 0: + binding = await _load_token_binding(engine, token_id=int(token_id), pool_id=int(pool_id)) + if binding is not None: + primary_proxy_id, override_proxy_id, override_expires_at = binding + override_active = bool( + override_proxy_id is not None + and override_expires_at + and str(override_expires_at) > str(now_iso) + ) + + candidates: list[int] = [] + if override_active and override_proxy_id is not None: + candidates.append(int(override_proxy_id)) + candidates.append(int(primary_proxy_id)) + + for endpoint_id in candidates: + picked_by_binding = await _load_endpoint_in_pool( + engine, + pool_id=int(pool_id), + endpoint_id=int(endpoint_id), + now_iso=str(now_iso), + ) + if picked_by_binding is None: + continue + + eid, scheme, host_v, port, username, password_enc = picked_by_binding + return _proxy_uri_from_endpoint_row( + settings, + endpoint_id=int(eid), + pool_id=int(pool_id), + scheme=scheme, + host=host_v, + port=int(port), + username=username, + password_enc=password_enc, + ) + + picked = await _pick_endpoint_in_pool(engine, pool_id=int(pool_id), now_iso=now_iso) + if picked is None: + return None + + endpoint_id, scheme, host_v, port, username, password_enc = picked + return _proxy_uri_from_endpoint_row( + settings, + endpoint_id=int(endpoint_id), + pool_id=int(pool_id), + scheme=scheme, + host=host_v, + port=int(port), + username=username, + password_enc=password_enc, + ) + + for candidate_pool_id in pool_candidates: + picked = await _select_in_pool(int(candidate_pool_id)) + if picked is not None: + return picked + + if bool(runtime.proxy_fail_closed): + pool_stats: list[dict[str, Any]] = [] + next_available_at: str | None = None + total_any = 0 + eligible_any = 0 + any_next = False + for pid in pool_candidates: + stats = await _pool_health_stats(engine, pool_id=int(pid), now_iso=str(now_iso)) + total = int(stats.get("endpoints_total") or 0) + eligible = int(stats.get("endpoints_eligible") or 0) + total_any += max(0, total) + eligible_any += max(0, eligible) + nxt = stats.get("next_available_at") + if isinstance(nxt, str) and nxt.strip(): + any_next = True + if next_available_at is None or str(nxt) < str(next_available_at): + next_available_at = str(nxt) + pool_stats.append({"pool_id": int(pid), **stats}) + + reason = "no_healthy_proxy_available" + if total_any <= 0: + reason = "pool_has_no_endpoints" + elif eligible_any <= 0 and any_next and next_available_at: + reason = "all_endpoints_blacklisted" + + primary_pool_id = ( + int(preferred_pool_id) + if preferred_pool_id is not None and int(preferred_pool_id) > 0 + else int(pool_candidates[0]) + ) + + raise ApiError( + code=ErrorCode.PROXY_REQUIRED, + message="需要代理，但当前没有可用代理节点", + status_code=502, + details={ + "reason": reason, + "host": host, + "url": url, + "pool_id": primary_pool_id, + "attempted_pool_ids": [int(pid) for pid in pool_candidates], + "pools": pool_stats, + "endpoints_total": int(total_any), + "endpoints_eligible": int(eligible_any), + "next_available_at": next_available_at, + }, + ) + + return None diff --git a/backend/app/core/proxy_uri.py b/backend/app/core/proxy_uri.py new file mode 100644 index 0000000..d51a6b3 --- /dev/null +++ b/backend/app/core/proxy_uri.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from dataclasses import dataclass +from urllib.parse import unquote + + +@dataclass(frozen=True, slots=True) +class ProxyUriParts: + scheme: str + host: str + port: int + username: str | None + password: str | None + + +_ALLOWED_PROXY_SCHEMES = {"http", "https", "socks4", "socks5"} + + +def _strip_authority(rest: str) -> str: + for sep in ("/", "?", "#"): + idx = rest.find(sep) + if idx >= 0: + return rest[:idx] + return rest + + +def _parse_hostport(hostport: str) -> tuple[str, int]: + hostport = hostport.strip() + if not hostport: + raise ValueError("missing hostport") + + if hostport.startswith("["): + rb = hostport.find("]") + if rb <= 0: + raise ValueError("invalid ipv6 hostport") + host = hostport[1:rb].strip() + rest = hostport[rb + 1 :].strip() + if not rest.startswith(":"): + raise ValueError("missing port") + port_s = rest[1:].strip() + else: + if ":" not in hostport: + raise ValueError("missing port") + host, port_s = hostport.rsplit(":", 1) + host = host.strip() + port_s = port_s.strip() + + if not host: + raise ValueError("missing host") + + try: + port = int(port_s) + except Exception as exc: + raise ValueError("invalid port") from exc + if port <= 0 or port > 65535: + raise ValueError("invalid port") + + return host, port + + +def parse_proxy_uri(uri: str) -> ProxyUriParts: + uri = uri.strip() + if not uri: + raise ValueError("uri is required") + if "://" not in uri: + raise ValueError("invalid uri") + + scheme_raw, rest_raw = uri.split("://", 1) + scheme = scheme_raw.strip().lower() + if scheme not in _ALLOWED_PROXY_SCHEMES: + raise ValueError("unsupported scheme") + + authority = _strip_authority(rest_raw.strip()) + if not authority: + raise ValueError("invalid uri") + + username: str | None = None + password: str | None = None + + if "@" in authority: + userinfo, hostport = authority.rsplit("@", 1) + if ":" not in userinfo: + raise ValueError("invalid userinfo") + username, password = userinfo.split(":", 1) + username = unquote(username.strip()) + password = unquote(password) + if not username: + raise ValueError("invalid userinfo") + else: + hostport = authority + + host, port = _parse_hostport(hostport) + + return ProxyUriParts( + scheme=scheme, + host=host, + port=port, + username=username, + password=password, + ) diff --git a/backend/app/core/random_request_stats.py b/backend/app/core/random_request_stats.py new file mode 100644 index 0000000..4e64b64 --- /dev/null +++ b/backend/app/core/random_request_stats.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import asyncio +import time +from collections import deque +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class RandomRequestStatsSnapshot: + total_requests: int + total_ok: int + total_error: int + in_flight: int + window_seconds: int + last_window_requests: int + last_window_ok: int + last_window_error: int + last_window_success_rate: float + + +class RandomRequestStats: + def __init__(self, *, window_seconds: int = 60) -> None: + self._window_s = max(1, int(window_seconds)) + self._lock = asyncio.Lock() + + self._total_requests = 0 + self._total_ok = 0 + self._total_error = 0 + self._in_flight = 0 + + # (monotonic_seconds, ok) + self._events: deque[tuple[float, bool]] = deque() + + def _purge(self, *, now_m: float) -> None: + cutoff = float(now_m) - float(self._window_s) + while self._events: + t, _ok = self._events[0] + if float(t) >= cutoff: + break + self._events.popleft() + + async def on_begin(self) -> float: + now_m = float(time.monotonic()) + async with self._lock: + self._total_requests += 1 + self._in_flight += 1 + return now_m + + async def on_end(self, *, status_code: int) -> None: + now_m = float(time.monotonic()) + ok = 200 <= int(status_code) < 400 + + async with self._lock: + self._in_flight = max(0, int(self._in_flight) - 1) + if ok: + self._total_ok += 1 + else: + self._total_error += 1 + self._events.append((now_m, bool(ok))) + self._purge(now_m=now_m) + + async def snapshot(self) -> RandomRequestStatsSnapshot: + now_m = float(time.monotonic()) + async with self._lock: + self._purge(now_m=now_m) + last_total = int(len(self._events)) + last_ok = int(sum(1 for _t, ok in self._events if ok)) + last_error = int(last_total - last_ok) + rate = float(last_ok) / float(last_total) if last_total > 0 else 0.0 + + return RandomRequestStatsSnapshot( + total_requests=int(self._total_requests), + total_ok=int(self._total_ok), + total_error=int(self._total_error), + in_flight=int(self._in_flight), + window_seconds=int(self._window_s), + last_window_requests=int(last_total), + last_window_ok=int(last_ok), + last_window_error=int(last_error), + last_window_success_rate=float(rate), + ) + diff --git a/backend/app/core/redact.py b/backend/app/core/redact.py new file mode 100644 index 0000000..017cf8f --- /dev/null +++ b/backend/app/core/redact.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import re +from collections.abc import Mapping +from typing import Any + +REDACTED = "***" + +_SENSITIVE_KEY_PARTS = ( + "refresh", + "token", + "api_key", + "apikey", + "authorization", + "password", + "secret", + "cookie", +) + +_BEARER_RE = re.compile(r"(?i)\bBearer\s+([^\s]+)") +_REFRESH_QUERY_RE = re.compile(r"(?i)(refresh_token=)([^&\s]+)") +_URI_IN_TEXT_RE = re.compile(r"(?i)(?:https?|socks[45])://[^\s\"']+") + +_TRAILING_PUNCT = ".,);:]}" + + +def is_sensitive_key(key: str) -> bool: + key_l = key.lower() + return any(part in key_l for part in _SENSITIVE_KEY_PARTS) + + +def _redact_single_proxy_uri(text: str) -> str: + # Handles both normal and the common \"password contains @\" variant by treating the last '@' as separator. + m = re.match(r"^(?Phttps?|socks[45])://(?P.+)$", text, flags=re.IGNORECASE) + if not m: + return text + + scheme = m.group("scheme") + rest = m.group("rest") + if "@" not in rest or ":" not in rest: + return text + + userinfo, hostpart = rest.rsplit("@", 1) + if ":" not in userinfo: + return text + + username, _password = userinfo.split(":", 1) + return f"{scheme}://{username}:{REDACTED}@{hostpart}" + + +def redact_proxy_uri(text: str) -> str: + def _strip_trailing_punct(uri: str) -> tuple[str, str]: + suffix = "" + while uri and uri[-1] in _TRAILING_PUNCT: + suffix = uri[-1] + suffix + uri = uri[:-1] + return uri, suffix + + def _repl(m: re.Match[str]) -> str: + full = m.group(0) + core, suffix = _strip_trailing_punct(full) + return _redact_single_proxy_uri(core) + suffix + + return _URI_IN_TEXT_RE.sub(_repl, text) + + +def redact_text(text: str) -> str: + text = redact_proxy_uri(text) + text = _BEARER_RE.sub("Bearer " + REDACTED, text) + text = _REFRESH_QUERY_RE.sub(r"\1" + REDACTED, text) + return text + + +def redact_any(value: Any) -> Any: + if value is None: + return None + if isinstance(value, str): + return redact_text(value) + if isinstance(value, bytes): + try: + return redact_text(value.decode("utf-8", errors="replace")).encode("utf-8") + except Exception: + return value + if isinstance(value, Mapping): + out: dict[Any, Any] = {} + for k, v in value.items(): + if isinstance(k, str) and is_sensitive_key(k): + out[k] = REDACTED + else: + out[k] = redact_any(v) + return out + if isinstance(value, (list, tuple)): + seq = [redact_any(v) for v in value] + return type(value)(seq) if isinstance(value, tuple) else seq + return value diff --git a/backend/app/core/request_id.py b/backend/app/core/request_id.py new file mode 100644 index 0000000..29cd0f4 --- /dev/null +++ b/backend/app/core/request_id.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import secrets +from typing import Any, Mapping + +REQUEST_ID_HEADER = "X-Request-Id" + + +def new_request_id() -> str: + return "req_" + secrets.token_hex(8) + + +def get_request_id_from_headers(headers: Mapping[str, str] | None) -> str | None: + if not headers: + return None + value = headers.get(REQUEST_ID_HEADER) or headers.get(REQUEST_ID_HEADER.lower()) + if not value: + return None + value = value.strip() + return value if value else None + + +def set_request_id_on_state(request: Any, request_id: str) -> None: + state = getattr(request, "state", None) + if state is None: + class _State: + pass + + state = _State() + setattr(request, "state", state) + setattr(state, "request_id", request_id) + + +def set_request_id_header(response: Any, request_id: str) -> None: + headers = getattr(response, "headers", None) + if headers is None: + setattr(response, "headers", {}) + headers = response.headers + headers[REQUEST_ID_HEADER] = request_id + + +def get_or_create_request_id(request: Any) -> str: + state = getattr(request, "state", None) + if state is not None: + rid = getattr(state, "request_id", None) + if rid: + return str(rid) + headers = getattr(request, "headers", None) + rid = get_request_id_from_headers(headers) + return rid or new_request_id() + + +def build_request_id_middleware() -> Any | None: + try: + from starlette.middleware.base import BaseHTTPMiddleware + except Exception: + return None + + class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request, call_next): # type: ignore[no-untyped-def] + rid = get_or_create_request_id(request) + set_request_id_on_state(request, rid) + response = await call_next(request) + set_request_id_header(response, rid) + return response + + return RequestIdMiddleware diff --git a/backend/app/core/runtime_settings.py b/backend/app/core/runtime_settings.py new file mode 100644 index 0000000..9a59cb3 --- /dev/null +++ b/backend/app/core/runtime_settings.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.logging import get_logger +from app.core.time import iso_utc_ms +from app.db.models.runtime_settings import RuntimeSetting +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + +log = get_logger(__name__) + + +def _as_bool(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"true", "1", "yes", "y", "on"}: + return True + if v in {"false", "0", "no", "n", "off"}: + return False + return None + + +def _as_str_list(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + out: list[str] = [] + seen: set[str] = set() + for item in value: + if not isinstance(item, str): + continue + v = item.strip() + if not v or v in seen: + continue + seen.add(v) + out.append(v) + return out + + +@dataclass(frozen=True, slots=True) +class RuntimeConfig: + proxy_enabled: bool + proxy_fail_closed: bool + proxy_route_mode: str + proxy_allowlist_domains: list[str] + proxy_route_pools: dict[str, int] + proxy_default_pool_id: int | None + random_defaults: dict[str, Any] + hide_origin_url_in_public_json: bool + rate_limit: dict[str, Any] + + @classmethod + def defaults(cls) -> "RuntimeConfig": + return cls( + proxy_enabled=False, + proxy_fail_closed=True, + proxy_route_mode="pixiv_only", + proxy_allowlist_domains=[], + proxy_route_pools={}, + proxy_default_pool_id=None, + random_defaults={}, + hide_origin_url_in_public_json=True, + rate_limit={}, + ) + + +async def fetch_runtime_settings(engine: AsyncEngine) -> dict[str, Any]: + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + result = await session.execute(select(RuntimeSetting.key, RuntimeSetting.value_json)) + values: dict[str, Any] = {} + for key, value_json in result.all(): + try: + values[str(key)] = json.loads(value_json) + except Exception: + log.warning("runtime_settings_invalid_json key=%s", key) + return values + + return await with_sqlite_busy_retry(_op) + + +def runtime_config_from_values(values: dict[str, Any]) -> RuntimeConfig: + defaults = RuntimeConfig.defaults() + + proxy_enabled = _as_bool(values.get("proxy.enabled")) + proxy_fail_closed = _as_bool(values.get("proxy.fail_closed")) + + proxy_route_mode_raw = values.get("proxy.route_mode") + proxy_route_mode = defaults.proxy_route_mode + if isinstance(proxy_route_mode_raw, str): + candidate = proxy_route_mode_raw.strip().lower() + if candidate in {"pixiv_only", "all", "allowlist", "off"}: + proxy_route_mode = candidate + + proxy_allowlist_domains = _as_str_list(values.get("proxy.allowlist_domains")) or defaults.proxy_allowlist_domains + + proxy_route_pools_raw = values.get("proxy.route_pools") + proxy_route_pools: dict[str, int] = {} + if isinstance(proxy_route_pools_raw, dict): + for k, v in proxy_route_pools_raw.items(): + key = str(k or "").strip().lower().strip(".") + if not key or len(key) > 200: + continue + try: + pool_id = int(v) + except Exception: + continue + if pool_id <= 0: + continue + proxy_route_pools[key] = pool_id + + proxy_default_pool_id: int | None = None + default_pool_raw = values.get("proxy.default_pool_id") + if default_pool_raw is not None: + try: + candidate = int(default_pool_raw) + except Exception: + candidate = 0 + proxy_default_pool_id = candidate if candidate > 0 else None + + random_defaults_raw = values.get("random.defaults") + random_defaults = defaults.random_defaults + if isinstance(random_defaults_raw, dict): + random_defaults = dict(random_defaults_raw) + + hide_origin_url_raw = values.get("security.hide_origin_url_in_public_json") + hide_origin_url = _as_bool(hide_origin_url_raw) + + rate_limit: dict[str, Any] = {} + for key, value in values.items(): + if key.startswith("rate_limit."): + rate_limit[key.removeprefix("rate_limit.")] = value + + return RuntimeConfig( + proxy_enabled=proxy_enabled if proxy_enabled is not None else defaults.proxy_enabled, + proxy_fail_closed=proxy_fail_closed if proxy_fail_closed is not None else defaults.proxy_fail_closed, + proxy_route_mode=proxy_route_mode, + proxy_allowlist_domains=proxy_allowlist_domains, + proxy_route_pools=proxy_route_pools, + proxy_default_pool_id=proxy_default_pool_id, + random_defaults=random_defaults, + hide_origin_url_in_public_json=hide_origin_url + if hide_origin_url is not None + else defaults.hide_origin_url_in_public_json, + rate_limit=rate_limit, + ) + + +async def load_runtime_config(engine: AsyncEngine) -> RuntimeConfig: + values = await fetch_runtime_settings(engine) + return runtime_config_from_values(values) + + +async def set_runtime_setting( + engine: AsyncEngine, + *, + key: str, + value: Any, + description: str | None = None, + updated_by: str | None = None, +) -> None: + key = (key or "").strip() + if not key: + raise ValueError("key is required") + + value_json = json.dumps(value, separators=(",", ":"), sort_keys=True, ensure_ascii=False) + now = iso_utc_ms() + + Session = create_sessionmaker(engine) + + async def _op() -> None: + async with Session() as session: + stmt = sqlite_insert(RuntimeSetting).values( + key=key, + value_json=value_json, + description=description, + updated_at=now, + updated_by=updated_by, + ) + set_values: dict[str, Any] = { + "value_json": value_json, + "updated_at": now, + "updated_by": updated_by, + } + if description is not None: + set_values["description"] = description + stmt = stmt.on_conflict_do_update( + index_elements=[RuntimeSetting.key], + set_=set_values, + ) + await session.execute(stmt) + await session.commit() + + await with_sqlite_busy_retry(_op) diff --git a/backend/app/core/security.py b/backend/app/core/security.py new file mode 100644 index 0000000..012ebdd --- /dev/null +++ b/backend/app/core/security.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import time +from collections.abc import Mapping +from typing import Any + +from app.core.errors import ApiError, ErrorCode + +JWT_ALG_HS256 = "HS256" +JWT_TYP = "JWT" + + +def _b64url_encode(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + + +def _b64url_decode(data: str) -> bytes: + data = data.strip() + padding = "=" * (-len(data) % 4) + return base64.urlsafe_b64decode(data + padding) + + +def create_jwt( + *, + secret_key: str, + subject: str, + ttl_s: int = 3600, + extra_claims: Mapping[str, Any] | None = None, + now_s: int | None = None, +) -> str: + secret_key = secret_key.strip() + if not secret_key: + raise ValueError("SECRET_KEY is required") + + now_i = int(now_s if now_s is not None else time.time()) + + header = {"alg": JWT_ALG_HS256, "typ": JWT_TYP} + payload: dict[str, Any] = {"sub": subject, "iat": now_i, "exp": now_i + ttl_s} + if extra_claims: + payload.update(dict(extra_claims)) + + header_b64 = _b64url_encode(json.dumps(header, separators=(",", ":"), sort_keys=True).encode("utf-8")) + payload_b64 = _b64url_encode(json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")) + signing_input = f"{header_b64}.{payload_b64}".encode("ascii") + signature = hmac.new(secret_key.encode("utf-8"), signing_input, hashlib.sha256).digest() + sig_b64 = _b64url_encode(signature) + return f"{header_b64}.{payload_b64}.{sig_b64}" + + +def decode_jwt(token: str, *, secret_key: str, leeway_s: int = 0, now_s: int | None = None) -> dict[str, Any]: + secret_key = secret_key.strip() + if not secret_key: + raise ValueError("SECRET_KEY is required") + + token = token.strip() + parts = token.split(".") + if len(parts) != 3: + raise ValueError("Invalid token format") + + header_raw = _b64url_decode(parts[0]) + payload_raw = _b64url_decode(parts[1]) + signature_raw = _b64url_decode(parts[2]) + + try: + header = json.loads(header_raw) + payload = json.loads(payload_raw) + except Exception as exc: + raise ValueError("Invalid token JSON") from exc + + if not isinstance(header, dict) or not isinstance(payload, dict): + raise ValueError("Invalid token JSON") + + if header.get("alg") != JWT_ALG_HS256 or header.get("typ") != JWT_TYP: + raise ValueError("Unsupported token") + + signing_input = f"{parts[0]}.{parts[1]}".encode("ascii") + expected_sig = hmac.new(secret_key.encode("utf-8"), signing_input, hashlib.sha256).digest() + if not hmac.compare_digest(signature_raw, expected_sig): + raise ValueError("Invalid token signature") + + now_i = int(now_s if now_s is not None else time.time()) + + exp = payload.get("exp") + if exp is not None: + try: + exp_i = int(exp) + except Exception as exc: + raise ValueError("Invalid exp") from exc + if now_i > exp_i + leeway_s: + raise ValueError("Token expired") + + iat = payload.get("iat") + if iat is not None: + try: + iat_i = int(iat) + except Exception as exc: + raise ValueError("Invalid iat") from exc + if iat_i > now_i + leeway_s: + raise ValueError("Token issued in the future") + + return payload + + +def _authorization_from_headers(headers: Mapping[str, str] | None) -> str | None: + if not headers: + return None + return headers.get("Authorization") or headers.get("authorization") + + +def parse_bearer_token(authorization: str | None) -> str | None: + if not authorization: + return None + parts = authorization.strip().split(None, 1) + if len(parts) != 2: + return None + scheme, token = parts + if scheme.lower() != "bearer": + return None + token = token.strip() + return token if token else None + + +def require_admin( + headers: Mapping[str, str] | None, + *, + secret_key: str, + admin_username: str, +) -> dict[str, Any]: + token = parse_bearer_token(_authorization_from_headers(headers)) + if not token: + raise ApiError(code=ErrorCode.UNAUTHORIZED, message="Missing admin token", status_code=401) + + try: + claims = decode_jwt(token, secret_key=secret_key) + except Exception as exc: + raise ApiError(code=ErrorCode.UNAUTHORIZED, message="Invalid admin token", status_code=401) from exc + + subject = str(claims.get("sub") or "") + if subject != admin_username: + raise ApiError(code=ErrorCode.FORBIDDEN, message="Forbidden", status_code=403) + + return claims + diff --git a/backend/app/core/time.py b/backend/app/core/time.py new file mode 100644 index 0000000..93a4033 --- /dev/null +++ b/backend/app/core/time.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from datetime import datetime, timezone + + +def iso_utc_ms(dt: datetime | None = None) -> str: + dt = dt or datetime.now(timezone.utc) + dt = dt.astimezone(timezone.utc) + ms = dt.microsecond // 1000 + return dt.strftime("%Y-%m-%dT%H:%M:%S.") + f"{ms:03d}Z" + diff --git a/backend/app/db/authors_list.py b/backend/app/db/authors_list.py new file mode 100644 index 0000000..2c00e83 --- /dev/null +++ b/backend/app/db/authors_list.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import sqlalchemy as sa +from sqlalchemy import func, select +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models.images import Image +from app.db.sqlite_utils import sqlite_fts_phrase_query, sqlite_table_exists + + +@dataclass(frozen=True, slots=True) +class AuthorListItem: + user_id: int + user_name: str | None + count_images: int + + +async def list_authors( + session: AsyncSession, + *, + limit: int, + cursor: int | None = None, + q: str | None = None, +) -> tuple[list[AuthorListItem], int | None]: + limit_i = int(limit) + if limit_i < 1: + raise ValueError("limit must be >= 1") + + cursor_i: int | None = None + if cursor is not None: + cursor_i = int(cursor) + if cursor_i <= 0: + raise ValueError("cursor must be > 0") + + q_norm = (q or "").strip() + use_fts = False + fts_q = "" + if q_norm and len(q_norm) >= 3: + use_fts = await sqlite_table_exists(session, name="authors_fts") + if use_fts: + fts_q = sqlite_fts_phrase_query(q_norm) + + def _build_stmt(*, use_fts_filter: bool) -> sa.Select: + clauses: list[object] = [Image.status == 1, Image.user_id.is_not(None)] + + if cursor_i is not None: + clauses.append(Image.user_id > cursor_i) + + if q_norm: + if use_fts_filter: + fts_user_ids = ( + sa.text("SELECT rowid AS user_id FROM authors_fts WHERE authors_fts MATCH :q") + .bindparams(sa.bindparam("q", fts_q)) + .columns(user_id=sa.Integer) + ) + fts_user_ids_sq = fts_user_ids.subquery() + clauses.append(Image.user_id.in_(sa.select(fts_user_ids_sq.c.user_id))) + else: + clauses.append(Image.user_name.like(f"%{q_norm}%")) + + return ( + select( + Image.user_id, + func.max(Image.user_name).label("user_name"), + func.count(Image.id).label("count_images"), + ) + .where(*clauses) + .group_by(Image.user_id) + .order_by(Image.user_id.asc()) + .limit(limit_i + 1) + ) + + stmt = _build_stmt(use_fts_filter=use_fts) + try: + rows = (await session.execute(stmt)).all() + except DBAPIError: + if not use_fts: + raise + stmt = _build_stmt(use_fts_filter=False) + rows = (await session.execute(stmt)).all() + next_cursor: int | None = None + + if len(rows) > limit_i: + rows = rows[:limit_i] + next_cursor = int(rows[-1][0] or 0) + + items = [ + AuthorListItem( + user_id=int(row[0]), + user_name=str(row[1]) if row[1] is not None else None, + count_images=int(row[2] or 0), + ) + for row in rows + ] + + return items, next_cursor diff --git a/backend/app/db/engine.py b/backend/app/db/engine.py new file mode 100644 index 0000000..b663243 --- /dev/null +++ b/backend/app/db/engine.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import os +from typing import Any + +from sqlalchemy import event +from sqlalchemy.engine.url import make_url +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine + +SQLITE_BUSY_TIMEOUT_MS = 30_000 +SQLITE_POOL_SIZE = 5 +SQLITE_MAX_OVERFLOW = 0 + + +def apply_sqlite_pragmas(dbapi_connection: Any) -> None: + try: + busy_timeout_ms = int((os.environ.get("SQLITE_BUSY_TIMEOUT_MS") or str(SQLITE_BUSY_TIMEOUT_MS)).strip() or SQLITE_BUSY_TIMEOUT_MS) + except Exception: + busy_timeout_ms = int(SQLITE_BUSY_TIMEOUT_MS) + busy_timeout_ms = max(1000, min(int(busy_timeout_ms), 5 * 60_000)) + + cursor = dbapi_connection.cursor() + try: + cursor.execute("PRAGMA foreign_keys = ON") + cursor.execute("PRAGMA journal_mode = WAL") + cursor.fetchone() + cursor.execute("PRAGMA synchronous = NORMAL") + cursor.execute("PRAGMA temp_store = MEMORY") + cursor.execute(f"PRAGMA busy_timeout = {busy_timeout_ms}") + finally: + cursor.close() + + +def _is_sqlite_file_url(database_url: str) -> bool: + try: + url = make_url(database_url) + except Exception: + return False + if (url.get_backend_name() or "").lower() != "sqlite": + return False + db = str(url.database or "").strip() + return bool(db and db != ":memory:") + + +def create_engine(database_url: str) -> AsyncEngine: + kwargs: dict[str, Any] = {} + if database_url.lower().startswith("sqlite"): + try: + busy_timeout_ms = int((os.environ.get("SQLITE_BUSY_TIMEOUT_MS") or str(SQLITE_BUSY_TIMEOUT_MS)).strip() or SQLITE_BUSY_TIMEOUT_MS) + except Exception: + busy_timeout_ms = int(SQLITE_BUSY_TIMEOUT_MS) + busy_timeout_ms = max(1000, min(int(busy_timeout_ms), 5 * 60_000)) + + kwargs["connect_args"] = {"timeout": float(busy_timeout_ms) / 1000.0} + if _is_sqlite_file_url(database_url): + # 限制单进程内同时打开的 SQLite 连接数，减少并发写导致的 "database is locked"。 + kwargs.update( + { + "pool_size": int(SQLITE_POOL_SIZE), + "max_overflow": int(SQLITE_MAX_OVERFLOW), + } + ) + + engine = create_async_engine(database_url, **kwargs) + + if database_url.lower().startswith("sqlite"): + def _on_connect(dbapi_connection: Any, _record: Any) -> None: + apply_sqlite_pragmas(dbapi_connection) + + event.listen(engine.sync_engine, "connect", _on_connect) + + return engine diff --git a/backend/app/db/images_get.py b/backend/app/db/images_get.py new file mode 100644 index 0000000..6d56abf --- /dev/null +++ b/backend/app/db/images_get.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models.images import Image + + +async def get_image_by_id(session: AsyncSession, *, image_id: int) -> Image | None: + stmt = select(Image).where(Image.id == int(image_id), Image.status == 1).limit(1) + return (await session.execute(stmt)).scalars().first() + diff --git a/backend/app/db/images_get_by_illust.py b/backend/app/db/images_get_by_illust.py new file mode 100644 index 0000000..c272215 --- /dev/null +++ b/backend/app/db/images_get_by_illust.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models.images import Image + + +async def get_image_by_illust_page( + session: AsyncSession, + *, + illust_id: int, + page_index: int, +) -> Image | None: + stmt = ( + select(Image) + .where( + Image.status == 1, + Image.illust_id == int(illust_id), + Image.page_index == int(page_index), + ) + .limit(1) + ) + return (await session.execute(stmt)).scalars().first() + diff --git a/backend/app/db/images_list.py b/backend/app/db/images_list.py new file mode 100644 index 0000000..4cd38e3 --- /dev/null +++ b/backend/app/db/images_list.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models.images import Image +from app.db.random_pick import ( + _excluded_tags_where_clause, + _included_tags_where_clause, + _orientation_where_clause, + _r18_where_clause, +) + + +async def list_images( + session: AsyncSession, + *, + limit: int, + cursor: int | None = None, + r18: int = 0, + r18_strict: bool = True, + orientation: int | None = None, + ai_type: int | None = None, + min_width: int = 0, + min_height: int = 0, + min_pixels: int = 0, + included_tags: Sequence[str] | None = None, + excluded_tags: Sequence[str] | None = None, + user_id: int | None = None, + illust_id: int | None = None, + created_from: str | None = None, + created_to: str | None = None, +) -> tuple[list[Image], int | None]: + limit_i = int(limit) + if limit_i < 1: + raise ValueError("limit must be >= 1") + + r18_clause = _r18_where_clause(r18=r18, r18_strict=r18_strict) + orientation_clause = _orientation_where_clause(orientation=orientation) + included_tags_clause = _included_tags_where_clause(tag_names=included_tags or []) + excluded_tags_clause = _excluded_tags_where_clause(tag_names=excluded_tags or []) + + min_width_i = int(min_width) + min_height_i = int(min_height) + min_pixels_i = int(min_pixels) + if min_width_i < 0 or min_height_i < 0 or min_pixels_i < 0: + raise ValueError("min_* must be >= 0") + + clauses: list[object] = [Image.status == 1] + if cursor is not None: + clauses.append(Image.id < int(cursor)) + if r18_clause is not None: + clauses.append(r18_clause) + if orientation_clause is not None: + clauses.append(orientation_clause) + if ai_type is not None: + ai_type_i = int(ai_type) + if ai_type_i not in {0, 1}: + raise ValueError("ai_type must be 0, 1, or None") + clauses.append(Image.ai_type == ai_type_i) + if included_tags_clause is not None: + clauses.append(included_tags_clause) + if excluded_tags_clause is not None: + clauses.append(excluded_tags_clause) + if min_width_i > 0: + clauses.append(Image.width >= min_width_i) + if min_height_i > 0: + clauses.append(Image.height >= min_height_i) + if min_pixels_i > 0: + clauses.append((Image.width * Image.height) >= min_pixels_i) + if user_id is not None: + clauses.append(Image.user_id == int(user_id)) + if illust_id is not None: + clauses.append(Image.illust_id == int(illust_id)) + if created_from is not None: + clauses.append(Image.created_at_pixiv >= str(created_from)) + if created_to is not None: + clauses.append(Image.created_at_pixiv <= str(created_to)) + + stmt = select(Image).where(*clauses).order_by(Image.id.desc()).limit(limit_i + 1) + images = (await session.execute(stmt)).scalars().all() + + next_cursor: int | None = None + if len(images) > limit_i: + images = images[:limit_i] + next_cursor = int(images[-1].id) + + return images, next_cursor + diff --git a/backend/app/db/images_mark.py b/backend/app/db/images_mark.py new file mode 100644 index 0000000..35fe3fa --- /dev/null +++ b/backend/app/db/images_mark.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from sqlalchemy import update +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.redact import redact_text +from app.db.models.images import Image +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + + +def _truncate(text: str, *, max_len: int = 500) -> str: + if len(text) <= max_len: + return text + return text[: max_len - 3] + "..." + + +async def mark_image_failure( + engine: AsyncEngine, + *, + image_id: int, + now: str, + error_code: str, + error_message: str, +) -> None: + Session = create_sessionmaker(engine) + msg = _truncate(redact_text(error_message or "")) + + async def _op() -> None: + async with Session() as session: + await session.execute( + update(Image) + .where(Image.id == int(image_id)) + .values( + fail_count=Image.fail_count + 1, + last_fail_at=str(now), + last_error_code=str(error_code), + last_error_msg=str(msg), + ) + ) + await session.commit() + + await with_sqlite_busy_retry(_op) + + +async def mark_image_ok(engine: AsyncEngine, *, image_id: int, now: str) -> None: + Session = create_sessionmaker(engine) + + async def _op() -> None: + async with Session() as session: + await session.execute( + update(Image) + .where(Image.id == int(image_id)) + .values( + last_ok_at=str(now), + last_error_code=None, + last_error_msg=None, + ) + ) + await session.commit() + + await with_sqlite_busy_retry(_op) + diff --git a/backend/app/db/images_upsert.py b/backend/app/db/images_upsert.py new file mode 100644 index 0000000..022ff51 --- /dev/null +++ b/backend/app/db/images_upsert.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models.images import Image + + +async def upsert_image_by_illust_page( + session: AsyncSession, + *, + illust_id: int, + page_index: int, + ext: str, + original_url: str, + proxy_path: str, + random_key: float, + created_import_id: int | None, +) -> int: + stmt = sqlite_insert(Image).values( + illust_id=illust_id, + page_index=page_index, + ext=ext, + original_url=original_url, + proxy_path=proxy_path, + random_key=random_key, + created_import_id=created_import_id, + ) + + now_expr = sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))") + stmt = stmt.on_conflict_do_update( + index_elements=["illust_id", "page_index"], + set_={ + "ext": stmt.excluded.ext, + "original_url": stmt.excluded.original_url, + "proxy_path": stmt.excluded.proxy_path, + "created_import_id": stmt.excluded.created_import_id, + "updated_at": now_expr, + }, + ).returning(Image.id) + + result = await session.execute(stmt) + return int(result.scalar_one()) diff --git a/backend/app/db/models/admin_audit.py b/backend/app/db/models/admin_audit.py new file mode 100644 index 0000000..8cff092 --- /dev/null +++ b/backend/app/db/models/admin_audit.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class AdminAudit(Base): + __tablename__ = "admin_audit" + __table_args__ = ( + sa.Index("idx_admin_audit_created_at", "created_at"), + sa.Index("idx_admin_audit_action", "action"), + sa.Index("idx_admin_audit_resource", "resource"), + sa.Index("idx_admin_audit_record_id", "record_id"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + actor: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + action: Mapped[str] = mapped_column(sa.Text(), nullable=False) + resource: Mapped[str] = mapped_column(sa.Text(), nullable=False) + record_id: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + request_id: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + ip: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + user_agent: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + detail_json: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + diff --git a/backend/app/db/models/api_keys.py b/backend/app/db/models/api_keys.py new file mode 100644 index 0000000..a712c94 --- /dev/null +++ b/backend/app/db/models/api_keys.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class ApiKey(Base): + __tablename__ = "api_keys" + __table_args__ = ( + sa.UniqueConstraint("name", name="uq_api_keys_name"), + sa.CheckConstraint("enabled IN (0,1)", name="ck_api_keys_enabled"), + sa.Index("idx_api_keys_enabled", "enabled"), + sa.Index("idx_api_keys_created_at", "created_at"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + + name: Mapped[str] = mapped_column(sa.Text(), nullable=False) + description: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + + key_hash: Mapped[str] = mapped_column(sa.Text(), nullable=False) + hint: Mapped[str] = mapped_column(sa.Text(), nullable=False, server_default=sa.text("''")) + + enabled: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("1")) + last_used_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + diff --git a/backend/app/db/models/base.py b/backend/app/db/models/base.py new file mode 100644 index 0000000..2e5e598 --- /dev/null +++ b/backend/app/db/models/base.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from sqlalchemy.orm import DeclarativeBase + + +class Base(DeclarativeBase): + pass + + +# Import all model modules so Base.metadata is fully populated for create_all(). +# This keeps tests and one-off scripts consistent, regardless of import order. +from app.db.models import admin_audit as _admin_audit # noqa: F401,E402 +from app.db.models import api_keys as _api_keys # noqa: F401,E402 +from app.db.models import hydration_runs as _hydration_runs # noqa: F401,E402 +from app.db.models import image_tags as _image_tags # noqa: F401,E402 +from app.db.models import images as _images # noqa: F401,E402 +from app.db.models import imports as _imports # noqa: F401,E402 +from app.db.models import jobs as _jobs # noqa: F401,E402 +from app.db.models import pixiv_tokens as _pixiv_tokens # noqa: F401,E402 +from app.db.models import proxy_endpoints as _proxy_endpoints # noqa: F401,E402 +from app.db.models import proxy_pool_endpoints as _proxy_pool_endpoints # noqa: F401,E402 +from app.db.models import proxy_pools as _proxy_pools # noqa: F401,E402 +from app.db.models import request_logs as _request_logs # noqa: F401,E402 +from app.db.models import runtime_settings as _runtime_settings # noqa: F401,E402 +from app.db.models import tags as _tags # noqa: F401,E402 +from app.db.models import token_proxy_bindings as _token_proxy_bindings # noqa: F401,E402 diff --git a/backend/app/db/models/hydration_runs.py b/backend/app/db/models/hydration_runs.py new file mode 100644 index 0000000..bf9d00c --- /dev/null +++ b/backend/app/db/models/hydration_runs.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class HydrationRun(Base): + __tablename__ = "hydration_runs" + __table_args__ = ( + sa.CheckConstraint( + "status IN ('pending','running','paused','canceled','completed','failed')", + name="ck_hr_status", + ), + sa.Index("idx_hr_status_updated", "status", "updated_at"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + + type: Mapped[str] = mapped_column(sa.Text(), nullable=False) + status: Mapped[str] = mapped_column(sa.Text(), nullable=False) + criteria_json: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + cursor_json: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + total: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + processed: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + success: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + failed: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + started_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + finished_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_error: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + diff --git a/backend/app/db/models/image_tags.py b/backend/app/db/models/image_tags.py new file mode 100644 index 0000000..3bb7861 --- /dev/null +++ b/backend/app/db/models/image_tags.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class ImageTag(Base): + __tablename__ = "image_tags" + __table_args__ = (sa.Index("idx_image_tags_tag_image", "tag_id", "image_id"),) + + image_id: Mapped[int] = mapped_column( + sa.Integer(), + sa.ForeignKey("images.id", ondelete="CASCADE"), + primary_key=True, + ) + tag_id: Mapped[int] = mapped_column( + sa.Integer(), + sa.ForeignKey("tags.id", ondelete="CASCADE"), + primary_key=True, + ) + diff --git a/backend/app/db/models/images.py b/backend/app/db/models/images.py new file mode 100644 index 0000000..2e7602a --- /dev/null +++ b/backend/app/db/models/images.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class Image(Base): + __tablename__ = "images" + __table_args__ = ( + sa.UniqueConstraint("illust_id", "page_index", name="uq_images_illust_page"), + sa.CheckConstraint("status IN (1,2,3,4)", name="ck_images_status"), + sa.CheckConstraint("random_key >= 0.0 AND random_key < 1.0", name="ck_images_random_key"), + sa.Index( + "idx_images_filter", + "status", + "x_restrict", + "orientation", + "width", + "height", + "random_key", + ), + sa.Index("idx_images_illust_type_random", "status", "illust_type", "random_key"), + sa.Index("idx_images_user_random", "status", "user_id", "random_key"), + sa.Index("idx_images_created_at_pixiv", "created_at_pixiv"), + sa.Index("idx_images_created_import_id", "created_import_id"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + + illust_id: Mapped[int] = mapped_column(sa.Integer(), nullable=False) + page_index: Mapped[int] = mapped_column(sa.Integer(), nullable=False) + ext: Mapped[str] = mapped_column(sa.Text(), nullable=False) + + original_url: Mapped[str] = mapped_column(sa.Text(), nullable=False) + proxy_path: Mapped[str] = mapped_column(sa.Text(), nullable=False) + random_key: Mapped[float] = mapped_column(sa.Float(), nullable=False) + + width: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + height: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + aspect_ratio: Mapped[float | None] = mapped_column(sa.Float(), nullable=True) + orientation: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + x_restrict: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + ai_type: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + illust_type: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + user_id: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + user_name: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + title: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + created_at_pixiv: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + + bookmark_count: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + view_count: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + comment_count: Mapped[int | None] = mapped_column(sa.Integer(), nullable=True) + + status: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("1")) + fail_count: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + last_fail_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_ok_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_error_code: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_error_msg: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + + created_import_id: Mapped[int | None] = mapped_column( + sa.Integer(), + sa.ForeignKey("imports.id", ondelete="SET NULL"), + nullable=True, + ) + + added_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) diff --git a/backend/app/db/models/imports.py b/backend/app/db/models/imports.py new file mode 100644 index 0000000..4d6c15f --- /dev/null +++ b/backend/app/db/models/imports.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class Import(Base): + __tablename__ = "imports" + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + created_by: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + source: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + + total: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + accepted: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + success: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + failed: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + + detail_json: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + diff --git a/backend/app/db/models/jobs.py b/backend/app/db/models/jobs.py new file mode 100644 index 0000000..23aeaaa --- /dev/null +++ b/backend/app/db/models/jobs.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class JobRow(Base): + __tablename__ = "jobs" + __table_args__ = ( + sa.CheckConstraint( + "status IN ('pending','running','paused','canceled','completed','failed','dlq')", + name="ck_jobs_status", + ), + sa.Index("idx_jobs_status_priority", "status", "priority", "id"), + sa.Index("idx_jobs_run_after", "run_after"), + sa.Index("idx_jobs_ref", "ref_type", "ref_id"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + + type: Mapped[str] = mapped_column(sa.Text(), nullable=False) + status: Mapped[str] = mapped_column(sa.Text(), nullable=False) + priority: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + run_after: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + + attempt: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + max_attempts: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("3")) + + payload_json: Mapped[str] = mapped_column(sa.Text(), nullable=False) + last_error: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + + locked_by: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + locked_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + + ref_type: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + ref_id: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + diff --git a/backend/app/db/models/pixiv_tokens.py b/backend/app/db/models/pixiv_tokens.py new file mode 100644 index 0000000..8ee0189 --- /dev/null +++ b/backend/app/db/models/pixiv_tokens.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class PixivToken(Base): + __tablename__ = "pixiv_tokens" + __table_args__ = ( + sa.CheckConstraint("enabled IN (0,1)", name="ck_pixiv_tokens_enabled"), + sa.Index("idx_pixiv_tokens_enabled", "enabled"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + + label: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + enabled: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("1")) + + refresh_token_enc: Mapped[str] = mapped_column(sa.Text(), nullable=False) + refresh_token_masked: Mapped[str] = mapped_column(sa.Text(), nullable=False) + weight: Mapped[float] = mapped_column(sa.Float(), nullable=False, server_default=sa.text("1.0")) + + error_count: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + backoff_until: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_ok_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_fail_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_error_code: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_error_msg: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + diff --git a/backend/app/db/models/proxy_endpoints.py b/backend/app/db/models/proxy_endpoints.py new file mode 100644 index 0000000..6c92dd0 --- /dev/null +++ b/backend/app/db/models/proxy_endpoints.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class ProxyEndpoint(Base): + __tablename__ = "proxy_endpoints" + __table_args__ = ( + sa.UniqueConstraint("scheme", "host", "port", "username", name="uq_proxy_identity"), + sa.CheckConstraint("enabled IN (0,1)", name="ck_proxy_enabled"), + sa.Index("idx_proxy_endpoints_enabled", "enabled"), + sa.Index("idx_proxy_endpoints_source", "source"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + + scheme: Mapped[str] = mapped_column(sa.Text(), nullable=False) + host: Mapped[str] = mapped_column(sa.Text(), nullable=False) + port: Mapped[int] = mapped_column(sa.Integer(), nullable=False) + username: Mapped[str] = mapped_column(sa.Text(), nullable=False, server_default=sa.text("''")) + password_enc: Mapped[str] = mapped_column(sa.Text(), nullable=False, server_default=sa.text("''")) + + enabled: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("1")) + source: Mapped[str] = mapped_column(sa.Text(), nullable=False, server_default=sa.text("'manual'")) + source_ref: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + + last_latency_ms: Mapped[float | None] = mapped_column(sa.Float(), nullable=True) + last_ok_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_fail_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + success_count: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + failure_count: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("0")) + blacklisted_until: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + last_error: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + diff --git a/backend/app/db/models/proxy_pool_endpoints.py b/backend/app/db/models/proxy_pool_endpoints.py new file mode 100644 index 0000000..385387b --- /dev/null +++ b/backend/app/db/models/proxy_pool_endpoints.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class ProxyPoolEndpoint(Base): + __tablename__ = "proxy_pool_endpoints" + __table_args__ = ( + sa.Index("idx_ppe_pool_enabled", "pool_id", "enabled"), + sa.Index("idx_ppe_endpoint_pool", "endpoint_id", "pool_id"), + sa.CheckConstraint("enabled IN (0,1)", name="ck_ppe_enabled"), + ) + + pool_id: Mapped[int] = mapped_column( + sa.Integer(), + sa.ForeignKey("proxy_pools.id", ondelete="CASCADE"), + primary_key=True, + ) + endpoint_id: Mapped[int] = mapped_column( + sa.Integer(), + sa.ForeignKey("proxy_endpoints.id", ondelete="CASCADE"), + primary_key=True, + ) + + enabled: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("1")) + weight: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("1")) + + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + diff --git a/backend/app/db/models/proxy_pools.py b/backend/app/db/models/proxy_pools.py new file mode 100644 index 0000000..aedf293 --- /dev/null +++ b/backend/app/db/models/proxy_pools.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class ProxyPool(Base): + __tablename__ = "proxy_pools" + __table_args__ = ( + sa.UniqueConstraint("name", name="uq_proxy_pools_name"), + sa.CheckConstraint("enabled IN (0,1)", name="ck_proxy_pools_enabled"), + sa.Index("idx_proxy_pools_enabled", "enabled"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + + name: Mapped[str] = mapped_column(sa.Text(), nullable=False) + description: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + enabled: Mapped[int] = mapped_column(sa.Integer(), nullable=False, server_default=sa.text("1")) + diff --git a/backend/app/db/models/request_logs.py b/backend/app/db/models/request_logs.py new file mode 100644 index 0000000..eda9afe --- /dev/null +++ b/backend/app/db/models/request_logs.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class RequestLog(Base): + __tablename__ = "request_logs" + __table_args__ = ( + sa.Index("idx_request_logs_created_at", "created_at"), + sa.Index("idx_request_logs_route", "route"), + sa.Index("idx_request_logs_status", "status"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + request_id: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + method: Mapped[str] = mapped_column(sa.Text(), nullable=False) + route: Mapped[str] = mapped_column(sa.Text(), nullable=False) + status: Mapped[int] = mapped_column(sa.Integer(), nullable=False) + duration_ms: Mapped[int] = mapped_column(sa.Integer(), nullable=False) + ip: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + user_agent: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + sample_rate: Mapped[float | None] = mapped_column(sa.Float(), nullable=True) + diff --git a/backend/app/db/models/runtime_settings.py b/backend/app/db/models/runtime_settings.py new file mode 100644 index 0000000..e3dcb09 --- /dev/null +++ b/backend/app/db/models/runtime_settings.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class RuntimeSetting(Base): + __tablename__ = "runtime_settings" + + key: Mapped[str] = mapped_column(sa.Text(), primary_key=True) + value_json: Mapped[str] = mapped_column(sa.Text(), nullable=False) + description: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_by: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + diff --git a/backend/app/db/models/tags.py b/backend/app/db/models/tags.py new file mode 100644 index 0000000..3781dc3 --- /dev/null +++ b/backend/app/db/models/tags.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class Tag(Base): + __tablename__ = "tags" + __table_args__ = (sa.UniqueConstraint("name", name="uq_tags_name"),) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(sa.Text(), nullable=False) + translated_name: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + diff --git a/backend/app/db/models/token_proxy_bindings.py b/backend/app/db/models/token_proxy_bindings.py new file mode 100644 index 0000000..27fcc83 --- /dev/null +++ b/backend/app/db/models/token_proxy_bindings.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.models.base import Base + + +class TokenProxyBinding(Base): + __tablename__ = "token_proxy_bindings" + __table_args__ = ( + sa.UniqueConstraint("token_id", "pool_id", name="uq_token_pool"), + sa.Index("idx_tpb_pool", "pool_id"), + sa.Index("idx_tpb_primary", "primary_proxy_id"), + sa.Index("idx_tpb_override", "override_proxy_id"), + ) + + id: Mapped[int] = mapped_column(sa.Integer(), primary_key=True, autoincrement=True) + created_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + updated_at: Mapped[str] = mapped_column( + sa.Text(), + nullable=False, + server_default=sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))"), + ) + + token_id: Mapped[int] = mapped_column( + sa.Integer(), + sa.ForeignKey("pixiv_tokens.id", ondelete="CASCADE"), + nullable=False, + ) + pool_id: Mapped[int] = mapped_column( + sa.Integer(), + sa.ForeignKey("proxy_pools.id", ondelete="CASCADE"), + nullable=False, + ) + primary_proxy_id: Mapped[int] = mapped_column( + sa.Integer(), + sa.ForeignKey("proxy_endpoints.id", ondelete="RESTRICT"), + nullable=False, + ) + override_proxy_id: Mapped[int | None] = mapped_column( + sa.Integer(), + sa.ForeignKey("proxy_endpoints.id", ondelete="SET NULL"), + nullable=True, + ) + override_expires_at: Mapped[str | None] = mapped_column(sa.Text(), nullable=True) + diff --git a/backend/app/db/random_pick.py b/backend/app/db/random_pick.py new file mode 100644 index 0000000..0394fe2 --- /dev/null +++ b/backend/app/db/random_pick.py @@ -0,0 +1,366 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from sqlalchemy import distinct, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag + + +def _r18_where_clause(*, r18: int, r18_strict: bool) -> object | None: + r18_i = int(r18) + if r18_i not in {0, 1, 2}: + raise ValueError("r18 must be 0, 1, or 2") + + if r18_i == 2: + return None + + if r18_i == 1: + return Image.x_restrict == 1 + + if bool(r18_strict): + return Image.x_restrict == 0 + + return (Image.x_restrict == 0) | (Image.x_restrict.is_(None)) + + +def _orientation_where_clause(*, orientation: int | None) -> object | None: + if orientation is None: + return None + orientation_i = int(orientation) + if orientation_i not in {1, 2, 3}: + raise ValueError("orientation must be 1, 2, or 3") + return Image.orientation == orientation_i + + +def _clean_tag_names(values: Sequence[str] | None) -> list[str]: + if not values: + return [] + out: list[str] = [] + seen: set[str] = set() + for v in values: + name = str(v or "").strip() + if not name: + continue + if name in seen: + continue + seen.add(name) + out.append(name) + return out + + +def _included_tags_where_clause(*, tag_names: Sequence[str]) -> object | None: + names = _clean_tag_names(tag_names) + if not names: + return None + subq = ( + select(ImageTag.image_id) + .join(Tag, Tag.id == ImageTag.tag_id) + .where(Tag.name.in_(names)) + .group_by(ImageTag.image_id) + .having(func.count(distinct(Tag.name)) == len(names)) + ) + return Image.id.in_(subq) + + +def _excluded_tags_where_clause(*, tag_names: Sequence[str]) -> object | None: + names = _clean_tag_names(tag_names) + if not names: + return None + subq = ( + select(ImageTag.image_id) + .join(Tag, Tag.id == ImageTag.tag_id) + .where(Tag.name.in_(names)) + ) + return Image.id.not_in(subq) + + +def _exclude_image_ids_where_clause(*, image_ids: Sequence[int] | None) -> object | None: + if not image_ids: + return None + ids: list[int] = [] + seen: set[int] = set() + for raw in image_ids: + try: + value = int(raw) + except Exception: + continue + if value <= 0 or value in seen: + continue + seen.add(value) + ids.append(value) + if not ids: + return None + return Image.id.not_in(ids) + + +async def pick_random_image( + session: AsyncSession, + *, + r: float, + r18: int = 0, + r18_strict: bool = True, + orientation: int | None = None, + ai_type: int | None = None, + illust_type: int | None = None, + min_width: int = 0, + min_height: int = 0, + min_pixels: int = 0, + min_bookmarks: int = 0, + min_views: int = 0, + min_comments: int = 0, + included_tags: Sequence[str] | None = None, + excluded_tags: Sequence[str] | None = None, + user_id: int | None = None, + illust_id: int | None = None, + created_from: str | None = None, + created_to: str | None = None, + exclude_image_ids: Sequence[int] | None = None, + fail_cooldown_before: str | None = None, +) -> Image | None: + r = float(r) + if r < 0.0: + r = 0.0 + if r >= 1.0: + r = 0.999999999 + + r18_clause = _r18_where_clause(r18=r18, r18_strict=r18_strict) + orientation_clause = _orientation_where_clause(orientation=orientation) + included_tags_clause = _included_tags_where_clause(tag_names=included_tags or []) + excluded_tags_clause = _excluded_tags_where_clause(tag_names=excluded_tags or []) + exclude_ids_clause = _exclude_image_ids_where_clause(image_ids=exclude_image_ids) + + min_width_i = int(min_width) + min_height_i = int(min_height) + min_pixels_i = int(min_pixels) + min_bookmarks_i = int(min_bookmarks) + min_views_i = int(min_views) + min_comments_i = int(min_comments) + if min_width_i < 0 or min_height_i < 0 or min_pixels_i < 0: + raise ValueError("min_* must be >= 0") + if min_bookmarks_i < 0 or min_views_i < 0 or min_comments_i < 0: + raise ValueError("min_* must be >= 0") + + clauses = [Image.status == 1] + if r18_clause is not None: + clauses.append(r18_clause) + if orientation_clause is not None: + clauses.append(orientation_clause) + if ai_type is not None: + ai_type_i = int(ai_type) + if ai_type_i not in {0, 1}: + raise ValueError("ai_type must be 0, 1, or None") + clauses.append(Image.ai_type == ai_type_i) + if illust_type is not None: + illust_type_i = int(illust_type) + if illust_type_i not in {0, 1, 2}: + raise ValueError("illust_type must be 0, 1, 2, or None") + clauses.append(Image.illust_type == illust_type_i) + if included_tags_clause is not None: + clauses.append(included_tags_clause) + if excluded_tags_clause is not None: + clauses.append(excluded_tags_clause) + if exclude_ids_clause is not None: + clauses.append(exclude_ids_clause) + if min_width_i > 0: + clauses.append(Image.width >= min_width_i) + if min_height_i > 0: + clauses.append(Image.height >= min_height_i) + if min_pixels_i > 0: + clauses.append((Image.width * Image.height) >= min_pixels_i) + if min_bookmarks_i > 0: + clauses.append(Image.bookmark_count >= min_bookmarks_i) + if min_views_i > 0: + clauses.append(Image.view_count >= min_views_i) + if min_comments_i > 0: + clauses.append(Image.comment_count >= min_comments_i) + if user_id is not None: + clauses.append(Image.user_id == int(user_id)) + if illust_id is not None: + clauses.append(Image.illust_id == int(illust_id)) + if created_from is not None: + clauses.append(Image.created_at_pixiv >= str(created_from)) + if created_to is not None: + clauses.append(Image.created_at_pixiv <= str(created_to)) + if fail_cooldown_before is not None: + clauses.append((Image.last_fail_at.is_(None)) | (Image.last_fail_at <= str(fail_cooldown_before))) + + stmt = ( + select(Image) + .where(*clauses, Image.random_key >= r) + .order_by(Image.random_key.asc()) + .limit(1) + ) + image = (await session.execute(stmt)).scalars().first() + if image is not None: + return image + + stmt2 = ( + select(Image) + .where(*clauses) + .order_by(Image.random_key.asc()) + .limit(1) + ) + return (await session.execute(stmt2)).scalars().first() + + +def _allowed_int_or_null_clause(column: object, *, allowed: set[int | None]) -> object | None: + allowed_norm: set[int | None] = set(allowed) + ints = sorted({int(v) for v in allowed_norm if v is not None}) + has_null = None in allowed_norm + + if ints and has_null and set(ints) in ({0, 1}, {0, 1, 2}): + return None + + if ints and has_null: + return or_(column.in_(ints), column.is_(None)) # type: ignore[attr-defined] + if ints: + if len(ints) == 1: + return column == ints[0] # type: ignore[operator] + return column.in_(ints) # type: ignore[attr-defined] + if has_null: + return column.is_(None) # type: ignore[attr-defined] + return None + + +async def pick_random_images( + session: AsyncSession, + *, + r: float, + limit: int, + r18: int = 0, + r18_strict: bool = True, + orientation: int | None = None, + ai_type: int | None = None, + illust_type: int | None = None, + ai_type_allowed: set[int | None] | None = None, + illust_type_allowed: set[int | None] | None = None, + min_width: int = 0, + min_height: int = 0, + min_pixels: int = 0, + min_bookmarks: int = 0, + min_views: int = 0, + min_comments: int = 0, + included_tags: Sequence[str] | None = None, + excluded_tags: Sequence[str] | None = None, + user_id: int | None = None, + illust_id: int | None = None, + created_from: str | None = None, + created_to: str | None = None, + exclude_image_ids: Sequence[int] | None = None, + fail_cooldown_before: str | None = None, +) -> list[Image]: + limit_i = int(limit) + if limit_i <= 0: + return [] + if limit_i > 5000: + limit_i = 5000 + + r = float(r) + if r < 0.0: + r = 0.0 + if r >= 1.0: + r = 0.999999999 + + r18_clause = _r18_where_clause(r18=r18, r18_strict=r18_strict) + orientation_clause = _orientation_where_clause(orientation=orientation) + included_tags_clause = _included_tags_where_clause(tag_names=included_tags or []) + excluded_tags_clause = _excluded_tags_where_clause(tag_names=excluded_tags or []) + exclude_ids_clause = _exclude_image_ids_where_clause(image_ids=exclude_image_ids) + + min_width_i = int(min_width) + min_height_i = int(min_height) + min_pixels_i = int(min_pixels) + min_bookmarks_i = int(min_bookmarks) + min_views_i = int(min_views) + min_comments_i = int(min_comments) + if min_width_i < 0 or min_height_i < 0 or min_pixels_i < 0: + raise ValueError("min_* must be >= 0") + if min_bookmarks_i < 0 or min_views_i < 0 or min_comments_i < 0: + raise ValueError("min_* must be >= 0") + + clauses = [Image.status == 1] + if r18_clause is not None: + clauses.append(r18_clause) + if orientation_clause is not None: + clauses.append(orientation_clause) + if ai_type is not None: + ai_type_i = int(ai_type) + if ai_type_i not in {0, 1}: + raise ValueError("ai_type must be 0, 1, or None") + clauses.append(Image.ai_type == ai_type_i) + if illust_type is not None: + illust_type_i = int(illust_type) + if illust_type_i not in {0, 1, 2}: + raise ValueError("illust_type must be 0, 1, 2, or None") + clauses.append(Image.illust_type == illust_type_i) + if ai_type_allowed is not None: + allowed = set(ai_type_allowed) + if not allowed: + return [] + clause = _allowed_int_or_null_clause(Image.ai_type, allowed=allowed) + if clause is not None: + clauses.append(clause) + if illust_type_allowed is not None: + allowed = set(illust_type_allowed) + if not allowed: + return [] + clause = _allowed_int_or_null_clause(Image.illust_type, allowed=allowed) + if clause is not None: + clauses.append(clause) + if included_tags_clause is not None: + clauses.append(included_tags_clause) + if excluded_tags_clause is not None: + clauses.append(excluded_tags_clause) + if exclude_ids_clause is not None: + clauses.append(exclude_ids_clause) + if min_width_i > 0: + clauses.append(Image.width >= min_width_i) + if min_height_i > 0: + clauses.append(Image.height >= min_height_i) + if min_pixels_i > 0: + clauses.append((Image.width * Image.height) >= min_pixels_i) + if min_bookmarks_i > 0: + clauses.append(Image.bookmark_count >= min_bookmarks_i) + if min_views_i > 0: + clauses.append(Image.view_count >= min_views_i) + if min_comments_i > 0: + clauses.append(Image.comment_count >= min_comments_i) + if user_id is not None: + clauses.append(Image.user_id == int(user_id)) + if illust_id is not None: + clauses.append(Image.illust_id == int(illust_id)) + if created_from is not None: + clauses.append(Image.created_at_pixiv >= str(created_from)) + if created_to is not None: + clauses.append(Image.created_at_pixiv <= str(created_to)) + if fail_cooldown_before is not None: + clauses.append((Image.last_fail_at.is_(None)) | (Image.last_fail_at <= str(fail_cooldown_before))) + + stmt = ( + select(Image) + .where(*clauses, Image.random_key >= r) + .order_by(Image.random_key.asc()) + .limit(limit_i) + ) + items = (await session.execute(stmt)).scalars().all() + if len(items) >= limit_i: + return list(items) + + remain = int(limit_i - len(items)) + if remain <= 0: + return list(items) + + stmt2 = ( + select(Image) + .where(*clauses, Image.random_key < r) + .order_by(Image.random_key.asc()) + .limit(remain) + ) + more = (await session.execute(stmt2)).scalars().all() + if not more: + return list(items) + return list(items) + list(more) diff --git a/backend/app/db/request_logs_cleanup.py b/backend/app/db/request_logs_cleanup.py new file mode 100644 index 0000000..acc5f7f --- /dev/null +++ b/backend/app/db/request_logs_cleanup.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone + +import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.time import iso_utc_ms +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + +DEFAULT_REQUEST_LOGS_KEEP_DAYS = 30 +DEFAULT_REQUEST_LOGS_MAX_DELETE_ROWS = 50_000 +DEFAULT_REQUEST_LOGS_CHUNK_SIZE = 1_000 + + +@dataclass(frozen=True, slots=True) +class RequestLogsCleanupResult: + cutoff: str + deleted: int + has_more: bool + + +@dataclass(frozen=True, slots=True) +class RequestLogsCleanupPreview: + cutoff: str + would_delete: int + has_more: bool + + +def _clamp_int(value: int, *, min_v: int, max_v: int) -> int: + return max(min_v, min(int(value), max_v)) + + +def _cutoff_iso(*, keep_days: int, now: datetime | None = None) -> str: + keep_days_i = int(keep_days) + if keep_days_i < 0: + raise ValueError("keep_days must be >= 0") + + now_dt = now or datetime.now(timezone.utc) + cutoff_dt = now_dt - timedelta(days=keep_days_i) + return iso_utc_ms(cutoff_dt) + + +async def preview_request_logs_cleanup( + engine: AsyncEngine, + *, + keep_days: int = DEFAULT_REQUEST_LOGS_KEEP_DAYS, + max_delete_rows: int = DEFAULT_REQUEST_LOGS_MAX_DELETE_ROWS, + now: datetime | None = None, +) -> RequestLogsCleanupPreview: + cutoff = _cutoff_iso(keep_days=int(keep_days), now=now) + max_delete_rows_i = _clamp_int(int(max_delete_rows), min_v=1, max_v=10_000_000) + + Session = create_sessionmaker(engine) + sql = """ +SELECT id +FROM request_logs +WHERE created_at < :cutoff +ORDER BY created_at ASC +LIMIT :limit; +""".strip() + + async def _op() -> RequestLogsCleanupPreview: + async with Session() as session: + res = await session.execute(sa.text(sql), {"cutoff": cutoff, "limit": int(max_delete_rows_i) + 1}) + rows = res.all() + would_delete = min(len(rows), int(max_delete_rows_i)) + has_more = len(rows) > int(max_delete_rows_i) + return RequestLogsCleanupPreview(cutoff=cutoff, would_delete=would_delete, has_more=has_more) + + return await with_sqlite_busy_retry(_op) + + +async def cleanup_request_logs( + engine: AsyncEngine, + *, + keep_days: int = DEFAULT_REQUEST_LOGS_KEEP_DAYS, + max_delete_rows: int = DEFAULT_REQUEST_LOGS_MAX_DELETE_ROWS, + chunk_size: int = DEFAULT_REQUEST_LOGS_CHUNK_SIZE, + now: datetime | None = None, +) -> RequestLogsCleanupResult: + cutoff = _cutoff_iso(keep_days=int(keep_days), now=now) + max_delete_rows_i = _clamp_int(int(max_delete_rows), min_v=1, max_v=10_000_000) + chunk_size_i = _clamp_int(int(chunk_size), min_v=1, max_v=100_000) + chunk_size_i = min(int(chunk_size_i), int(max_delete_rows_i)) + + Session = create_sessionmaker(engine) + + delete_sql = """ +DELETE FROM request_logs +WHERE id IN ( + SELECT id + FROM request_logs + WHERE created_at < :cutoff + ORDER BY created_at ASC + LIMIT :chunk +); +""".strip() + + has_more_sql = """ +SELECT 1 +FROM request_logs +WHERE created_at < :cutoff +LIMIT 1; +""".strip() + + async def _op() -> RequestLogsCleanupResult: + deleted = 0 + has_more = False + + async with Session() as session: + while deleted < int(max_delete_rows_i): + chunk = min(int(chunk_size_i), int(max_delete_rows_i) - int(deleted)) + result = await session.execute(sa.text(delete_sql), {"cutoff": cutoff, "chunk": int(chunk)}) + n = int(result.rowcount or 0) + if n <= 0: + await session.commit() + break + deleted += n + await session.commit() + + if deleted >= int(max_delete_rows_i): + res = await session.execute(sa.text(has_more_sql), {"cutoff": cutoff}) + has_more = res.first() is not None + + return RequestLogsCleanupResult(cutoff=cutoff, deleted=int(deleted), has_more=bool(has_more)) + + return await with_sqlite_busy_retry(_op) + diff --git a/backend/app/db/session.py b/backend/app/db/session.py new file mode 100644 index 0000000..fd87f17 --- /dev/null +++ b/backend/app/db/session.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import asyncio +import sqlite3 +from collections.abc import AsyncIterator, Awaitable, Callable +from typing import TypeVar + +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +T = TypeVar("T") + + +def is_sqlite_busy_error(exc: BaseException) -> bool: + if isinstance(exc, sqlite3.OperationalError): + msg = str(exc).lower() + return ( + "database is locked" in msg + or "database table is locked" in msg + or "database schema is locked" in msg + or "database is busy" in msg + ) + if isinstance(exc, OperationalError): + orig = getattr(exc, "orig", None) + return is_sqlite_busy_error(orig) if isinstance(orig, BaseException) else False + return False + + +async def with_sqlite_busy_retry( + op: Callable[[], Awaitable[T]], + *, + retries: int = 3, + base_delay_s: float = 0.05, +) -> T: + attempt = 0 + while True: + try: + return await op() + except Exception as exc: + if attempt >= retries or not is_sqlite_busy_error(exc): + raise + await asyncio.sleep(base_delay_s * (2**attempt)) + attempt += 1 + + +def create_sessionmaker(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]: + return async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + + +async def get_session(sessionmaker: async_sessionmaker[AsyncSession]) -> AsyncIterator[AsyncSession]: + async with sessionmaker() as session: + yield session diff --git a/backend/app/db/sqlite_utils.py b/backend/app/db/sqlite_utils.py new file mode 100644 index 0000000..cebba98 --- /dev/null +++ b/backend/app/db/sqlite_utils.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncSession + + +def sqlite_fts_phrase_query(q: str) -> str: + q = (q or "").strip() + q = q.replace('"', '""') + return f'"{q}"' + + +async def sqlite_table_exists(session: AsyncSession, *, name: str) -> bool: + name = (name or "").strip() + if not name: + return False + result = await session.execute( + sa.text("SELECT 1 FROM sqlite_master WHERE type='table' AND name=:name LIMIT 1;"), + {"name": name}, + ) + return result.first() is not None diff --git a/backend/app/db/tags_get.py b/backend/app/db/tags_get.py new file mode 100644 index 0000000..119704f --- /dev/null +++ b/backend/app/db/tags_get.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models.image_tags import ImageTag +from app.db.models.tags import Tag + + +async def get_tag_names_for_image(session: AsyncSession, *, image_id: int) -> list[str]: + stmt = ( + select(Tag.name) + .join(ImageTag, ImageTag.tag_id == Tag.id) + .where(ImageTag.image_id == int(image_id)) + .order_by(Tag.name.asc()) + ) + rows = (await session.execute(stmt)).scalars().all() + return [str(r) for r in rows] + diff --git a/backend/app/db/tags_list.py b/backend/app/db/tags_list.py new file mode 100644 index 0000000..6b586c9 --- /dev/null +++ b/backend/app/db/tags_list.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import sqlalchemy as sa +from sqlalchemy import distinct, func, select +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.sqlite_utils import sqlite_fts_phrase_query, sqlite_table_exists + + +@dataclass(frozen=True, slots=True) +class TagListItem: + id: int + name: str + translated_name: str | None + count_images: int + + +async def list_tags( + session: AsyncSession, + *, + limit: int, + cursor: str | None = None, + q: str | None = None, +) -> tuple[list[TagListItem], str | None]: + limit_i = int(limit) + if limit_i < 1: + raise ValueError("limit must be >= 1") + + cursor_name = (cursor or "").strip() + cursor_name = cursor_name if cursor_name else "" + + q_norm = (q or "").strip() + + use_fts = False + fts_q = "" + if q_norm and len(q_norm) >= 3: + use_fts = await sqlite_table_exists(session, name="tags_fts") + if use_fts: + fts_q = sqlite_fts_phrase_query(q_norm) + + def _build_stmt(*, use_fts_filter: bool) -> sa.Select: + stmt = ( + select( + Tag.id, + Tag.name, + Tag.translated_name, + func.count(distinct(ImageTag.image_id)).label("count_images"), + ) + .join(ImageTag, ImageTag.tag_id == Tag.id) + .join(Image, Image.id == ImageTag.image_id) + .where(Image.status == 1) + ) + + if q_norm: + if use_fts_filter: + fts_ids = ( + sa.text("SELECT rowid AS tag_id FROM tags_fts WHERE tags_fts MATCH :q") + .bindparams(sa.bindparam("q", fts_q)) + .columns(tag_id=sa.Integer) + ) + fts_ids_sq = fts_ids.subquery() + stmt = stmt.where(Tag.id.in_(sa.select(fts_ids_sq.c.tag_id))) + else: + like = f"%{q_norm}%" + stmt = stmt.where(sa.or_(Tag.name.like(like), Tag.translated_name.like(like))) + if cursor_name: + stmt = stmt.where(Tag.name > cursor_name) + + return stmt.group_by(Tag.id).order_by(Tag.name.asc()).limit(limit_i + 1) + + stmt = _build_stmt(use_fts_filter=use_fts) + try: + rows = (await session.execute(stmt)).all() + except DBAPIError: + if not use_fts: + raise + stmt = _build_stmt(use_fts_filter=False) + rows = (await session.execute(stmt)).all() + next_cursor: str | None = None + + if len(rows) > limit_i: + rows = rows[:limit_i] + next_cursor = str(rows[-1][1] or "") + + items = [ + TagListItem( + id=int(row[0]), + name=str(row[1]), + translated_name=str(row[2]) if row[2] is not None else None, + count_images=int(row[3] or 0), + ) + for row in rows + ] + + return items, next_cursor diff --git a/backend/app/easy_proxies/__init__.py b/backend/app/easy_proxies/__init__.py new file mode 100644 index 0000000..3ce55a6 --- /dev/null +++ b/backend/app/easy_proxies/__init__.py @@ -0,0 +1,2 @@ +from __future__ import annotations + diff --git a/backend/app/easy_proxies/auto_refresh.py b/backend/app/easy_proxies/auto_refresh.py new file mode 100644 index 0000000..3969f9e --- /dev/null +++ b/backend/app/easy_proxies/auto_refresh.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import json +import time +from dataclasses import dataclass +from typing import Callable + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.logging import get_logger +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + +log = get_logger(__name__) + + +@dataclass(frozen=True, slots=True) +class EasyProxiesAutoRefreshConfig: + base_url: str + interval_s: float + conflict_policy: str = "skip_non_easy_proxies" + host_override: str | None = None + auto_attach: bool = True + attach_pool_id: int | None = None + attach_weight: int = 1 + recompute_bindings: bool = True + max_tokens_per_proxy: int = 2 + strict: bool = False + + +async def _first_enabled_pool_id(engine: AsyncEngine) -> int | None: + sql = "SELECT id FROM proxy_pools WHERE enabled=1 ORDER BY id ASC LIMIT 1;" + + async def _op() -> int | None: + async with engine.connect() as conn: + result = await conn.exec_driver_sql(sql) + value = result.scalar_one_or_none() + return int(value) if value is not None else None + + return await with_sqlite_busy_retry(_op) + + +async def _enqueue_if_needed( + engine: AsyncEngine, + *, + base_url: str, + conflict_policy: str, + payload: dict[str, object], +) -> int | None: + Session = create_sessionmaker(engine) + + async def _op() -> int | None: + async with Session() as session: + existing = await session.execute( + select(JobRow.id).where( + JobRow.type == "easy_proxies_import", + JobRow.ref_type == "easy_proxies", + JobRow.ref_id == base_url, + JobRow.status.in_(("pending", "running")), + ) + ) + if existing.first() is not None: + return None + + job = JobRow( + type="easy_proxies_import", + status="pending", + priority=0, + payload_json=json.dumps( + {"base_url": base_url, "conflict_policy": conflict_policy, **payload}, + ensure_ascii=False, + ), + ref_type="easy_proxies", + ref_id=base_url, + ) + session.add(job) + await session.flush() + await session.commit() + return int(job.id) + + return await with_sqlite_busy_retry(_op) + + +class EasyProxiesAutoRefresher: + def __init__( + self, + config: EasyProxiesAutoRefreshConfig, + *, + now: Callable[[], float] | None = None, + ) -> None: + self._config = config + self._now = now or time.time + self._last_enqueued_at: float | None = None + + @property + def enabled(self) -> bool: + return bool(self._config.base_url.strip()) and float(self._config.interval_s) > 0 + + async def tick(self, engine: AsyncEngine) -> None: + if not self.enabled: + return + + now = float(self._now()) + last = self._last_enqueued_at + if last is not None and (now - last) < float(self._config.interval_s): + return + + base_url = self._config.base_url.strip() + resolved_pool_id = self._config.attach_pool_id + if bool(self._config.auto_attach) and resolved_pool_id is None: + resolved_pool_id = await _first_enabled_pool_id(engine) + + payload: dict[str, object] = {} + if self._config.host_override: + payload["host_override"] = str(self._config.host_override) + if resolved_pool_id is not None and int(resolved_pool_id) > 0: + payload["attach_pool_id"] = int(resolved_pool_id) + payload["attach_weight"] = int(self._config.attach_weight) + if bool(self._config.recompute_bindings): + payload["recompute_bindings"] = True + payload["max_tokens_per_proxy"] = int(self._config.max_tokens_per_proxy) + payload["strict"] = bool(self._config.strict) + + job_id = await _enqueue_if_needed( + engine, + base_url=base_url, + conflict_policy=self._config.conflict_policy, + payload=payload, + ) + self._last_enqueued_at = now + if job_id is not None: + log.info("easy_proxies_auto_refresh_enqueued base_url=%s job_id=%s", base_url, job_id) diff --git a/backend/app/easy_proxies/client.py b/backend/app/easy_proxies/client.py new file mode 100644 index 0000000..9d8eb49 --- /dev/null +++ b/backend/app/easy_proxies/client.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any +from urllib.parse import urljoin, urlparse + +import httpx + + +def _normalize_base_url(base_url: str) -> str: + base_url = (base_url or "").strip() + if not base_url: + raise ValueError("base_url is required") + + parsed = urlparse(base_url) + scheme = (parsed.scheme or "").lower() + if scheme not in {"http", "https"}: + raise ValueError("base_url must be http(s)") + + if not parsed.netloc: + raise ValueError("base_url must include host") + + return base_url.rstrip("/") + "/" + + +class EasyProxiesError(RuntimeError): + def __init__(self, message: str, *, status_code: int | None = None) -> None: + super().__init__(message) + self.status_code = status_code + + +@dataclass(frozen=True, slots=True) +class EasyProxiesAuthResult: + token: str + + +async def easy_proxies_auth( + *, + base_url: str, + password: str, + transport: httpx.BaseTransport | None = None, + timeout_s: float = 30.0, +) -> EasyProxiesAuthResult: + base_url_n = _normalize_base_url(base_url) + password = (password or "").strip() + if not password: + raise ValueError("password is required") + + url = urljoin(base_url_n, "api/auth") + async with httpx.AsyncClient( + transport=transport, + timeout=httpx.Timeout(timeout_s, connect=10.0), + follow_redirects=True, + ) as client: + resp = await client.post(url, json={"password": password}) + + if resp.status_code != 200: + raise EasyProxiesError("easy_proxies auth failed", status_code=resp.status_code) + + try: + data: Any = resp.json() + except Exception as exc: + raise EasyProxiesError("easy_proxies auth response is not JSON", status_code=resp.status_code) from exc + + token = data.get("token") if isinstance(data, dict) else None + if not isinstance(token, str) or not token.strip(): + raise EasyProxiesError("easy_proxies auth response missing token", status_code=resp.status_code) + + return EasyProxiesAuthResult(token=token.strip()) + + +async def easy_proxies_export( + *, + base_url: str, + bearer_token: str | None = None, + transport: httpx.BaseTransport | None = None, + timeout_s: float = 30.0, +) -> list[str]: + base_url_n = _normalize_base_url(base_url) + url = urljoin(base_url_n, "api/export") + + headers: dict[str, str] = {} + if bearer_token: + headers["Authorization"] = f"Bearer {bearer_token}" + + async with httpx.AsyncClient( + transport=transport, + timeout=httpx.Timeout(timeout_s, connect=10.0), + follow_redirects=True, + headers=headers, + ) as client: + resp = await client.get(url) + + if resp.status_code != 200: + raise EasyProxiesError("easy_proxies export failed", status_code=resp.status_code) + + text = resp.text + out: list[str] = [] + for line in text.splitlines(): + line = line.strip() + if not line: + continue + out.append(line) + return out + diff --git a/backend/app/easy_proxies/normalize.py b/backend/app/easy_proxies/normalize.py new file mode 100644 index 0000000..0cd30e6 --- /dev/null +++ b/backend/app/easy_proxies/normalize.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from urllib.parse import urlparse + +_INVALID_EXPORT_HOSTS = { + "0.0.0.0", + "127.0.0.1", + "localhost", + "::", + "::1", +} + + +def resolve_export_host(*, base_url: str, host_override: str | None = None) -> str | None: + override = str(host_override or "").strip() + if override: + host = override + else: + try: + parsed = urlparse(str(base_url or "").strip()) + except Exception: + return None + host = str(parsed.hostname or "").strip() + + if not host: + return None + if host.lower() in _INVALID_EXPORT_HOSTS: + return None + return host + + +def normalize_exported_proxy_host(*, exported_host: str, export_host: str | None) -> tuple[str, bool]: + host = str(exported_host or "").strip() + if not host or not export_host: + return host, False + if host.lower() in _INVALID_EXPORT_HOSTS: + return str(export_host), True + return host, False + diff --git a/backend/app/jobs/backoff.py b/backend/app/jobs/backoff.py new file mode 100644 index 0000000..94463d3 --- /dev/null +++ b/backend/app/jobs/backoff.py @@ -0,0 +1,23 @@ +from __future__ import annotations + + +def backoff_seconds(attempt: int) -> int: + attempt_i = int(attempt) + if attempt_i <= 0: + return 0 + + schedule = { + 1: 5, + 2: 30, + 3: 120, + 4: 600, + 5: 1800, + } + if attempt_i in schedule: + return schedule[attempt_i] + + # Exponential after the documented schedule, with a hard cap. + base = 1800 + seconds = base * (2 ** (attempt_i - 5)) + return min(seconds, 6 * 3600) + diff --git a/backend/app/jobs/claim.py b/backend/app/jobs/claim.py new file mode 100644 index 0000000..6a7ab06 --- /dev/null +++ b/backend/app/jobs/claim.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.metrics import JOBS_CLAIM_TOTAL +from app.db.session import with_sqlite_busy_retry + +DEFAULT_LOCK_TTL_S = 300 + + +def _iso_utc_ms(dt: datetime) -> str: + dt = dt.astimezone(timezone.utc) + ms = dt.microsecond // 1000 + return dt.strftime("%Y-%m-%dT%H:%M:%S.") + f"{ms:03d}Z" + + +async def claim_next_job( + engine: AsyncEngine, + *, + worker_id: str, + lock_ttl_s: int = DEFAULT_LOCK_TTL_S, + now: datetime | None = None, +) -> dict[str, Any] | None: + now_dt = now or datetime.now(timezone.utc) + expired_before_dt = now_dt - timedelta(seconds=lock_ttl_s) + now_s = _iso_utc_ms(now_dt) + expired_before_s = _iso_utc_ms(expired_before_dt) + + sql = """ +WITH candidate AS ( + SELECT id + FROM jobs + WHERE status IN ('pending','failed','running') + AND (run_after IS NULL OR run_after <= :now) + AND (locked_at IS NULL OR locked_at <= :lock_expired_before) + ORDER BY priority DESC, id ASC + LIMIT 1 +) +UPDATE jobs +SET status='running', + locked_by=:worker_id, + locked_at=:now, + updated_at=:now +WHERE id IN (SELECT id FROM candidate) +RETURNING *; +""".strip() + + async def _op() -> dict[str, Any] | None: + async with engine.begin() as conn: + result = await conn.exec_driver_sql( + sql, + { + "now": now_s, + "lock_expired_before": expired_before_s, + "worker_id": worker_id, + }, + ) + row = result.mappings().first() + return dict(row) if row else None + + job = await with_sqlite_busy_retry(_op) + if job is not None: + JOBS_CLAIM_TOTAL.inc() + return job + + +async def renew_job_lock( + engine: AsyncEngine, + *, + job_id: int, + worker_id: str, + now: datetime | None = None, +) -> bool: + now_dt = now or datetime.now(timezone.utc) + now_s = _iso_utc_ms(now_dt) + + sql = """ +UPDATE jobs +SET locked_at=:now, + updated_at=:now +WHERE id=:id AND locked_by=:worker_id AND status='running'; +""".strip() + + async def _op() -> bool: + async with engine.begin() as conn: + result = await conn.exec_driver_sql( + sql, + {"now": now_s, "id": job_id, "worker_id": worker_id}, + ) + return (result.rowcount or 0) == 1 + + return await with_sqlite_busy_retry(_op) diff --git a/backend/app/jobs/dispatch.py b/backend/app/jobs/dispatch.py new file mode 100644 index 0000000..2c2d65a --- /dev/null +++ b/backend/app/jobs/dispatch.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import Any + +JobHandler = Callable[[dict[str, Any]], Awaitable[None]] + + +@dataclass(slots=True) +class JobDispatcher: + handlers: dict[str, JobHandler] = field(default_factory=dict) + + def register(self, job_type: str, handler: JobHandler) -> None: + job_type = job_type.strip() + if not job_type: + raise ValueError("job_type is required") + self.handlers[job_type] = handler + + def handler(self, job_type: str) -> Callable[[JobHandler], JobHandler]: + def _wrap(fn: JobHandler) -> JobHandler: + self.register(job_type, fn) + return fn + + return _wrap + + async def dispatch(self, job: dict[str, Any]) -> None: + job_type = str(job.get("type") or "").strip() + handler = self.handlers.get(job_type) + if handler is None: + raise ValueError(f"Unknown job type: {job_type or ''}") + await handler(job) + diff --git a/backend/app/jobs/enqueue.py b/backend/app/jobs/enqueue.py new file mode 100644 index 0000000..8bb85ff --- /dev/null +++ b/backend/app/jobs/enqueue.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import json +import os + +import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.metrics import RANDOM_OPPORTUNISTIC_HYDRATE_ENQUEUED_TOTAL +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker, with_sqlite_busy_retry + +OPPORTUNISTIC_HYDRATE_REF_TYPE = "opportunistic_hydrate" +OPPORTUNISTIC_HYDRATE_PRIORITY = -10 + + +def _pixiv_oauth_configured() -> bool: + # Opportunistic hydration needs OAuth credentials to refresh access tokens. + client_id = (os.environ.get("PIXIV_OAUTH_CLIENT_ID") or "").strip() + client_secret = (os.environ.get("PIXIV_OAUTH_CLIENT_SECRET") or "").strip() + return bool(client_id and client_secret) + + +async def enqueue_opportunistic_hydrate_metadata( + engine: AsyncEngine, + *, + illust_id: int, + reason: str, +) -> int | None: + if int(illust_id) <= 0: + return None + if not _pixiv_oauth_configured(): + return None + + ref_id = str(int(illust_id)) + Session = create_sessionmaker(engine) + + payload_json = json.dumps( + {"illust_id": int(illust_id), "reason": str(reason or "random").strip() or "random"}, + ensure_ascii=False, + separators=(",", ":"), + ) + + async def _op() -> int | None: + async with Session() as session: + existing = await session.execute( + sa.select(JobRow.id).where( + JobRow.type == "hydrate_metadata", + JobRow.ref_type == OPPORTUNISTIC_HYDRATE_REF_TYPE, + JobRow.ref_id == ref_id, + JobRow.status.in_(("pending", "running")), + ) + ) + if existing.first() is not None: + return None + + job = JobRow( + type="hydrate_metadata", + status="pending", + priority=int(OPPORTUNISTIC_HYDRATE_PRIORITY), + payload_json=payload_json, + ref_type=OPPORTUNISTIC_HYDRATE_REF_TYPE, + ref_id=ref_id, + ) + session.add(job) + await session.flush() + await session.commit() + return int(job.id) + + job_id = await with_sqlite_busy_retry(_op) + if job_id is not None: + RANDOM_OPPORTUNISTIC_HYDRATE_ENQUEUED_TOTAL.inc() + return job_id diff --git a/backend/app/jobs/errors.py b/backend/app/jobs/errors.py new file mode 100644 index 0000000..83e454b --- /dev/null +++ b/backend/app/jobs/errors.py @@ -0,0 +1,11 @@ +from __future__ import annotations + + +class JobPermanentError(RuntimeError): + pass + + +class JobDeferError(RuntimeError): + def __init__(self, message: str, *, run_after: str) -> None: + super().__init__(message) + self.run_after = run_after diff --git a/backend/app/jobs/executor.py b/backend/app/jobs/executor.py new file mode 100644 index 0000000..952829e --- /dev/null +++ b/backend/app/jobs/executor.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import random +from datetime import datetime, timedelta, timezone +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.time import iso_utc_ms +from app.core.metrics import JOBS_FAILED_TOTAL +from app.db.session import is_sqlite_busy_error, with_sqlite_busy_retry +from app.jobs.dispatch import JobDispatcher +from app.jobs.errors import JobDeferError, JobPermanentError +from app.jobs.model import Job, JobStatus, JobTransition, on_job_defer, on_job_failure, on_job_success + + +def _as_int(value: Any, *, default: int = 0) -> int: + try: + return int(value) + except Exception: + return default + + +def _as_str(value: Any) -> str | None: + if value is None: + return None + s = str(value).strip() + return s if s else None + + +def _job_from_row(row: dict[str, Any]) -> Job: + return Job( + id=_as_int(row.get("id")), + status=JobStatus(str(row.get("status") or "").strip() or JobStatus.RUNNING.value), + attempt=_as_int(row.get("attempt")), + max_attempts=max(1, _as_int(row.get("max_attempts"), default=3)), + run_after=_as_str(row.get("run_after")), + last_error=_as_str(row.get("last_error")), + locked_by=_as_str(row.get("locked_by")), + locked_at=_as_str(row.get("locked_at")), + ) + + +async def _apply_transition( + engine: AsyncEngine, + *, + job_id: int, + worker_id: str, + transition: JobTransition, +) -> bool: + sql = """ +UPDATE jobs +SET status=:status, + attempt=:attempt, + run_after=:run_after, + last_error=:last_error, + locked_by=:locked_by, + locked_at=:locked_at, + updated_at=:updated_at +WHERE id=:id AND status='running' AND locked_by=:worker_id; +""".strip() + + async def _op() -> bool: + async with engine.begin() as conn: + result = await conn.exec_driver_sql( + sql, + { + "status": transition.status.value, + "attempt": int(transition.attempt), + "run_after": transition.run_after, + "last_error": transition.last_error, + "locked_by": transition.locked_by, + "locked_at": transition.locked_at, + "updated_at": transition.updated_at, + "id": int(job_id), + "worker_id": worker_id, + }, + ) + return (result.rowcount or 0) == 1 + + return await with_sqlite_busy_retry(_op) + + +async def execute_claimed_job( + engine: AsyncEngine, + dispatcher: JobDispatcher, + *, + job_row: dict[str, Any], + worker_id: str, + now: datetime | None = None, +) -> JobTransition | None: + worker_id = worker_id.strip() + if not worker_id: + raise ValueError("worker_id is required") + + job = _job_from_row(job_row) + if job.id <= 0: + raise ValueError("job_row.id is required") + + now_dt = now or datetime.now(timezone.utc) + + try: + await dispatcher.dispatch(job_row) + except JobDeferError as exc: + transition = on_job_defer(job, run_after=exc.run_after, error=f"{type(exc).__name__}: {exc}", now=now_dt) + except JobPermanentError as exc: + forced = Job( + id=job.id, + status=job.status, + attempt=job.attempt, + max_attempts=max(1, job.attempt + 1), + run_after=job.run_after, + last_error=job.last_error, + locked_by=job.locked_by, + locked_at=job.locked_at, + ) + transition = on_job_failure(forced, error=f"{type(exc).__name__}: {exc}", now=now_dt) + except ValueError as exc: + msg = str(exc) + if "Unknown job type" in msg: + forced = Job( + id=job.id, + status=job.status, + attempt=job.attempt, + max_attempts=max(1, job.attempt + 1), + run_after=job.run_after, + last_error=job.last_error, + locked_by=job.locked_by, + locked_at=job.locked_at, + ) + transition = on_job_failure(forced, error=f"{type(exc).__name__}: {exc}", now=now_dt) + else: + transition = on_job_failure(job, error=f"{type(exc).__name__}: {exc}", now=now_dt) + except Exception as exc: + if is_sqlite_busy_error(exc): + delay_s = 2.0 + random.random() * 3.0 + run_after = iso_utc_ms(now_dt + timedelta(seconds=delay_s)) + transition = on_job_defer(job, run_after=run_after, error=f"{type(exc).__name__}: {exc}", now=now_dt) + else: + transition = on_job_failure(job, error=f"{type(exc).__name__}: {exc}", now=now_dt) + else: + transition = on_job_success(job, now=now_dt) + + ok = await _apply_transition(engine, job_id=job.id, worker_id=worker_id, transition=transition) + if ok and transition.status in {JobStatus.FAILED, JobStatus.DLQ}: + JOBS_FAILED_TOTAL.inc() + return transition if ok else None diff --git a/backend/app/jobs/handlers/easy_proxies_import.py b/backend/app/jobs/handlers/easy_proxies_import.py new file mode 100644 index 0000000..6b01996 --- /dev/null +++ b/backend/app/jobs/handlers/easy_proxies_import.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import os +import json +from typing import Any + +import httpx +import sqlalchemy as sa +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.ext.asyncio import AsyncEngine +from urllib.parse import urlparse, urlunparse + +from app.core.bindings_recompute import recompute_token_proxy_bindings +from app.core.config import load_settings +from app.core.crypto import FieldEncryptor +from app.core.proxy_uri import parse_proxy_uri +from app.core.time import iso_utc_ms +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.session import create_sessionmaker, with_sqlite_busy_retry +from app.easy_proxies.client import EasyProxiesError, easy_proxies_auth, easy_proxies_export +from app.easy_proxies.normalize import normalize_exported_proxy_host, resolve_export_host +from app.jobs.errors import JobPermanentError + + +def _parse_payload(payload_json: str) -> dict[str, Any]: + try: + data = json.loads(payload_json) + except Exception as exc: + raise JobPermanentError("payload_json is not valid JSON") from exc + if not isinstance(data, dict): + raise JobPermanentError("payload_json must be an object") + return data + + +def _parse_conflict_policy(value: Any) -> str: + v = str(value or "").strip().lower() or "skip_non_easy_proxies" + if v not in {"skip_non_easy_proxies", "skip", "overwrite"}: + raise JobPermanentError("payload.conflict_policy invalid") + return v + + +def _sanitize_source_ref(value: str | None) -> str | None: + raw = str(value or "").strip() + if not raw: + return None + + try: + parsed = urlparse(raw) + except Exception: + return raw[:200] + + if not parsed.scheme or not parsed.netloc: + return raw[:200] + + host = parsed.hostname + if not host: + return raw[:200] + + port = parsed.port + netloc = f"{host}:{int(port)}" if port else host + path = parsed.path or "" + + return urlunparse((parsed.scheme, netloc, path, "", "", "")) + + +def build_easy_proxies_import_handler(engine: AsyncEngine, *, transport: httpx.BaseTransport | None = None) -> Any: + settings = load_settings() + encryptor = FieldEncryptor.from_key(settings.field_encryption_key) + Session = create_sessionmaker(engine) + + async def _handler(job: dict[str, Any]) -> None: + payload_json = str(job.get("payload_json") or "") + payload = _parse_payload(payload_json) + + base_url = str(payload.get("base_url") or "").strip() + if not base_url: + raise JobPermanentError("payload.base_url is required") + + password = str(payload.get("password") or "").strip() + if not password: + password = str(os.environ.get("EASY_PROXIES_PASSWORD") or "").strip() + conflict_policy = _parse_conflict_policy(payload.get("conflict_policy")) + + host_override = str(payload.get("host_override") or "").strip() or None + + attach_pool_id_raw = payload.get("attach_pool_id") + attach_pool_id: int | None = None + if attach_pool_id_raw is not None and str(attach_pool_id_raw).strip(): + try: + attach_pool_id = int(attach_pool_id_raw) + except Exception as exc: + raise JobPermanentError("payload.attach_pool_id invalid") from exc + if attach_pool_id <= 0: + raise JobPermanentError("payload.attach_pool_id invalid") + + attach_weight_raw = payload.get("attach_weight", 1) + try: + attach_weight = int(attach_weight_raw) + except Exception as exc: + raise JobPermanentError("payload.attach_weight invalid") from exc + if attach_weight < 0 or attach_weight > 1000: + raise JobPermanentError("payload.attach_weight invalid") + + recompute_bindings = bool(payload.get("recompute_bindings") or False) + max_tokens_per_proxy_raw = payload.get("max_tokens_per_proxy", 2) + try: + max_tokens_per_proxy = int(max_tokens_per_proxy_raw) + except Exception as exc: + raise JobPermanentError("payload.max_tokens_per_proxy invalid") from exc + if max_tokens_per_proxy <= 0 or max_tokens_per_proxy > 1000: + raise JobPermanentError("payload.max_tokens_per_proxy invalid") + + strict = bool(payload.get("strict", True)) + if recompute_bindings and attach_pool_id is None: + raise JobPermanentError("payload.recompute_bindings requires attach_pool_id") + + base_url_ref = _sanitize_source_ref(base_url) or base_url + export_host = resolve_export_host(base_url=base_url, host_override=host_override) + + try: + bearer_token: str | None = None + if password: + auth = await easy_proxies_auth(base_url=base_url, password=password, transport=transport) + bearer_token = auth.token + uris = await easy_proxies_export(base_url=base_url, bearer_token=bearer_token, transport=transport) + except EasyProxiesError as exc: + raise RuntimeError("easy_proxies import failed") from exc + except Exception as exc: + raise RuntimeError("easy_proxies import failed") from exc + + seen_keys: set[tuple[str, str, int, str]] = set() + deduped: list[str] = [] + for raw in uris: + uri = (raw or "").strip() + if not uri: + continue + try: + parsed = parse_proxy_uri(uri) + except Exception: + continue + desired_host, _rewrote = normalize_exported_proxy_host(exported_host=str(parsed.host), export_host=export_host) + key = (str(parsed.scheme), str(desired_host), int(parsed.port), str((parsed.username or "").strip())) + if key in seen_keys: + continue + seen_keys.add(key) + deduped.append(uri) + uris = deduped + + now = iso_utc_ms() + + async def _op() -> None: + async with Session() as session: + for uri in uris: + uri = (uri or "").strip() + if not uri: + continue + try: + parsed = parse_proxy_uri(uri) + except Exception: + continue + + username = (parsed.username or "").strip() + password_v = parsed.password + password_enc = encryptor.encrypt_text(password_v) if password_v else "" + + desired_host, rewrote = normalize_exported_proxy_host( + exported_host=str(parsed.host), + export_host=export_host, + ) + original_host = str(parsed.host) + + existing = ( + ( + await session.execute( + sa.select(ProxyEndpoint).where( + ProxyEndpoint.scheme == parsed.scheme, + ProxyEndpoint.host == desired_host, + ProxyEndpoint.port == int(parsed.port), + ProxyEndpoint.username == username, + ) + ) + ) + .scalars() + .first() + ) + + if existing is None and rewrote and desired_host and desired_host != original_host: + placeholder = ( + ( + await session.execute( + sa.select(ProxyEndpoint).where( + ProxyEndpoint.scheme == parsed.scheme, + ProxyEndpoint.host == original_host, + ProxyEndpoint.port == int(parsed.port), + ProxyEndpoint.username == username, + ) + ) + ) + .scalars() + .first() + ) + if placeholder is not None: + can_update_placeholder = conflict_policy == "overwrite" or ( + conflict_policy == "skip_non_easy_proxies" + and (placeholder.source or "") == "easy_proxies" + ) + if can_update_placeholder: + existing = placeholder + existing.host = desired_host + + if existing is None: + session.add( + ProxyEndpoint( + scheme=parsed.scheme, + host=desired_host or original_host, + port=int(parsed.port), + username=username, + password_enc=password_enc, + enabled=1, + source="easy_proxies", + source_ref=base_url_ref, + updated_at=now, + ) + ) + continue + + if conflict_policy == "skip_non_easy_proxies" and (existing.source or "") != "easy_proxies": + continue + if conflict_policy == "skip": + continue + + existing.password_enc = password_enc + existing.enabled = 1 + existing.source = "easy_proxies" + existing.source_ref = base_url_ref + # easy-proxies export only contains currently-healthy endpoints; clear local blacklist so they can be used immediately. + existing.blacklisted_until = None + existing.last_error = None + existing.updated_at = now + + if attach_pool_id is not None: + pool = await session.get(ProxyPool, int(attach_pool_id)) + if pool is None: + raise JobPermanentError("payload.attach_pool_id not found") + + endpoint_ids = ( + ( + await session.execute( + sa.select(ProxyEndpoint.id) + .where(ProxyEndpoint.source == "easy_proxies") + .where(ProxyEndpoint.source_ref == base_url_ref) + .where(ProxyEndpoint.enabled == 1) + .order_by(ProxyEndpoint.id.asc()) + ) + ) + .scalars() + .all() + ) + + for endpoint_id in endpoint_ids: + stmt = sqlite_insert(ProxyPoolEndpoint).values( + pool_id=int(attach_pool_id), + endpoint_id=int(endpoint_id), + enabled=1, + weight=int(attach_weight), + updated_at=now, + ) + stmt = stmt.on_conflict_do_update( + index_elements=[ProxyPoolEndpoint.pool_id, ProxyPoolEndpoint.endpoint_id], + set_={"enabled": 1, "weight": int(attach_weight), "updated_at": now}, + ) + await session.execute(stmt) + + if recompute_bindings: + await recompute_token_proxy_bindings( + session, + pool_id=int(attach_pool_id), + now=str(now), + max_tokens_per_proxy=int(max_tokens_per_proxy), + strict=bool(strict), + ) + + await session.commit() + + await with_sqlite_busy_retry(_op) + + return _handler diff --git a/backend/app/jobs/handlers/heal_url.py b/backend/app/jobs/handlers/heal_url.py new file mode 100644 index 0000000..58838f1 --- /dev/null +++ b/backend/app/jobs/handlers/heal_url.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any + +import httpx +import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.time import iso_utc_ms +from app.db.models.images import Image +from app.db.session import create_sessionmaker, with_sqlite_busy_retry +from app.jobs.errors import JobPermanentError +from app.jobs.handlers.hydrate_metadata import build_hydrate_metadata_handler + + +def _parse_payload(payload_json: str) -> dict[str, Any]: + try: + data = json.loads(payload_json) + except Exception as exc: + raise JobPermanentError("payload_json is not valid JSON") from exc + if not isinstance(data, dict): + raise JobPermanentError("payload_json must be an object") + return data + + +def build_heal_url_handler(engine: AsyncEngine, *, transport: httpx.BaseTransport | None = None) -> Any: + hydrate = build_hydrate_metadata_handler(engine, transport=transport) + Session = create_sessionmaker(engine) + + async def _handler(job: dict[str, Any]) -> None: + payload_json = str(job.get("payload_json") or "") + payload = _parse_payload(payload_json) + + try: + illust_id = int(payload.get("illust_id")) + except Exception as exc: + raise JobPermanentError("payload.illust_id is required") from exc + if illust_id <= 0: + raise JobPermanentError("payload.illust_id is required") + + await hydrate(job) + + now_iso = iso_utc_ms(datetime.now(timezone.utc)) + + async def _op() -> None: + async with Session() as session: + await session.execute( + sa.update(Image) + .where(Image.illust_id == int(illust_id)) + .where(Image.status == 3) + .values( + status=1, + last_ok_at=now_iso, + last_error_code=None, + last_error_msg=None, + updated_at=now_iso, + ) + ) + await session.commit() + + await with_sqlite_busy_retry(_op) + + return _handler + diff --git a/backend/app/jobs/handlers/hydrate_metadata.py b/backend/app/jobs/handlers/hydrate_metadata.py new file mode 100644 index 0000000..1e66df9 --- /dev/null +++ b/backend/app/jobs/handlers/hydrate_metadata.py @@ -0,0 +1,1495 @@ +from __future__ import annotations + +import asyncio +import json +import os +import random +import time +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any + +import httpx +import sqlalchemy as sa +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.config import load_settings +from app.core.crypto import FieldEncryptor, mask_secret +from app.core.errors import ApiError, ErrorCode +from app.core.failover import classify_pixiv_rate_limit, pixiv_rate_limit_backoff_seconds +from app.core.metrics import TOKEN_REFRESH_FAIL_TOTAL +from app.core.proxy_routing import select_proxy_uri_for_url +from app.core.redact import redact_text +from app.core.runtime_settings import RuntimeConfig, load_runtime_config +from app.core.time import iso_utc_ms +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.hydration_runs import HydrationRun +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.tags import Tag +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker, with_sqlite_busy_retry +from app.jobs.errors import JobDeferError, JobPermanentError +from app.pixiv.access_token_cache import AccessTokenCache +from app.pixiv.oauth import OAUTH_TOKEN_PATH, PixivOauthConfig, PixivOauthError, refresh_access_token +from app.pixiv.refresh_backoff import refresh_backoff_seconds +from app.pixiv.token_strategy import NoTokenAvailable, TokenCandidate, choose_token + +PIXIV_APP_API_BASE_URL = "https://app-api.pixiv.net" +PIXIV_ILLUST_DETAIL_URL = PIXIV_APP_API_BASE_URL + "/v1/illust/detail" + +_MAX_TAGS = 200 + + +@dataclass(frozen=True, slots=True) +class _IllustPage: + page_index: int + original_url: str + ext: str + + +class TokenDisabledError(RuntimeError): + pass + + +def _parse_payload(payload_json: str) -> dict[str, Any]: + try: + data = json.loads(payload_json) + except Exception as exc: + raise JobPermanentError("payload_json is not valid JSON") from exc + if not isinstance(data, dict): + raise JobPermanentError("payload_json must be an object") + return data + + +def _parse_iso_utc_to_epoch(value: str, *, now_epoch: float) -> float: + raw = (value or "").strip() + if not raw: + return 0.0 + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(raw) + except Exception: + return now_epoch + 24 * 3600 + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return float(dt.astimezone(timezone.utc).timestamp()) + + +def _normalize_iso_utc_seconds(value: str | None) -> str | None: + raw = (value or "").strip() + if not raw: + return None + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + dt = datetime.fromisoformat(raw) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + dt = dt.astimezone(timezone.utc).replace(microsecond=0) + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _as_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool): + return None + try: + return int(value) + except Exception: + return None + + +def _as_str(value: Any) -> str | None: + if value is None: + return None + s = str(value).strip() + return s if s else None + + +def _derive_orientation(width: int | None, height: int | None) -> tuple[float | None, int | None]: + if width is None or height is None or width <= 0 or height <= 0: + return None, None + if width > height: + orientation = 2 + elif height > width: + orientation = 1 + else: + orientation = 3 + return float(width) / float(height), orientation + + +def _extract_tags(illust: dict[str, Any]) -> list[tuple[str, str | None]]: + raw_tags = illust.get("tags") + if not isinstance(raw_tags, list): + return [] + out: list[tuple[str, str | None]] = [] + seen: set[str] = set() + for raw in raw_tags[: _MAX_TAGS * 2]: + if not isinstance(raw, dict): + continue + name = _as_str(raw.get("name")) + if not name or name in seen: + continue + seen.add(name) + translated = _as_str(raw.get("translated_name")) + out.append((name, translated)) + if len(out) >= _MAX_TAGS: + break + return out + + +def _extract_original_urls(illust: dict[str, Any], *, page_count: int) -> list[str]: + urls: list[str] = [] + if page_count <= 1: + meta_single = illust.get("meta_single_page") + if isinstance(meta_single, dict): + url = _as_str(meta_single.get("original_image_url")) + if url: + return [url] + meta_pages = illust.get("meta_pages") + if isinstance(meta_pages, list) and meta_pages: + first = meta_pages[0] + if isinstance(first, dict): + image_urls = first.get("image_urls") + if isinstance(image_urls, dict): + url = _as_str(image_urls.get("original")) + if url: + return [url] + raise ValueError("missing original_image_url") + + meta_pages = illust.get("meta_pages") + if not isinstance(meta_pages, list) or not meta_pages: + raise ValueError("missing meta_pages") + + for idx in range(page_count): + page = meta_pages[idx] if idx < len(meta_pages) else None + if not isinstance(page, dict): + raise ValueError("invalid meta_pages") + image_urls = page.get("image_urls") + if not isinstance(image_urls, dict): + raise ValueError("invalid meta_pages.image_urls") + url = _as_str(image_urls.get("original")) + if not url: + raise ValueError("missing meta_pages.image_urls.original") + urls.append(url) + return urls + + +def _parse_pximg_ext(url: str) -> str: + from app.core.pixiv_urls import parse_pixiv_original_url + + parsed = parse_pixiv_original_url(url) + return str(parsed.ext) + + +def _parse_source_import_id(job: dict[str, Any]) -> int | None: + if str(job.get("ref_type") or "").strip() != "import": + return None + ref_id = str(job.get("ref_id") or "").strip() + if not ref_id: + return None + if ":" in ref_id: + prefix, _rest = ref_id.split(":", 1) + else: + prefix = ref_id + try: + import_id = int(prefix) + except Exception: + return None + return import_id if import_id > 0 else None + + +def build_hydrate_metadata_handler( + engine: AsyncEngine, + *, + transport: httpx.BaseTransport | None = None, + token_strategy: str = "least_error", +) -> Any: + settings = load_settings() + encryptor = FieldEncryptor.from_key(settings.field_encryption_key) + client_id = (settings.pixiv_oauth_client_id or "").strip() + client_secret = (settings.pixiv_oauth_client_secret or "").strip() + if not client_id or not client_secret: + raise ValueError("Pixiv OAuth not configured (PIXIV_OAUTH_CLIENT_ID/PIXIV_OAUTH_CLIENT_SECRET)") + oauth_config = PixivOauthConfig( + client_id=client_id, + client_secret=client_secret, + hash_secret=(settings.pixiv_oauth_hash_secret or "").strip() or None, + ) + + Session = create_sessionmaker(engine) + + token_cache = AccessTokenCache() + choose_lock = asyncio.Lock() + last_token_id: int | None = None + pixiv_throttle_global_lock = asyncio.Lock() + last_pixiv_request_m_global: float = 0.0 + pixiv_throttle_locks_guard = asyncio.Lock() + pixiv_throttle_locks_by_token: dict[int, asyncio.Lock] = {} + last_pixiv_request_m_by_token: dict[int, float] = {} + + def _env_int(name: str, *, default: int, min_v: int, max_v: int) -> int: + raw = (os.environ.get(name) or "").strip() + try: + value = int(raw) + except Exception: + value = int(default) + return max(int(min_v), min(int(value), int(max_v))) + + proxy_blacklist_ttl_s = _env_int( + "HYDRATE_PROXY_BLACKLIST_TTL_S", + default=5 * 60, + min_v=0, + max_v=24 * 60 * 60, + ) + proxy_override_ttl_s = _env_int( + "HYDRATE_PROXY_OVERRIDE_TTL_S", + default=30 * 60, + min_v=0, + max_v=7 * 24 * 60 * 60, + ) + proxy_failover_attempts = _env_int( + "HYDRATE_PROXY_FAILOVER_ATTEMPTS", + default=4, + min_v=0, + max_v=50, + ) + recoverable_defer_base_s = _env_int( + "HYDRATE_RECOVERABLE_DEFER_BASE_S", + default=20, + min_v=1, + max_v=24 * 60 * 60, + ) + recoverable_defer_jitter_s = _env_int( + "HYDRATE_RECOVERABLE_DEFER_JITTER_S", + default=20, + min_v=0, + max_v=24 * 60 * 60, + ) + + def _truncate(text: str, *, max_len: int = 500) -> str: + if len(text) <= max_len: + return text + return text[: max_len - 3] + "..." + + async def _mark_proxy_ok(endpoint_id: int, *, latency_ms: float | None, now_iso: str) -> None: + if int(endpoint_id) <= 0: + return + + async def _op() -> None: + async with Session() as session: + await session.execute( + sa.update(ProxyEndpoint) + .where(ProxyEndpoint.id == int(endpoint_id)) + .values( + last_latency_ms=float(latency_ms) if latency_ms is not None else None, + last_ok_at=now_iso, + success_count=ProxyEndpoint.success_count + 1, + last_error=None, + blacklisted_until=None, + updated_at=now_iso, + ) + ) + await session.commit() + + await with_sqlite_busy_retry(_op) + + async def _mark_proxy_fail( + endpoint_id: int, + *, + latency_ms: float | None, + now_dt: datetime, + error: BaseException | str, + ) -> None: + if int(endpoint_id) <= 0: + return + + now_iso = iso_utc_ms(now_dt) + blacklist_until_iso = ( + iso_utc_ms(now_dt + timedelta(seconds=int(proxy_blacklist_ttl_s))) + if int(proxy_blacklist_ttl_s) > 0 + else None + ) + msg_raw = f"{type(error).__name__}: {error}" if isinstance(error, BaseException) else str(error) + msg = _truncate(redact_text(msg_raw)) + + async def _op() -> None: + async with Session() as session: + if blacklist_until_iso: + blacklist_expr = sa.case( + ( + sa.and_( + ProxyEndpoint.blacklisted_until.isnot(None), + ProxyEndpoint.blacklisted_until > blacklist_until_iso, + ), + ProxyEndpoint.blacklisted_until, + ), + else_=blacklist_until_iso, + ) + else: + blacklist_expr = ProxyEndpoint.blacklisted_until + + await session.execute( + sa.update(ProxyEndpoint) + .where(ProxyEndpoint.id == int(endpoint_id)) + .values( + last_latency_ms=float(latency_ms) if latency_ms is not None else None, + last_fail_at=now_iso, + failure_count=ProxyEndpoint.failure_count + 1, + blacklisted_until=blacklist_expr, + last_error=msg, + updated_at=now_iso, + ) + ) + await session.commit() + + await with_sqlite_busy_retry(_op) + + async def _set_token_proxy_override( + *, + token_id: int, + pool_id: int, + endpoint_id: int, + now_dt: datetime, + ) -> None: + if int(proxy_override_ttl_s) <= 0: + return + if int(token_id) <= 0 or int(pool_id) <= 0 or int(endpoint_id) <= 0: + return + + now_iso = iso_utc_ms(now_dt) + expires_at = iso_utc_ms(now_dt + timedelta(seconds=int(proxy_override_ttl_s))) + + async def _op() -> None: + async with Session() as session: + await session.execute( + sa.update(TokenProxyBinding) + .where( + sa.and_( + TokenProxyBinding.token_id == int(token_id), + TokenProxyBinding.pool_id == int(pool_id), + ) + ) + .values( + override_proxy_id=int(endpoint_id), + override_expires_at=expires_at, + updated_at=now_iso, + ) + ) + await session.commit() + + await with_sqlite_busy_retry(_op) + + def _is_recoverable_exc(exc: BaseException) -> bool: + if isinstance(exc, httpx.RequestError): + return True + if isinstance(exc, ApiError) and exc.code in { + ErrorCode.PROXY_REQUIRED, + ErrorCode.PROXY_CONNECT_FAILED, + ErrorCode.PROXY_AUTH_FAILED, + }: + return True + if isinstance(exc, PixivOauthError): + status = exc.status_code + return status is None or int(status) >= 500 + if isinstance(exc, httpx.HTTPStatusError): + status_code = int(getattr(getattr(exc, "response", None), "status_code", 0) or 0) + return status_code >= 500 + return False + + def _recoverable_defer_run_after_iso() -> str: + base = float(recoverable_defer_base_s) + jitter = float(max(0, int(recoverable_defer_jitter_s))) + delay_s = base + random.random() * jitter + retry_dt = datetime.now(timezone.utc) + timedelta(seconds=float(delay_s)) + return iso_utc_ms(retry_dt) + + def _defer_run_after_for_exc(exc: BaseException, *, now_epoch: float) -> str: + if isinstance(exc, ApiError) and exc.code == ErrorCode.PROXY_REQUIRED: + details = exc.details or {} + next_available_at_raw = details.get("next_available_at") + if isinstance(next_available_at_raw, str): + next_available_at = next_available_at_raw.strip() + epoch = _parse_iso_utc_to_epoch(next_available_at, now_epoch=now_epoch) + if epoch is not None and float(epoch) > float(now_epoch): + return next_available_at + return _recoverable_defer_run_after_iso() + + def _rate_limit_int( + runtime: RuntimeConfig, + key: str, + *, + default: int, + min_v: int, + max_v: int, + ) -> int: + raw = (runtime.rate_limit or {}).get(key) + try: + value = int(raw) + except Exception: + return int(default) + return max(int(min_v), min(int(value), int(max_v))) + + async def _get_pixiv_token_lock(token_id: int) -> asyncio.Lock: + lock = pixiv_throttle_locks_by_token.get(int(token_id)) + if lock is not None: + return lock + async with pixiv_throttle_locks_guard: + lock2 = pixiv_throttle_locks_by_token.get(int(token_id)) + if lock2 is not None: + return lock2 + created = asyncio.Lock() + pixiv_throttle_locks_by_token[int(token_id)] = created + return created + + async def _pixiv_throttle(runtime: RuntimeConfig, *, token_id: int | None) -> None: + nonlocal last_pixiv_request_m_global + + default_min_ms = 800 if transport is None else 0 + default_jitter_ms = 200 if transport is None else 0 + + min_interval_ms = _rate_limit_int( + runtime, + "pixiv_hydrate_min_interval_ms", + default=int(default_min_ms), + min_v=0, + max_v=60_000, + ) + jitter_ms = _rate_limit_int( + runtime, + "pixiv_hydrate_jitter_ms", + default=int(default_jitter_ms), + min_v=0, + max_v=60_000, + ) + if min_interval_ms <= 0 and jitter_ms <= 0: + return + + interval_s = (float(min_interval_ms) + random.random() * float(max(0, jitter_ms))) / 1000.0 + + token_id_i = int(token_id or 0) + if token_id_i > 0: + lock = await _get_pixiv_token_lock(int(token_id_i)) + async with lock: + now_m = float(time.monotonic()) + last_m = float(last_pixiv_request_m_by_token.get(int(token_id_i), 0.0)) + wait_s = (last_m + float(interval_s)) - now_m + if wait_s > 0: + await asyncio.sleep(float(wait_s)) + last_pixiv_request_m_by_token[int(token_id_i)] = float(time.monotonic()) + return + + async with pixiv_throttle_global_lock: + now_m = float(time.monotonic()) + wait_s = (float(last_pixiv_request_m_global) + float(interval_s)) - now_m + if wait_s > 0: + await asyncio.sleep(float(wait_s)) + last_pixiv_request_m_global = float(time.monotonic()) + + def _as_int(value: Any, *, default: int = 0) -> int: + try: + return int(value) + except Exception: + return default + + def _missing_set_from_criteria(criteria: dict[str, Any]) -> set[str]: + default = {"tags", "geometry", "r18", "ai", "illust_type", "user", "title", "created_at", "popularity"} + raw = criteria.get("missing") + if not isinstance(raw, list): + return set(default) + out: set[str] = set() + for item in raw: + if not isinstance(item, str): + continue + v = item.strip().lower() + if not v: + continue + if v in {"all", "*"}: + return set(default) + out.add(v) + return out or set(default) + + def _build_missing_predicate_sql(missing: set[str]) -> str: + parts: list[str] = [] + + if "geometry" in missing: + parts.append("(width IS NULL OR height IS NULL OR orientation IS NULL OR aspect_ratio IS NULL)") + if "r18" in missing: + parts.append("(x_restrict IS NULL)") + if "ai" in missing: + parts.append("(ai_type IS NULL)") + if "illust_type" in missing: + parts.append("(illust_type IS NULL)") + if "user" in missing: + parts.append("(user_id IS NULL)") + if "title" in missing: + parts.append("(title IS NULL)") + if "created_at" in missing: + parts.append("(created_at_pixiv IS NULL)") + if "tags" in missing: + parts.append("NOT EXISTS (SELECT 1 FROM image_tags it WHERE it.image_id = images.id)") + if "popularity" in missing: + parts.append("(bookmark_count IS NULL OR view_count IS NULL OR comment_count IS NULL)") + + return "(" + " OR ".join(parts) + ")" if parts else "(1=1)" + + async def _load_run_state(run_id: int) -> dict[str, Any]: + Session = create_sessionmaker(engine) + + async def _op() -> dict[str, Any]: + async with Session() as session: + run = await session.get(HydrationRun, int(run_id)) + if run is None: + raise JobPermanentError("Hydration run not found") + + criteria: dict[str, Any] = {} + try: + criteria_raw = json.loads(run.criteria_json or "{}") + if isinstance(criteria_raw, dict): + criteria = dict(criteria_raw) + except Exception: + criteria = {} + + cursor_image_id = 0 + try: + cursor_raw = json.loads(run.cursor_json or "{}") + if isinstance(cursor_raw, dict): + cursor_image_id = _as_int(cursor_raw.get("cursor_image_id"), default=0) + except Exception: + cursor_image_id = 0 + + return { + "status": str(run.status), + "criteria": criteria, + "cursor_image_id": int(max(0, cursor_image_id)), + "processed": int(run.processed or 0), + "success": int(run.success or 0), + "failed": int(run.failed or 0), + "started_at": run.started_at, + "finished_at": run.finished_at, + } + + return await with_sqlite_busy_retry(_op) + + async def _mark_run_running(run_id: int) -> None: + now_iso = iso_utc_ms() + Session = create_sessionmaker(engine) + + async def _op() -> None: + async with Session() as session: + run = await session.get(HydrationRun, int(run_id)) + if run is None: + raise JobPermanentError("Hydration run not found") + if str(run.status) not in {"pending", "running"}: + return + run.status = "running" + if run.started_at is None: + run.started_at = now_iso + run.updated_at = now_iso + run.last_error = None + await session.commit() + + await with_sqlite_busy_retry(_op) + + async def _update_run_progress( + run_id: int, + *, + cursor_image_id: int, + cursor_illust_id: int, + processed_inc: int, + success_inc: int, + failed_inc: int, + last_error: str | None, + ) -> None: + now_iso = iso_utc_ms() + cursor_json = json.dumps( + {"cursor_image_id": int(cursor_image_id), "cursor_illust_id": int(cursor_illust_id)}, + separators=(",", ":"), + ensure_ascii=False, + ) + Session = create_sessionmaker(engine) + + async def _op() -> None: + async with Session() as session: + run = await session.get(HydrationRun, int(run_id)) + if run is None: + raise JobPermanentError("Hydration run not found") + run.cursor_json = cursor_json + run.processed = int(run.processed or 0) + int(processed_inc) + run.success = int(run.success or 0) + int(success_inc) + run.failed = int(run.failed or 0) + int(failed_inc) + run.updated_at = now_iso + if last_error is not None: + run.last_error = (str(last_error) or "")[:500] + await session.commit() + + await with_sqlite_busy_retry(_op) + + async def _mark_run_completed(run_id: int) -> None: + now_iso = iso_utc_ms() + Session = create_sessionmaker(engine) + + async def _op() -> None: + async with Session() as session: + run = await session.get(HydrationRun, int(run_id)) + if run is None: + raise JobPermanentError("Hydration run not found") + if str(run.status) not in {"pending", "running"}: + return + run.status = "completed" + if run.finished_at is None: + run.finished_at = now_iso + run.updated_at = now_iso + run.last_error = None + await session.commit() + + await with_sqlite_busy_retry(_op) + + async def _release_job_lock( + *, + job_id: int, + worker_id: str, + status: str, + run_after: str | None, + ) -> bool: + sql = """ +UPDATE jobs +SET status=:status, + run_after=:run_after, + last_error=NULL, + locked_by=NULL, + locked_at=NULL, + updated_at=:now +WHERE id=:id AND status='running' AND locked_by=:worker_id; +""".strip() + + async def _op() -> bool: + async with engine.begin() as conn: + result = await conn.exec_driver_sql( + sql, + { + "status": str(status), + "run_after": run_after, + "now": iso_utc_ms(), + "id": int(job_id), + "worker_id": worker_id, + }, + ) + return (result.rowcount or 0) == 1 + + return await with_sqlite_busy_retry(_op) + + async def _pick_next_candidate( + *, + cursor_image_id: int, + missing_predicate_sql: str, + ) -> tuple[int, int] | None: + sql = f""" +SELECT id, illust_id +FROM images +WHERE status=1 + AND id > :cursor + AND {missing_predicate_sql} +ORDER BY id ASC +LIMIT 1; +""".strip() + + async def _op() -> tuple[int, int] | None: + async with engine.connect() as conn: + result = await conn.exec_driver_sql(sql, {"cursor": int(cursor_image_id)}) + row = result.first() + if row is None: + return None + return int(row[0]), int(row[1]) + + return await with_sqlite_busy_retry(_op) + + async def _load_tokens(now_epoch: float) -> list[TokenCandidate]: + async with Session() as session: + rows = ( + (await session.execute(sa.select(PixivToken).order_by(PixivToken.id.asc()))) + .scalars() + .all() + ) + out: list[TokenCandidate] = [] + for row in rows: + out.append( + TokenCandidate( + id=int(row.id), + enabled=bool(row.enabled), + weight=float(row.weight or 0.0), + error_count=int(row.error_count or 0), + backoff_until=_parse_iso_utc_to_epoch(str(row.backoff_until or ""), now_epoch=now_epoch), + ) + ) + return out + + async def _choose_token_id(*, now_epoch: float, exclude_ids: set[int]) -> int: + nonlocal last_token_id + tokens = await _load_tokens(now_epoch) + tokens2 = [t for t in tokens if int(t.id) not in exclude_ids] + try: + async with choose_lock: + chosen, new_last = choose_token( + tokens2, + strategy=token_strategy, + now=now_epoch, + last_id=last_token_id, + r=random.random(), + ) + last_token_id = int(new_last) + return int(chosen.id) + except NoTokenAvailable as exc: + retry_at = exc.next_retry_at + if retry_at is None: + retry_at = now_epoch + 60.0 + retry_dt = datetime.fromtimestamp(float(retry_at), tz=timezone.utc) + raise JobDeferError( + f"{ErrorCode.NO_TOKEN_AVAILABLE.value}: No eligible token available", + run_after=iso_utc_ms(retry_dt), + ) from exc + + async def _get_refresh_token(token_id: int) -> str: + async with Session() as session: + row = await session.get(PixivToken, int(token_id)) + if row is None: + raise JobPermanentError("Token not found") + if not bool(row.enabled): + raise TokenDisabledError("Token disabled") + return encryptor.decrypt_text(str(row.refresh_token_enc)) + + async def _is_token_enabled(token_id: int) -> bool: + async with Session() as session: + row = await session.get(PixivToken, int(token_id)) + if row is None: + raise JobPermanentError("Token not found") + return bool(row.enabled) + + async def _mark_token_backoff( + token_id: int, + *, + now_dt: datetime, + attempt: int, + backoff_s: int, + code: str, + message: str, + rotated_refresh_token: str | None = None, + ) -> str | None: + now_iso = iso_utc_ms(now_dt) + backoff_until = iso_utc_ms(now_dt + timedelta(seconds=int(backoff_s))) if backoff_s > 0 else None + msg = (message or "")[:500] + + async def _op() -> None: + async with Session() as session: + row = await session.get(PixivToken, int(token_id)) + if row is None: + return + row.error_count = int(attempt) + row.backoff_until = backoff_until + row.last_fail_at = now_iso + row.last_error_code = str(code) + row.last_error_msg = msg + row.updated_at = now_iso + if rotated_refresh_token: + row.refresh_token_enc = encryptor.encrypt_text(rotated_refresh_token) + row.refresh_token_masked = mask_secret(rotated_refresh_token) + await session.commit() + + await with_sqlite_busy_retry(_op) + return backoff_until + + async def _rotate_refresh_token(token_id: int, *, rotated_refresh_token: str, now_dt: datetime) -> None: + rotated_refresh_token = (rotated_refresh_token or "").strip() + if not rotated_refresh_token: + return + + now_iso = iso_utc_ms(now_dt) + + async def _op() -> None: + async with Session() as session: + row = await session.get(PixivToken, int(token_id)) + if row is None: + return + row.refresh_token_enc = encryptor.encrypt_text(rotated_refresh_token) + row.refresh_token_masked = mask_secret(rotated_refresh_token) + row.updated_at = now_iso + await session.commit() + + await with_sqlite_busy_retry(_op) + + async def _mark_token_ok(token_id: int, *, now_dt: datetime) -> None: + now_iso = iso_utc_ms(now_dt) + + async def _op() -> None: + async with Session() as session: + row = await session.get(PixivToken, int(token_id)) + if row is None: + return + row.error_count = 0 + row.backoff_until = None + row.last_ok_at = now_iso + row.last_fail_at = None + row.last_error_code = None + row.last_error_msg = None + row.updated_at = now_iso + await session.commit() + + await with_sqlite_busy_retry(_op) + + async def _get_access_token(token_id: int, *, now_dt: datetime, runtime: RuntimeConfig) -> str: + if not await _is_token_enabled(int(token_id)): + raise TokenDisabledError("Token disabled") + + oauth_url = oauth_config.base_url.rstrip("/") + OAUTH_TOKEN_PATH + + async def refresher() -> Any: + if not await _is_token_enabled(int(token_id)): + raise TokenDisabledError("Token disabled") + refresh_token = await _get_refresh_token(token_id) + + last_exc: BaseException | None = None + max_tries = max(1, int(proxy_failover_attempts) + 1) + + for _try in range(max_tries): + attempt_now_dt = datetime.now(timezone.utc) + attempt_now_iso = iso_utc_ms(attempt_now_dt) + + proxy_uri = None + picked_proxy = await select_proxy_uri_for_url( + engine, + settings, + runtime, + url=oauth_url, + token_id=int(token_id), + ) + if picked_proxy is not None: + proxy_uri = picked_proxy.uri + + start_m = float(time.monotonic()) + try: + await _pixiv_throttle(runtime, token_id=int(token_id)) + token = await refresh_access_token( + refresh_token=refresh_token, + config=oauth_config, + transport=transport, + proxy=proxy_uri, + ) + except PixivOauthError as exc: + latency_ms = (float(time.monotonic()) - start_m) * 1000.0 + if picked_proxy is not None: + if exc.status_code is not None and int(exc.status_code) < 500: + await _mark_proxy_ok( + int(picked_proxy.endpoint_id), + latency_ms=float(latency_ms), + now_iso=str(attempt_now_iso), + ) + else: + await _mark_proxy_fail( + int(picked_proxy.endpoint_id), + latency_ms=float(latency_ms), + now_dt=attempt_now_dt, + error=exc, + ) + + if exc.status_code is None or int(exc.status_code) >= 500: + last_exc = exc + if picked_proxy is None: + break + continue + raise + except httpx.RequestError as exc: + latency_ms = (float(time.monotonic()) - start_m) * 1000.0 + if picked_proxy is not None: + await _mark_proxy_fail( + int(picked_proxy.endpoint_id), + latency_ms=float(latency_ms), + now_dt=attempt_now_dt, + error=exc, + ) + last_exc = exc + if picked_proxy is None: + break + continue + else: + latency_ms = (float(time.monotonic()) - start_m) * 1000.0 + if picked_proxy is not None: + await _mark_proxy_ok( + int(picked_proxy.endpoint_id), + latency_ms=float(latency_ms), + now_iso=str(attempt_now_iso), + ) + await _set_token_proxy_override( + token_id=int(token_id), + pool_id=int(picked_proxy.pool_id), + endpoint_id=int(picked_proxy.endpoint_id), + now_dt=attempt_now_dt, + ) + + rotated = token.refresh_token + if rotated: + await _rotate_refresh_token(token_id, rotated_refresh_token=rotated, now_dt=attempt_now_dt) + return token + + if last_exc is not None: + raise last_exc + raise PixivOauthError("OAuth refresh failed", status_code=None) + + token = await token_cache.get_or_refresh(token_id, refresher=refresher) + if not await _is_token_enabled(int(token_id)): + token_cache.invalidate(token_id) + raise TokenDisabledError("Token disabled") + return str(token.access_token) + + async def _fetch_illust_detail( + *, + illust_id: int, + access_token: str, + token_id: int, + runtime: RuntimeConfig, + now_dt: datetime, + ) -> dict[str, Any]: + headers = oauth_config.build_headers(client_time=datetime.now(timezone.utc).isoformat(timespec="seconds")) + headers["Authorization"] = f"Bearer {access_token}" + + last_exc: BaseException | None = None + max_tries = max(1, int(proxy_failover_attempts) + 1) + + for _try in range(max_tries): + attempt_now_dt = datetime.now(timezone.utc) + attempt_now_iso = iso_utc_ms(attempt_now_dt) + + proxy_uri = None + picked_proxy = await select_proxy_uri_for_url( + engine, + settings, + runtime, + url=PIXIV_ILLUST_DETAIL_URL, + token_id=int(token_id), + ) + if picked_proxy is not None: + proxy_uri = picked_proxy.uri + + client_kwargs: dict[str, Any] = { + "timeout": httpx.Timeout(30.0, connect=10.0), + "follow_redirects": True, + } + if transport is not None: + client_kwargs["transport"] = transport + if proxy_uri: + client_kwargs["proxy"] = proxy_uri + + start_m = float(time.monotonic()) + try: + await _pixiv_throttle(runtime, token_id=int(token_id)) + async with httpx.AsyncClient(**client_kwargs) as client: + resp = await client.get( + PIXIV_ILLUST_DETAIL_URL, + params={"illust_id": int(illust_id), "filter": "for_android"}, + headers=headers, + ) + except httpx.RequestError as exc: + latency_ms = (float(time.monotonic()) - start_m) * 1000.0 + if picked_proxy is not None: + await _mark_proxy_fail( + int(picked_proxy.endpoint_id), + latency_ms=float(latency_ms), + now_dt=attempt_now_dt, + error=exc, + ) + last_exc = exc + if picked_proxy is None: + break + continue + + latency_ms = (float(time.monotonic()) - start_m) * 1000.0 + if picked_proxy is not None: + if int(resp.status_code) >= 500: + await _mark_proxy_fail( + int(picked_proxy.endpoint_id), + latency_ms=float(latency_ms), + now_dt=attempt_now_dt, + error=f"status={int(resp.status_code)}", + ) + else: + await _mark_proxy_ok( + int(picked_proxy.endpoint_id), + latency_ms=float(latency_ms), + now_iso=str(attempt_now_iso), + ) + + if resp.status_code == 200: + if picked_proxy is not None: + await _set_token_proxy_override( + token_id=int(token_id), + pool_id=int(picked_proxy.pool_id), + endpoint_id=int(picked_proxy.endpoint_id), + now_dt=attempt_now_dt, + ) + + try: + data = resp.json() + except Exception as exc: + raise ValueError("Pixiv App API response is not JSON") from exc + if not isinstance(data, dict): + raise ValueError("Pixiv App API response invalid") + return data + + http_exc = httpx.HTTPStatusError( + f"Pixiv App API error status={resp.status_code}", + request=resp.request, + response=resp, + ) + if int(resp.status_code) >= 500: + last_exc = http_exc + if picked_proxy is None: + break + continue + raise http_exc + + if last_exc is not None: + raise last_exc + raise RuntimeError("Pixiv App API request failed") + + async def _persist( + *, + illust_id: int, + pages: list[_IllustPage], + width: int | None, + height: int | None, + aspect_ratio: float | None, + orientation: int | None, + x_restrict: int | None, + ai_type: int | None, + illust_type: int | None, + user_id: int | None, + user_name: str | None, + title: str | None, + created_at_pixiv: str | None, + bookmark_count: int | None, + view_count: int | None, + comment_count: int | None, + tags: list[tuple[str, str | None]], + source_import_id: int | None, + ) -> None: + now_expr = sa.text("(strftime('%Y-%m-%dT%H:%M:%fZ','now'))") + normalized_tag_names = [name for name, _t in tags] + + async def _op() -> None: + async with Session() as session: + existing = {} + if normalized_tag_names: + rows = ( + ( + await session.execute( + sa.select(Tag).where(Tag.name.in_(normalized_tag_names)) + ) + ) + .scalars() + .all() + ) + existing = {str(t.name): t for t in rows} + + tag_ids: dict[str, int] = {} + for name, translated in tags: + row = existing.get(name) + if row is None: + row = Tag(name=name, translated_name=translated) + session.add(row) + await session.flush() + existing[name] = row + else: + if translated is not None and translated != row.translated_name: + row.translated_name = translated + row.updated_at = iso_utc_ms() + tag_ids[name] = int(row.id) + + image_ids: list[int] = [] + for page in pages: + stmt = sqlite_insert(Image).values( + illust_id=int(illust_id), + page_index=int(page.page_index), + ext=str(page.ext), + original_url=str(page.original_url), + proxy_path="", + random_key=random.random(), + width=width, + height=height, + aspect_ratio=aspect_ratio, + orientation=orientation, + x_restrict=x_restrict, + ai_type=ai_type, + illust_type=illust_type, + user_id=user_id, + user_name=user_name, + title=title, + created_at_pixiv=created_at_pixiv, + bookmark_count=bookmark_count, + view_count=view_count, + comment_count=comment_count, + created_import_id=int(source_import_id) if source_import_id else None, + ) + stmt = stmt.on_conflict_do_update( + index_elements=["illust_id", "page_index"], + set_={ + "ext": stmt.excluded.ext, + "original_url": stmt.excluded.original_url, + "width": stmt.excluded.width, + "height": stmt.excluded.height, + "aspect_ratio": stmt.excluded.aspect_ratio, + "orientation": stmt.excluded.orientation, + "x_restrict": stmt.excluded.x_restrict, + "ai_type": stmt.excluded.ai_type, + "illust_type": stmt.excluded.illust_type, + "user_id": stmt.excluded.user_id, + "user_name": stmt.excluded.user_name, + "title": stmt.excluded.title, + "created_at_pixiv": stmt.excluded.created_at_pixiv, + "bookmark_count": stmt.excluded.bookmark_count, + "view_count": stmt.excluded.view_count, + "comment_count": stmt.excluded.comment_count, + "updated_at": now_expr, + }, + ).returning(Image.id) + + result = await session.execute(stmt) + image_id = int(result.scalar_one()) + image_ids.append(image_id) + + proxy_path = f"/i/{image_id}.{page.ext}" + await session.execute( + sa.update(Image).where(Image.id == image_id).values(proxy_path=proxy_path) + ) + + if image_ids: + await session.execute(sa.delete(ImageTag).where(ImageTag.image_id.in_(image_ids))) + if tag_ids: + values = [ + {"image_id": int(img_id), "tag_id": int(tag_id)} + for img_id in image_ids + for tag_id in tag_ids.values() + ] + stmt2 = sqlite_insert(ImageTag).values(values).on_conflict_do_nothing() + await session.execute(stmt2) + + await session.commit() + + await with_sqlite_busy_retry(_op) + + async def _hydrate_single_illust(*, illust_id: int, source_import_id: int | None) -> None: + now_dt = datetime.now(timezone.utc) + now_epoch = float(time.time()) + runtime = await load_runtime_config(engine) + + tried: set[int] = set() + last_exc: BaseException | None = None + + for _ in range(0, 10): + try: + token_id = await _choose_token_id(now_epoch=now_epoch, exclude_ids=tried) + except JobDeferError as exc: + if last_exc is not None and _is_recoverable_exc(last_exc): + code = ErrorCode.PROXY_CONNECT_FAILED + if isinstance(last_exc, ApiError): + code = last_exc.code + raise JobDeferError( + f"{code.value}: 代理/网络异常，稍后重试", + run_after=_defer_run_after_for_exc(last_exc, now_epoch=now_epoch), + ) from last_exc + raise + tried.add(int(token_id)) + + try: + access_token = await _get_access_token(token_id, now_dt=now_dt, runtime=runtime) + except TokenDisabledError as exc: + last_exc = exc + continue + except ApiError as exc: + last_exc = exc + continue + except httpx.RequestError as exc: + TOKEN_REFRESH_FAIL_TOTAL.inc() + last_exc = exc + continue + except PixivOauthError as exc: + TOKEN_REFRESH_FAIL_TOTAL.inc() + if exc.status_code is None or int(exc.status_code) >= 500: + last_exc = exc + continue + attempt = 0 + async with Session() as session: + row = await session.get(PixivToken, int(token_id)) + if row is not None: + attempt = int(row.error_count or 0) + 1 + backoff_s = refresh_backoff_seconds(attempt=attempt, status_code=exc.status_code) + await _mark_token_backoff( + token_id, + now_dt=now_dt, + attempt=attempt, + backoff_s=backoff_s, + code=ErrorCode.TOKEN_REFRESH_FAILED.value, + message="Token refresh failed", + ) + last_exc = exc + continue + except Exception as exc: + TOKEN_REFRESH_FAIL_TOTAL.inc() + attempt = 0 + async with Session() as session: + row = await session.get(PixivToken, int(token_id)) + if row is not None: + attempt = int(row.error_count or 0) + 1 + backoff_s = refresh_backoff_seconds(attempt=attempt, status_code=None) + await _mark_token_backoff( + token_id, + now_dt=now_dt, + attempt=attempt, + backoff_s=backoff_s, + code=ErrorCode.TOKEN_REFRESH_FAILED.value, + message="Token refresh failed", + ) + last_exc = exc + continue + + try: + data = await _fetch_illust_detail( + illust_id=illust_id, + access_token=access_token, + token_id=int(token_id), + runtime=runtime, + now_dt=now_dt, + ) + except httpx.HTTPStatusError as exc: + status = int(getattr(exc.response, "status_code", 0) or 0) + body_text = getattr(exc.response, "text", None) + + if status == 404: + raise JobPermanentError("Pixiv illust not found") from exc + + rate_kind = classify_pixiv_rate_limit(status_code=status, body_text=body_text) + if rate_kind is not None: + attempt = 0 + async with Session() as session: + row = await session.get(PixivToken, int(token_id)) + if row is not None: + attempt = int(row.error_count or 0) + 1 + backoff_s = pixiv_rate_limit_backoff_seconds(attempt=attempt) + backoff_until = await _mark_token_backoff( + token_id, + now_dt=now_dt, + attempt=attempt, + backoff_s=backoff_s, + code=ErrorCode.TOKEN_BACKOFF.value, + message="Pixiv rate limited", + ) + if backoff_until: + last_exc = JobDeferError("Pixiv rate limited", run_after=backoff_until) + continue + + last_exc = exc + continue + except ApiError as exc: + last_exc = exc + continue + except httpx.RequestError as exc: + last_exc = exc + continue + except Exception as exc: + last_exc = exc + continue + + illust = data.get("illust") if isinstance(data, dict) else None + if not isinstance(illust, dict): + raise JobPermanentError("Pixiv illust detail missing illust") + + page_count = _as_int(illust.get("page_count")) or 1 + if page_count <= 0 or page_count > 1000: + raise JobPermanentError("Pixiv illust detail invalid page_count") + + try: + urls = _extract_original_urls(illust, page_count=int(page_count)) + except Exception as exc: + raise JobPermanentError("Pixiv illust detail missing original urls") from exc + + pages: list[_IllustPage] = [] + for idx, url in enumerate(urls): + try: + ext = _parse_pximg_ext(url) + except Exception as exc: + raise JobPermanentError("Pixiv illust detail invalid original url") from exc + pages.append(_IllustPage(page_index=int(idx), original_url=str(url), ext=ext)) + + width = _as_int(illust.get("width")) + height = _as_int(illust.get("height")) + aspect_ratio, orientation = _derive_orientation(width, height) + + x_restrict = _as_int(illust.get("x_restrict")) + ai_type = _as_int(illust.get("illust_ai_type")) + if ai_type is None: + ai_type = _as_int(illust.get("ai_type")) + + illust_type = _as_int(illust.get("illust_type")) + if illust_type is None: + kind = _as_str(illust.get("type")) + if kind == "illust": + illust_type = 0 + elif kind == "manga": + illust_type = 1 + elif kind == "ugoira": + illust_type = 2 + if illust_type is not None and int(illust_type) not in {0, 1, 2}: + illust_type = None + + user = illust.get("user") + user_id = _as_int(user.get("id")) if isinstance(user, dict) else None + user_name = _as_str(user.get("name")) if isinstance(user, dict) else None + + title = _as_str(illust.get("title")) + created_at_pixiv = None + try: + created_at_pixiv = _normalize_iso_utc_seconds(_as_str(illust.get("create_date"))) + except Exception: + created_at_pixiv = None + + bookmark_count = _as_int(illust.get("total_bookmarks")) + if bookmark_count is None: + bookmark_count = _as_int(illust.get("bookmark_count")) + + view_count = _as_int(illust.get("total_view")) + if view_count is None: + view_count = _as_int(illust.get("view_count")) + + comment_count = _as_int(illust.get("total_comments")) + if comment_count is None: + comment_count = _as_int(illust.get("comment_count")) + + tags = _extract_tags(illust) + + await _persist( + illust_id=int(illust_id), + pages=pages, + width=width, + height=height, + aspect_ratio=aspect_ratio, + orientation=orientation, + x_restrict=x_restrict, + ai_type=ai_type, + illust_type=illust_type, + user_id=user_id, + user_name=user_name, + title=title, + created_at_pixiv=created_at_pixiv, + bookmark_count=bookmark_count, + view_count=view_count, + comment_count=comment_count, + tags=tags, + source_import_id=source_import_id, + ) + await _mark_token_ok(token_id, now_dt=now_dt) + return + + if isinstance(last_exc, JobDeferError): + raise last_exc + if last_exc is None: + raise RuntimeError("hydrate_metadata failed") + if _is_recoverable_exc(last_exc): + code = ErrorCode.PROXY_CONNECT_FAILED + if isinstance(last_exc, ApiError): + code = last_exc.code + raise JobDeferError( + f"{code.value}: 代理/网络异常，稍后重试", + run_after=_defer_run_after_for_exc(last_exc, now_epoch=now_epoch), + ) from last_exc + raise last_exc + + async def _handler(job: dict[str, Any]) -> None: + payload_json = str(job.get("payload_json") or "") + payload = _parse_payload(payload_json) + + hydration_run_id = _as_int(payload.get("hydration_run_id"), default=0) + if hydration_run_id > 0: + job_id = _as_int(job.get("id"), default=0) + worker_id = str(job.get("locked_by") or "").strip() + if job_id <= 0 or not worker_id: + raise JobPermanentError("Invalid job state") + + run_state = await _load_run_state(int(hydration_run_id)) + status = str(run_state.get("status") or "") + if status in {"paused", "canceled"}: + await _release_job_lock(job_id=int(job_id), worker_id=worker_id, status=status, run_after=None) + return + if status not in {"pending", "running"}: + await _release_job_lock(job_id=int(job_id), worker_id=worker_id, status="completed", run_after=None) + return + + await _mark_run_running(int(hydration_run_id)) + + criteria = run_state.get("criteria") if isinstance(run_state.get("criteria"), dict) else {} + missing = _missing_set_from_criteria(criteria) + missing_predicate_sql = _build_missing_predicate_sql(missing) + + cursor_image_id = int(run_state.get("cursor_image_id") or 0) + batch_size = _as_int(os.environ.get("HYDRATION_RUN_BATCH_SIZE"), default=10) + batch_size = max(1, min(int(batch_size), 200)) + + processed = 0 + for _ in range(batch_size): + candidate = await _pick_next_candidate( + cursor_image_id=int(cursor_image_id), + missing_predicate_sql=missing_predicate_sql, + ) + if candidate is None: + await _mark_run_completed(int(hydration_run_id)) + return + + image_id, illust_id = candidate + + try: + await _hydrate_single_illust(illust_id=int(illust_id), source_import_id=None) + except JobDeferError: + raise + except Exception as exc: + await _update_run_progress( + int(hydration_run_id), + cursor_image_id=int(image_id), + cursor_illust_id=int(illust_id), + processed_inc=1, + success_inc=0, + failed_inc=1, + last_error=f"{type(exc).__name__}: {exc}", + ) + cursor_image_id = int(image_id) + processed += 1 + continue + + await _update_run_progress( + int(hydration_run_id), + cursor_image_id=int(image_id), + cursor_illust_id=int(illust_id), + processed_inc=1, + success_inc=1, + failed_inc=0, + last_error=None, + ) + cursor_image_id = int(image_id) + processed += 1 + + if processed > 0: + run_after = iso_utc_ms(datetime.now(timezone.utc) + timedelta(seconds=1)) + await _release_job_lock(job_id=int(job_id), worker_id=worker_id, status="pending", run_after=run_after) + return + + try: + illust_id = int(payload.get("illust_id")) + except Exception as exc: + raise JobPermanentError("payload.illust_id is required") from exc + if illust_id <= 0: + raise JobPermanentError("payload.illust_id is required") + + source_import_id = _parse_source_import_id(job) + await _hydrate_single_illust(illust_id=int(illust_id), source_import_id=source_import_id) + + return _handler diff --git a/backend/app/jobs/handlers/import_images.py b/backend/app/jobs/handlers/import_images.py new file mode 100644 index 0000000..2dbdf67 --- /dev/null +++ b/backend/app/jobs/handlers/import_images.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import json +import random +from collections.abc import Iterable +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.config import load_settings +from app.core.data_files import get_sqlite_db_dir, resolve_file_ref +from app.core.pixiv_urls import parse_pixiv_original_url +from app.db.images_upsert import upsert_image_by_illust_page +from app.db.models.images import Image +from app.db.models.imports import Import +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker, with_sqlite_busy_retry +from app.jobs.errors import JobPermanentError + +_MAX_ERRORS = 200 +_CHUNK_SIZE = 200 + + +@dataclass(frozen=True, slots=True) +class ImportLineError: + line: int + url: str + code: str + message: str + + +def _as_bool(value: Any, *, default: bool = False) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return default + + +def _resolve_payload_file(payload: dict[str, Any]) -> Path | None: + if "file_ref" not in payload: + return None + + file_ref = str(payload.get("file_ref") or "").strip() + if not file_ref: + raise JobPermanentError("payload.file_ref is required") + + settings = load_settings() + base_dir = get_sqlite_db_dir(settings.database_url) + try: + return resolve_file_ref(file_ref, base_dir=base_dir) + except Exception as exc: + raise JobPermanentError("payload.file_ref invalid") from exc + + +def _iter_lines(payload: dict[str, Any], *, file_path: Path | None) -> Iterable[tuple[int, str]]: + if file_path is not None: + try: + with file_path.open("r", encoding="utf-8", errors="replace") as f: + for i, line in enumerate(f, start=1): + yield i, line + except FileNotFoundError as exc: + raise JobPermanentError("payload.file_ref not found") from exc + return + + if "text_lines" in payload: + raw = payload.get("text_lines") + if not isinstance(raw, list): + raise JobPermanentError("payload.text_lines must be a list") + for i, v in enumerate(raw, start=1): + yield i, str(v) + return + + if "text" in payload: + text = str(payload.get("text") or "") + for i, line in enumerate(text.splitlines(), start=1): + yield i, line + return + + raise JobPermanentError("payload.text_lines or payload.text or payload.file_ref is required") + + +def _parse_payload(payload_json: str) -> dict[str, Any]: + try: + data = json.loads(payload_json) + except Exception as exc: + raise JobPermanentError("payload_json is not valid JSON") from exc + if not isinstance(data, dict): + raise JobPermanentError("payload_json must be an object") + return data + + +def build_import_images_handler(engine: AsyncEngine): + Session = create_sessionmaker(engine) + + async def _handler(job: dict[str, Any]) -> None: + payload_json = str(job.get("payload_json") or "") + payload = _parse_payload(payload_json) + + try: + import_id = int(payload.get("import_id")) + except Exception as exc: + raise JobPermanentError("payload.import_id is required") from exc + if import_id <= 0: + raise JobPermanentError("payload.import_id is required") + + hydrate_on_import = _as_bool(payload.get("hydrate_on_import"), default=False) + file_path = _resolve_payload_file(payload) + + async def _op() -> None: + async with Session() as session: + imp = await session.get(Import, import_id) + if imp is None: + raise JobPermanentError("Import not found") + + total = 0 + accepted = 0 + success = 0 + deduped = 0 + error_total = 0 + errors: list[ImportLineError] = [] + seen: set[tuple[int, int]] = set() + illust_ids: set[int] = set() + + chunk: list[tuple[str, Any]] = [] + + for line_no, raw in _iter_lines(payload, file_path=file_path): + url = raw.strip() + if not url: + continue + total += 1 + try: + parsed = parse_pixiv_original_url(url) + except Exception as exc: + error_total += 1 + if len(errors) < _MAX_ERRORS: + errors.append( + ImportLineError( + line=int(line_no), + url=url, + code="unsupported_url", + message=str(exc) or "unsupported_url", + ) + ) + continue + + key = (parsed.illust_id, parsed.page_index) + if key in seen: + deduped += 1 + continue + seen.add(key) + + accepted += 1 + if hydrate_on_import: + illust_ids.add(int(parsed.illust_id)) + + chunk.append((url, parsed)) + + if len(chunk) < _CHUNK_SIZE: + continue + + for url_v, parsed_v in chunk: + image_id = await upsert_image_by_illust_page( + session, + illust_id=int(parsed_v.illust_id), + page_index=int(parsed_v.page_index), + ext=str(parsed_v.ext), + original_url=url_v, + proxy_path="", + random_key=random.random(), + created_import_id=int(import_id), + ) + proxy_path = f"/i/{image_id}.{parsed_v.ext}" + await session.execute( + sa.update(Image).where(Image.id == image_id).values(proxy_path=proxy_path) + ) + success += len(chunk) + imp.total = max(int(imp.total or 0), int(total)) + imp.accepted = max(int(imp.accepted or 0), int(accepted)) + imp.success = max(int(imp.success or 0), int(success)) + imp.failed = max(int(imp.failed or 0), int(error_total)) + await session.commit() + chunk.clear() + + if chunk: + for url_v, parsed_v in chunk: + image_id = await upsert_image_by_illust_page( + session, + illust_id=int(parsed_v.illust_id), + page_index=int(parsed_v.page_index), + ext=str(parsed_v.ext), + original_url=url_v, + proxy_path="", + random_key=random.random(), + created_import_id=int(import_id), + ) + proxy_path = f"/i/{image_id}.{parsed_v.ext}" + await session.execute(sa.update(Image).where(Image.id == image_id).values(proxy_path=proxy_path)) + success += len(chunk) + imp.total = max(int(imp.total or 0), int(total)) + imp.accepted = max(int(imp.accepted or 0), int(accepted)) + imp.success = max(int(imp.success or 0), int(success)) + imp.failed = max(int(imp.failed or 0), int(error_total)) + await session.commit() + + imp.total = max(int(imp.total or 0), int(total)) + imp.accepted = max(int(imp.accepted or 0), int(accepted)) + imp.success = max(int(imp.success or 0), int(success)) + imp.failed = max(int(imp.failed or 0), int(error_total)) + imp.detail_json = json.dumps( + {"deduped": int(deduped), "errors": [asdict(e) for e in errors[:_MAX_ERRORS]]}, + ensure_ascii=False, + separators=(",", ":"), + ) + await session.commit() + + if not hydrate_on_import or not illust_ids: + return + + existing = set( + ( + await session.execute( + sa.select(JobRow.ref_id) + .where(JobRow.type == "hydrate_metadata") + .where(JobRow.ref_type == "import") + .where(JobRow.ref_id.like(f"{import_id}:%")) + ) + ) + .scalars() + .all() + ) + + added = 0 + for illust_id in sorted(illust_ids): + ref_id = f"{import_id}:{int(illust_id)}" + if ref_id in existing: + continue + session.add( + JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps( + {"illust_id": int(illust_id), "reason": "import"}, + ensure_ascii=False, + separators=(",", ":"), + ), + ref_type="import", + ref_id=ref_id, + ) + ) + added += 1 + if added % 500 == 0: + await session.commit() + if added: + await session.commit() + + await with_sqlite_busy_retry(_op) + + if file_path is not None: + try: + file_path.unlink(missing_ok=True) + except Exception: + pass + + return _handler diff --git a/backend/app/jobs/handlers/proxy_probe.py b/backend/app/jobs/handlers/proxy_probe.py new file mode 100644 index 0000000..2b839f3 --- /dev/null +++ b/backend/app/jobs/handlers/proxy_probe.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import asyncio +import json +import time +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any, Awaitable, Callable +from urllib.parse import quote + +import httpx +import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.config import load_settings +from app.core.crypto import FieldEncryptor +from app.core.metrics import PROXY_PROBE_LATENCY_MS +from app.core.redact import redact_text +from app.core.time import iso_utc_ms +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.session import create_sessionmaker, with_sqlite_busy_retry +from app.jobs.errors import JobPermanentError + +DEFAULT_PROBE_URL = "https://www.pixiv.net/robots.txt" +DEFAULT_TIMEOUT_MS = 8000 +DEFAULT_CONCURRENCY = 10 + +BLACKLIST_AFTER_FAILURES = 3 +BLACKLIST_TTL_S = 30 * 60 + + +@dataclass(frozen=True, slots=True) +class ProbeConfig: + url: str + timeout_s: float + + +@dataclass(frozen=True, slots=True) +class ProbeTarget: + endpoint_id: int + proxy_uri: str + + +@dataclass(frozen=True, slots=True) +class ProbeResult: + endpoint_id: int + ok: bool + latency_ms: float | None + error: str | None = None + + +ProbeFunc = Callable[[ProbeTarget, ProbeConfig], Awaitable[ProbeResult]] + + +def _parse_payload(payload_json: str) -> dict[str, Any]: + try: + data = json.loads(payload_json) + except Exception as exc: + raise JobPermanentError("payload_json is not valid JSON") from exc + if not isinstance(data, dict): + raise JobPermanentError("payload_json must be an object") + return data + + +def _truncate(text: str, *, max_len: int = 500) -> str: + if len(text) <= max_len: + return text + return text[: max_len - 3] + "..." + + +def _build_proxy_uri( + encryptor: FieldEncryptor, + *, + scheme: str, + host: str, + port: int, + username: str, + password_enc: str, +) -> str: + scheme = (scheme or "").strip().lower() + host = (host or "").strip() + username = (username or "").strip() + password_enc = (password_enc or "").strip() + if not scheme or not host or int(port) <= 0: + raise ValueError("invalid proxy endpoint") + + password = encryptor.decrypt_text(password_enc) if password_enc else "" + + host_part = host + if ":" in host_part and not host_part.startswith("["): + host_part = f"[{host_part}]" + + auth = "" + if username: + user_q = quote(username, safe="") + pass_q = quote(password or "", safe="") + auth = f"{user_q}:{pass_q}@" + + return f"{scheme}://{auth}{host_part}:{int(port)}" + + +async def _default_probe(target: ProbeTarget, cfg: ProbeConfig) -> ProbeResult: + start = time.monotonic() + ok = False + err: str | None = None + + try: + async with httpx.AsyncClient( + proxy=target.proxy_uri, + timeout=httpx.Timeout(cfg.timeout_s, connect=min(10.0, cfg.timeout_s)), + follow_redirects=True, + ) as client: + resp = await client.get(cfg.url) + ok = int(resp.status_code) < 400 + if not ok: + err = f"status={resp.status_code}" + except Exception as exc: + err = f"{type(exc).__name__}: {exc}" + + latency_ms = (time.monotonic() - start) * 1000.0 + return ProbeResult(endpoint_id=int(target.endpoint_id), ok=bool(ok), latency_ms=float(latency_ms), error=err) + + +def build_proxy_probe_handler( + engine: AsyncEngine, + *, + prober: ProbeFunc | None = None, +) -> Any: + settings = load_settings() + encryptor = FieldEncryptor.from_key(settings.field_encryption_key) + Session = create_sessionmaker(engine) + + probe_fn = prober or _default_probe + + async def _handler(job: dict[str, Any]) -> None: + payload_json = str(job.get("payload_json") or "") + payload = _parse_payload(payload_json) + + probe_url = str(payload.get("probe_url") or DEFAULT_PROBE_URL).strip() or DEFAULT_PROBE_URL + timeout_ms_raw = payload.get("timeout_ms", DEFAULT_TIMEOUT_MS) + concurrency_raw = payload.get("concurrency", DEFAULT_CONCURRENCY) + + try: + timeout_ms = int(timeout_ms_raw) + except Exception as exc: + raise JobPermanentError("payload.timeout_ms invalid") from exc + if timeout_ms <= 0 or timeout_ms > 600_000: + raise JobPermanentError("payload.timeout_ms invalid") + + try: + concurrency = int(concurrency_raw) + except Exception as exc: + raise JobPermanentError("payload.concurrency invalid") from exc + if concurrency < 1: + concurrency = 1 + if concurrency > 200: + concurrency = 200 + + cfg = ProbeConfig(url=probe_url, timeout_s=float(timeout_ms) / 1000.0) + + async with Session() as session: + endpoints = ( + ( + await session.execute( + sa.select(ProxyEndpoint).where(ProxyEndpoint.enabled == 1).order_by(ProxyEndpoint.id.asc()) + ) + ) + .scalars() + .all() + ) + + if not endpoints: + return + + targets: list[ProbeTarget] = [] + immediate_results: list[ProbeResult] = [] + for ep in endpoints: + try: + proxy_uri = _build_proxy_uri( + encryptor, + scheme=str(ep.scheme), + host=str(ep.host), + port=int(ep.port), + username=str(ep.username or ""), + password_enc=str(ep.password_enc or ""), + ) + except Exception as exc: + immediate_results.append( + ProbeResult(endpoint_id=int(ep.id), ok=False, latency_ms=None, error=f"{type(exc).__name__}: {exc}") + ) + continue + targets.append(ProbeTarget(endpoint_id=int(ep.id), proxy_uri=proxy_uri)) + + sem = asyncio.Semaphore(int(concurrency)) + + async def _run_one(t: ProbeTarget) -> ProbeResult: + async with sem: + return await probe_fn(t, cfg) + + tasks = [asyncio.create_task(_run_one(t)) for t in targets] + probed = await asyncio.gather(*tasks) if tasks else [] + + results = list(immediate_results) + list(probed) + for r in results: + if r.latency_ms is not None and float(r.latency_ms) >= 0: + PROXY_PROBE_LATENCY_MS.observe(float(r.latency_ms)) + now_dt = datetime.now(timezone.utc) + now_iso = iso_utc_ms(now_dt) + blacklist_until_iso = iso_utc_ms(now_dt + timedelta(seconds=int(BLACKLIST_TTL_S))) + + async def _op() -> None: + async with Session() as session: + for r in results: + latency = float(r.latency_ms) if r.latency_ms is not None else None + if r.ok: + await session.execute( + sa.update(ProxyEndpoint) + .where(ProxyEndpoint.id == int(r.endpoint_id)) + .values( + last_latency_ms=latency, + last_ok_at=now_iso, + success_count=ProxyEndpoint.success_count + 1, + last_error=None, + blacklisted_until=None, + updated_at=now_iso, + ) + ) + continue + + msg = _truncate(redact_text(r.error or "probe_failed")) + blacklist_expr = sa.case( + ( + (ProxyEndpoint.failure_count + 1) >= int(BLACKLIST_AFTER_FAILURES), + blacklist_until_iso, + ), + else_=ProxyEndpoint.blacklisted_until, + ) + await session.execute( + sa.update(ProxyEndpoint) + .where(ProxyEndpoint.id == int(r.endpoint_id)) + .values( + last_latency_ms=latency, + last_fail_at=now_iso, + failure_count=ProxyEndpoint.failure_count + 1, + blacklisted_until=blacklist_expr, + last_error=msg, + updated_at=now_iso, + ) + ) + + await session.commit() + + await with_sqlite_busy_retry(_op) + + return _handler diff --git a/backend/app/jobs/model.py b/backend/app/jobs/model.py new file mode 100644 index 0000000..0f552f3 --- /dev/null +++ b/backend/app/jobs/model.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from enum import Enum + +from app.core.redact import redact_text +from app.jobs.backoff import backoff_seconds + + +class JobStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + PAUSED = "paused" + CANCELED = "canceled" + COMPLETED = "completed" + FAILED = "failed" + DLQ = "dlq" + + +@dataclass(frozen=True, slots=True) +class Job: + id: int + status: JobStatus + attempt: int + max_attempts: int + run_after: str | None = None + last_error: str | None = None + locked_by: str | None = None + locked_at: str | None = None + + +@dataclass(frozen=True, slots=True) +class JobTransition: + status: JobStatus + attempt: int + run_after: str | None + last_error: str | None + locked_by: str | None + locked_at: str | None + updated_at: str + + +def _iso_utc_ms(dt: datetime) -> str: + dt = dt.astimezone(timezone.utc) + ms = dt.microsecond // 1000 + return dt.strftime("%Y-%m-%dT%H:%M:%S.") + f"{ms:03d}Z" + + +def _truncate(text: str, *, max_len: int = 2000) -> str: + if len(text) <= max_len: + return text + return text[: max_len - 3] + "..." + + +def on_job_success(job: Job, *, now: datetime | None = None) -> JobTransition: + now_dt = now or datetime.now(timezone.utc) + return JobTransition( + status=JobStatus.COMPLETED, + attempt=job.attempt, + run_after=None, + last_error=None, + locked_by=None, + locked_at=None, + updated_at=_iso_utc_ms(now_dt), + ) + + +def on_job_failure(job: Job, *, error: str, now: datetime | None = None) -> JobTransition: + now_dt = now or datetime.now(timezone.utc) + next_attempt = job.attempt + 1 + redacted_error = _truncate(redact_text(error)) + + if next_attempt >= job.max_attempts: + return JobTransition( + status=JobStatus.DLQ, + attempt=next_attempt, + run_after=None, + last_error=redacted_error, + locked_by=None, + locked_at=None, + updated_at=_iso_utc_ms(now_dt), + ) + + delay_s = backoff_seconds(next_attempt) + run_after_dt = now_dt + timedelta(seconds=delay_s) + return JobTransition( + status=JobStatus.FAILED, + attempt=next_attempt, + run_after=_iso_utc_ms(run_after_dt), + last_error=redacted_error, + locked_by=None, + locked_at=None, + updated_at=_iso_utc_ms(now_dt), + ) + + +def on_job_defer(job: Job, *, run_after: str, error: str, now: datetime | None = None) -> JobTransition: + now_dt = now or datetime.now(timezone.utc) + run_after = (run_after or "").strip() + if not run_after: + raise ValueError("run_after is required") + redacted_error = _truncate(redact_text(error)) + + return JobTransition( + status=JobStatus.FAILED, + attempt=job.attempt, + run_after=run_after, + last_error=redacted_error, + locked_by=None, + locked_at=None, + updated_at=_iso_utc_ms(now_dt), + ) diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..1071e1a --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import json +import time + +from fastapi import FastAPI, Request + +from app.api.admin.router import router as admin_router +from app.api.metrics import router as metrics_router +from app.api.public.healthz import router as healthz_router +from app.api.public.docs_page import router as docs_page_router +from app.api.public.authors import router as authors_router +from app.api.public.images import router as images_router +from app.api.public.legacy import router as legacy_router +from app.api.public.random import router as random_router +from app.api.public.tags import router as tags_router +from app.api.public.version import router as version_router +from app.core.config import load_settings +from app.core.api_keys import ApiKeyAuthConfig, ApiKeyAuthenticator, ApiKeyRateLimiter, require_public_api_key +from app.core.errors import ApiError, ErrorCode, json_error_response +from app.core.logging import configure_logging, get_logger +from app.core.metrics import observe_random_result +from app.core.random_request_stats import RandomRequestStats +from app.core.request_id import build_request_id_middleware, get_or_create_request_id, set_request_id_on_state +from app.core.security import decode_jwt, parse_bearer_token +from app.db.engine import create_engine +from app.db.models.admin_audit import AdminAudit +from app.db.session import create_sessionmaker, with_sqlite_busy_retry +from app.web.admin_ui import mount_admin_ui + +log = get_logger(__name__) + + +def create_app() -> FastAPI: + configure_logging() + settings = load_settings() + + # NOTE: We reserve `/docs` for a public human-readable documentation page. + # Keep Swagger/Redoc available under `/api/*` paths for troubleshooting. + app = FastAPI(title="new-pixiv-api", docs_url="/api/docs", redoc_url="/api/redoc") + + @app.exception_handler(ApiError) + async def _api_error_handler(request: Request, exc: ApiError): # type: ignore[no-redef] + return json_error_response( + code=exc.code, + message=exc.message, + status_code=exc.status_code, + request=request, + details=exc.details, + ) + + @app.exception_handler(Exception) + async def _unhandled_exception_handler(request: Request, exc: Exception): # type: ignore[no-redef] + try: + log.exception("unhandled_exception path=%s", str(getattr(request, "url", ""))) + except Exception: + pass + return json_error_response( + code=ErrorCode.INTERNAL_ERROR, + message="服务器内部错误", + status_code=500, + request=request, + details={"error_type": type(exc).__name__}, + ) + + request_id_middleware = build_request_id_middleware() + if request_id_middleware is not None: + app.add_middleware(request_id_middleware) + + engine = create_engine(settings.database_url) + app.state.engine = engine + + api_key_cfg = ApiKeyAuthConfig( + required=bool(settings.public_api_key_required), + rpm=int(settings.public_api_key_rpm), + burst=int(settings.public_api_key_burst), + secret_key=str(settings.secret_key), + ) + app.state.api_key_authenticator = ApiKeyAuthenticator(engine, api_key_cfg) + app.state.api_key_limiter = ApiKeyRateLimiter(rpm=int(api_key_cfg.rpm), burst=int(api_key_cfg.burst)) + app.state.random_request_stats = RandomRequestStats(window_seconds=60) + + @app.middleware("http") + async def _public_api_key_middleware(request: Request, call_next): # type: ignore[no-redef] + if not bool(settings.public_api_key_required): + return await call_next(request) + + path = request.url.path + if path.startswith("/admin") or path.startswith("/metrics"): + return await call_next(request) + if path in {"/healthz", "/version", "/openapi.json", "/docs", "/api/docs", "/api/redoc"}: + return await call_next(request) + + rid = get_or_create_request_id(request) + set_request_id_on_state(request, rid) + + try: + api_key_id = await require_public_api_key( + request.app.state.api_key_authenticator, + request.app.state.api_key_limiter, + headers=request.headers, + ) + except ApiError as exc: + return json_error_response( + code=exc.code, + message=exc.message, + status_code=exc.status_code, + request=request, + details=exc.details, + ) + request.state.api_key_id = int(api_key_id) + + return await call_next(request) + + def _best_effort_admin_actor(request: Request) -> str | None: + authorization = request.headers.get("Authorization") or request.headers.get("authorization") + token = parse_bearer_token(authorization) + if not token: + return None + try: + claims = decode_jwt(token, secret_key=str(settings.secret_key)) + except Exception: + return None + sub = str(claims.get("sub") or "").strip() + return sub or None + + def _best_effort_client_ip(request: Request) -> str | None: + xff = request.headers.get("X-Forwarded-For") or request.headers.get("x-forwarded-for") + if xff: + ip = str(xff).split(",", 1)[0].strip() + return ip or None + client = getattr(request, "client", None) + host = getattr(client, "host", None) if client is not None else None + return str(host).strip() if host else None + + @app.middleware("http") + async def _admin_audit_middleware(request: Request, call_next): # type: ignore[no-redef] + path = request.url.path + method = (request.method or "").upper() + + response = await call_next(request) + + if not path.startswith("/admin/api/"): + return response + if method in {"GET", "HEAD", "OPTIONS"}: + return response + + try: + status_code = int(getattr(response, "status_code", 0) or 0) + except Exception: + status_code = 0 + if status_code >= 400: + return response + + try: + rid = get_or_create_request_id(request) + actor = _best_effort_admin_actor(request) + ip = _best_effort_client_ip(request) + user_agent = request.headers.get("User-Agent") or request.headers.get("user-agent") + + segments = [seg for seg in path.split("/") if seg] + record_id = next((seg for seg in reversed(segments) if seg.isdigit()), None) + + detail_json = {"status": status_code, "query": dict(request.query_params)} + + engine = request.app.state.engine + Session = create_sessionmaker(engine) + + async def _op() -> None: + async with Session() as session: + session.add( + AdminAudit( + actor=actor, + action=method, + resource=path, + record_id=str(record_id) if record_id else None, + request_id=str(rid), + ip=ip, + user_agent=str(user_agent) if user_agent else None, + detail_json=json.dumps(detail_json, ensure_ascii=False, separators=(",", ":")), + ) + ) + await session.commit() + + await with_sqlite_busy_retry(_op) + except Exception: + pass + + return response + + def _random_result_from_status(status: int) -> str: + if status in {200, 301, 302, 303, 307, 308}: + return "ok" + if status == 404: + return "no_match" + if status == 502: + return "upstream_error" + if status == 400: + return "bad_request" + return "error" + + @app.middleware("http") + async def _metrics_middleware(request: Request, call_next): # type: ignore[no-redef] + if request.url.path != "/random": + return await call_next(request) + + started = time.monotonic() + stats = getattr(request.app.state, "random_request_stats", None) + if stats is not None: + try: + await stats.on_begin() + except Exception: + pass + response = None + try: + response = await call_next(request) + return response + finally: + duration_s = time.monotonic() - started + status_code = int(getattr(response, "status_code", 0) or 0) + observe_random_result(result=_random_result_from_status(status_code), duration_s=duration_s) + if stats is not None: + try: + await stats.on_end(status_code=int(status_code)) + except Exception: + pass + + app.state.settings = settings + + @app.on_event("shutdown") + async def _shutdown() -> None: # type: ignore[no-redef] + engine = getattr(app.state, "engine", None) + if engine is not None: + await engine.dispose() + + app.include_router(healthz_router) + app.include_router(docs_page_router) + app.include_router(authors_router) + app.include_router(images_router) + app.include_router(legacy_router) + app.include_router(random_router) + app.include_router(tags_router) + app.include_router(version_router) + app.include_router(metrics_router) + app.include_router(admin_router) + + mount_admin_ui(app) + + return app + + +app = create_app() diff --git a/backend/app/pixiv/__init__.py b/backend/app/pixiv/__init__.py new file mode 100644 index 0000000..3ce55a6 --- /dev/null +++ b/backend/app/pixiv/__init__.py @@ -0,0 +1,2 @@ +from __future__ import annotations + diff --git a/backend/app/pixiv/access_token_cache.py b/backend/app/pixiv/access_token_cache.py new file mode 100644 index 0000000..c93b473 --- /dev/null +++ b/backend/app/pixiv/access_token_cache.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass +from typing import Awaitable, Callable, Hashable + +from app.pixiv.oauth import PixivOauthToken + + +DEFAULT_REFRESH_MARGIN_S = 60.0 + + +@dataclass(frozen=True, slots=True) +class CachedAccessToken: + access_token: str + expires_at: float + + def is_valid(self, *, now: float, refresh_margin_s: float) -> bool: + if not self.access_token: + return False + return now < (self.expires_at - refresh_margin_s) + + +class AccessTokenCache: + def __init__( + self, + *, + now: Callable[[], float] | None = None, + refresh_margin_s: float = DEFAULT_REFRESH_MARGIN_S, + ) -> None: + self._now = now or time.time + self._refresh_margin_s = float(refresh_margin_s) + if self._refresh_margin_s < 0: + self._refresh_margin_s = 0.0 + + self._items: dict[Hashable, CachedAccessToken] = {} + self._locks: dict[Hashable, asyncio.Lock] = {} + + def get(self, key: Hashable) -> str | None: + item = self._items.get(key) + if item is None: + return None + if not item.is_valid(now=float(self._now()), refresh_margin_s=self._refresh_margin_s): + return None + return item.access_token + + def set(self, key: Hashable, *, access_token: str, expires_in_s: float) -> None: + access_token = (access_token or "").strip() + expires_in_s = float(expires_in_s) + if not access_token or expires_in_s <= 0: + self._items.pop(key, None) + return + expires_at = float(self._now()) + expires_in_s + self._items[key] = CachedAccessToken(access_token=access_token, expires_at=expires_at) + + def invalidate(self, key: Hashable) -> None: + self._items.pop(key, None) + + async def get_or_refresh( + self, + key: Hashable, + *, + refresher: Callable[[], Awaitable[PixivOauthToken]], + ) -> PixivOauthToken: + cached = self.get(key) + if cached is not None: + return PixivOauthToken( + access_token=cached, + token_type="bearer", + expires_in=int(self._items[key].expires_at - float(self._now())), + refresh_token=None, + scope=None, + user_id=None, + ) + + lock = self._locks.get(key) + if lock is None: + lock = asyncio.Lock() + self._locks[key] = lock + + async with lock: + cached2 = self.get(key) + if cached2 is not None: + return PixivOauthToken( + access_token=cached2, + token_type="bearer", + expires_in=int(self._items[key].expires_at - float(self._now())), + refresh_token=None, + scope=None, + user_id=None, + ) + + token = await refresher() + self.set(key, access_token=token.access_token, expires_in_s=float(token.expires_in)) + return token + diff --git a/backend/app/pixiv/oauth.py b/backend/app/pixiv/oauth.py new file mode 100644 index 0000000..e16a5a2 --- /dev/null +++ b/backend/app/pixiv/oauth.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Mapping + +import httpx + + +DEFAULT_OAUTH_BASE_URL = "https://oauth.secure.pixiv.net" +OAUTH_TOKEN_PATH = "/auth/token" + + +class PixivOauthError(RuntimeError): + def __init__(self, message: str, *, status_code: int | None = None) -> None: + super().__init__(message) + self.status_code = status_code + + +@dataclass(frozen=True, slots=True) +class PixivOauthConfig: + client_id: str + client_secret: str + hash_secret: str | None = None + + base_url: str = DEFAULT_OAUTH_BASE_URL + user_agent: str = "PixivAndroidApp/5.0.234 (Android 11; Pixel 5)" + accept_language: str = "en_US" + app_os: str = "android" + app_os_version: str = "11" + app_version: str = "5.0.234" + + def build_headers(self, *, client_time: str) -> dict[str, str]: + headers: dict[str, str] = { + "User-Agent": self.user_agent, + "Accept-Language": self.accept_language, + "App-OS": self.app_os, + "App-OS-Version": self.app_os_version, + "App-Version": self.app_version, + } + if self.hash_secret: + headers["X-Client-Time"] = client_time + headers["X-Client-Hash"] = hashlib.md5((client_time + self.hash_secret).encode("utf-8")).hexdigest() + return headers + + +@dataclass(frozen=True, slots=True) +class PixivOauthToken: + access_token: str + token_type: str + expires_in: int + refresh_token: str | None + scope: str | None + user_id: str | None + + +def _now_client_time() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _unwrap_response(data: Any) -> Mapping[str, Any]: + if isinstance(data, dict) and isinstance(data.get("response"), dict): + return data["response"] + if isinstance(data, dict): + return data + raise PixivOauthError("Invalid OAuth response shape") + + +def _parse_token_response(data: Any) -> PixivOauthToken: + payload = _unwrap_response(data) + + access_token = payload.get("access_token") + token_type = payload.get("token_type") + expires_in = payload.get("expires_in") + refresh_token = payload.get("refresh_token") + scope = payload.get("scope") + user_id = None + + user = payload.get("user") + if isinstance(user, dict) and user.get("id") is not None: + user_id = str(user.get("id")) + + if not isinstance(access_token, str) or not access_token: + raise PixivOauthError("OAuth response missing access_token") + if not isinstance(token_type, str) or not token_type: + raise PixivOauthError("OAuth response missing token_type") + if isinstance(expires_in, str) and expires_in.isdigit(): + expires_in = int(expires_in) + if not isinstance(expires_in, int) or expires_in <= 0: + raise PixivOauthError("OAuth response missing expires_in") + + return PixivOauthToken( + access_token=access_token, + token_type=token_type, + expires_in=expires_in, + refresh_token=refresh_token if isinstance(refresh_token, str) and refresh_token else None, + scope=scope if isinstance(scope, str) and scope else None, + user_id=user_id, + ) + + +async def refresh_access_token( + *, + refresh_token: str, + config: PixivOauthConfig, + transport: httpx.BaseTransport | None = None, + proxy: str | None = None, + timeout_s: float = 30.0, + client_time: str | None = None, +) -> PixivOauthToken: + refresh_token = (refresh_token or "").strip() + if not refresh_token: + raise ValueError("refresh_token is required") + + if not config.client_id.strip() or not config.client_secret.strip(): + raise ValueError("PixivOauthConfig.client_id/client_secret are required") + + client_time = client_time or _now_client_time() + headers = config.build_headers(client_time=client_time) + + payload = { + "client_id": config.client_id, + "client_secret": config.client_secret, + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "get_secure_url": "1", + "include_policy": "1", + } + + url = config.base_url.rstrip("/") + OAUTH_TOKEN_PATH + + async with httpx.AsyncClient( + transport=transport, + proxy=proxy, + headers=headers, + timeout=httpx.Timeout(timeout_s, connect=10.0), + follow_redirects=True, + ) as client: + resp = await client.post(url, data=payload) + + if resp.status_code != 200: + raise PixivOauthError("OAuth refresh failed", status_code=resp.status_code) + + try: + data = resp.json() + except Exception as exc: + raise PixivOauthError("OAuth response is not JSON", status_code=resp.status_code) from exc + + return _parse_token_response(data) + diff --git a/backend/app/pixiv/refresh_backoff.py b/backend/app/pixiv/refresh_backoff.py new file mode 100644 index 0000000..9ba2b14 --- /dev/null +++ b/backend/app/pixiv/refresh_backoff.py @@ -0,0 +1,40 @@ +from __future__ import annotations + + +def refresh_backoff_seconds(*, attempt: int, status_code: int | None) -> int: + attempt_i = int(attempt) + if attempt_i <= 0: + return 0 + + status = int(status_code) if status_code is not None else None + + # Auth / token invalid style failures: back off much longer. + if status in {400, 401, 403}: + schedule = { + 1: 3600, + 2: 6 * 3600, + 3: 24 * 3600, + 4: 3 * 24 * 3600, + 5: 7 * 24 * 3600, + } + if attempt_i in schedule: + return schedule[attempt_i] + + base = 7 * 24 * 3600 + seconds = base * (2 ** (attempt_i - 5)) + return min(seconds, 30 * 24 * 3600) + + # Network / transient failures: short backoff. + schedule = { + 1: 5, + 2: 30, + 3: 120, + 4: 600, + 5: 1800, + } + if attempt_i in schedule: + return schedule[attempt_i] + + base = 1800 + seconds = base * (2 ** (attempt_i - 5)) + return min(seconds, 6 * 3600) diff --git a/backend/app/pixiv/token_strategy.py b/backend/app/pixiv/token_strategy.py new file mode 100644 index 0000000..c30a5a8 --- /dev/null +++ b/backend/app/pixiv/token_strategy.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable, Sequence + + +class NoTokenAvailable(RuntimeError): + def __init__(self, *, next_retry_at: float | None) -> None: + super().__init__("No eligible token available") + self.next_retry_at = next_retry_at + + +@dataclass(frozen=True, slots=True) +class TokenCandidate: + id: int + enabled: bool + weight: float + error_count: int + backoff_until: float = 0.0 + + +def _eligible(tokens: Iterable[TokenCandidate], *, now: float) -> list[TokenCandidate]: + out: list[TokenCandidate] = [] + for t in tokens: + if not t.enabled: + continue + if float(t.backoff_until) > float(now): + continue + out.append(t) + out.sort(key=lambda x: int(x.id)) + return out + + +def _next_retry_at(tokens: Sequence[TokenCandidate]) -> float | None: + values: list[float] = [] + for t in tokens: + if not t.enabled: + continue + values.append(float(t.backoff_until)) + return min(values) if values else None + + +def _choose_round_robin(tokens: Sequence[TokenCandidate], *, last_id: int | None) -> TokenCandidate: + if not tokens: + raise ValueError("tokens is empty") + if last_id is None: + return tokens[0] + ids = [int(t.id) for t in tokens] + try: + idx = ids.index(int(last_id)) + except ValueError: + return tokens[0] + return tokens[(idx + 1) % len(tokens)] + + +def _choose_weighted( + tokens: Sequence[TokenCandidate], + *, + r: float, + last_id: int | None, +) -> TokenCandidate: + if not tokens: + raise ValueError("tokens is empty") + + weights = [max(0.0, float(t.weight)) for t in tokens] + total = sum(weights) + if total <= 0.0: + return _choose_round_robin(tokens, last_id=last_id) + + r = float(r) + if r < 0.0: + r = 0.0 + if r >= 1.0: + r = 0.999999999 + + target = r * total + chosen = tokens[0] + for token, w in zip(tokens, weights, strict=True): + if w <= 0.0: + continue + chosen = token + if target < w: + return token + target -= w + return chosen + + +def choose_token( + tokens: Sequence[TokenCandidate], + *, + strategy: str, + now: float, + last_id: int | None, + r: float = 0.0, +) -> tuple[TokenCandidate, int]: + eligible = _eligible(tokens, now=now) + if not eligible: + raise NoTokenAvailable(next_retry_at=_next_retry_at(tokens)) + + strategy = (strategy or "").strip().lower() + if strategy in {"round_robin", ""}: + token = _choose_round_robin(eligible, last_id=last_id) + return token, int(token.id) + + if strategy == "least_error": + min_err = min(int(t.error_count) for t in eligible) + best = [t for t in eligible if int(t.error_count) == min_err] + token = _choose_round_robin(best, last_id=last_id) + return token, int(token.id) + + if strategy == "weighted": + token = _choose_weighted(eligible, r=r, last_id=last_id) + return token, int(token.id) + + raise ValueError("Unsupported strategy") + diff --git a/backend/app/web/__init__.py b/backend/app/web/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/app/web/__init__.py @@ -0,0 +1 @@ + diff --git a/backend/app/web/admin_ui.py b/backend/app/web/admin_ui.py new file mode 100644 index 0000000..7034110 --- /dev/null +++ b/backend/app/web/admin_ui.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi import FastAPI, HTTPException +from fastapi.responses import FileResponse, RedirectResponse +from starlette.staticfiles import StaticFiles + + +def mount_admin_ui(app: FastAPI, *, dist_dir: str | Path = "/app/web/dist") -> bool: + dist_path = Path(dist_dir) + index_file = dist_path / "index.html" + if not index_file.is_file(): + return False + + assets_dir = dist_path / "assets" + if assets_dir.is_dir(): + app.mount( + "/admin/assets", + StaticFiles(directory=str(assets_dir)), + name="admin-assets", + ) + + @app.get("/", include_in_schema=False) + async def _root() -> RedirectResponse: # type: ignore[no-redef] + return RedirectResponse(url="/admin", status_code=302) + + @app.get("/admin", include_in_schema=False) + async def _admin_index() -> FileResponse: # type: ignore[no-redef] + return FileResponse(index_file) + + @app.get("/admin/{path:path}", include_in_schema=False) + async def _admin_spa(path: str) -> FileResponse: # type: ignore[no-redef] + p = (path or "").lstrip("/") + if p.startswith("api") or p.startswith("assets"): + raise HTTPException(status_code=404) + return FileResponse(index_file) + + return True + diff --git a/backend/app/worker.py b/backend/app/worker.py new file mode 100644 index 0000000..426d62f --- /dev/null +++ b/backend/app/worker.py @@ -0,0 +1,451 @@ +from __future__ import annotations + +import asyncio +import os +import signal +import time +from collections.abc import Awaitable, Callable +from typing import Any + +from app.easy_proxies.auto_refresh import EasyProxiesAutoRefreshConfig, EasyProxiesAutoRefresher +from app.core.config import load_settings +from app.core.logging import configure_logging, get_logger +from app.core.redact import redact_text +from app.core.time import iso_utc_ms +from app.core.runtime_settings import set_runtime_setting +from app.db.engine import create_engine +from app.jobs.claim import DEFAULT_LOCK_TTL_S, claim_next_job +from app.jobs.dispatch import JobDispatcher +from app.jobs.errors import JobPermanentError +from app.jobs.executor import execute_claimed_job +from app.jobs.handlers.easy_proxies_import import build_easy_proxies_import_handler +from app.jobs.handlers.heal_url import build_heal_url_handler +from app.jobs.handlers.hydrate_metadata import build_hydrate_metadata_handler +from app.jobs.handlers.import_images import build_import_images_handler +from app.jobs.handlers.proxy_probe import build_proxy_probe_handler + +log = get_logger(__name__) + + +def _install_signal_handlers(stop_event: asyncio.Event) -> None: + loop = asyncio.get_running_loop() + signals = [signal.SIGINT] + if hasattr(signal, "SIGTERM"): + signals.append(signal.SIGTERM) + + for sig in signals: + try: + loop.add_signal_handler(sig, stop_event.set) + except NotImplementedError: + signal.signal(sig, lambda *_: stop_event.set()) + except Exception: + continue + + +def _disabled_handler(job_type: str, *, reason: str): + async def _handler(_job: dict[str, Any]) -> None: + raise JobPermanentError(f"{job_type} handler disabled: {reason}") + + return _handler + + +def build_default_dispatcher(engine) -> JobDispatcher: + dispatcher = JobDispatcher() + dispatcher.register("import_images", build_import_images_handler(engine)) + + def _safe_register(job_type: str, builder: Callable[[], Any]) -> None: + try: + dispatcher.register(job_type, builder()) + except Exception as exc: + msg = redact_text(f"{type(exc).__name__}: {exc}") + log.warning("jobs_handler_disabled type=%s reason=%s", job_type, msg) + dispatcher.register(job_type, _disabled_handler(job_type, reason=msg)) + + _safe_register("hydrate_metadata", lambda: build_hydrate_metadata_handler(engine)) + _safe_register("heal_url", lambda: build_heal_url_handler(engine)) + _safe_register("proxy_probe", lambda: build_proxy_probe_handler(engine)) + _safe_register("easy_proxies_import", lambda: build_easy_proxies_import_handler(engine)) + return dispatcher + + +def _parse_int_env(name: str, *, default: int, min_v: int, max_v: int) -> int: + raw = (os.environ.get(name) or "").strip() + if not raw: + return int(default) + try: + value = int(raw) + except Exception: + return int(default) + return max(int(min_v), min(int(value), int(max_v))) + + +def _parse_bool_env(name: str, *, default: bool) -> bool: + raw = (os.environ.get(name) or "").strip() + if raw == "": + return bool(default) + v = raw.lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return bool(default) + + +def compute_desired_worker_concurrency( + *, + auto_enabled: bool, + enabled_tokens: int | None, + max_concurrency: int, +) -> int: + max_c = max(1, int(max_concurrency)) + if not bool(auto_enabled): + return max_c + n = int(enabled_tokens or 0) + if n <= 0: + n = 1 + return max(1, min(max_c, n)) + + +async def _count_enabled_tokens(engine) -> int | None: + try: + async with engine.connect() as conn: + value = (await conn.exec_driver_sql("SELECT COUNT(*) FROM pixiv_tokens WHERE enabled=1;")).scalar_one() + return int(value or 0) + except Exception: + return None + + +class _JobScheduler: + def __init__(self, engine, dispatcher: JobDispatcher, *, worker_id: str, lock_ttl_s: int) -> None: + self._engine = engine + self._dispatcher = dispatcher + self._worker_id = str(worker_id) + self._lock_ttl_s = int(lock_ttl_s) + self._tasks: set[asyncio.Task] = set() + self._stopping = False + + def _on_task_done(self, task: asyncio.Task) -> None: + self._tasks.discard(task) + try: + _ = task.result() + except asyncio.CancelledError: + return + except Exception as exc: + msg = redact_text(f"{type(exc).__name__}: {exc}") + log.warning("job_task_failed err=%s", msg) + + async def tick(self, *, desired_concurrency: int, max_claims: int) -> int: + if self._stopping: + return 0 + + desired = max(1, int(desired_concurrency)) + slots = max(0, int(desired - len(self._tasks))) + if slots <= 0: + return 0 + + claimed = 0 + for _ in range(int(min(slots, max(1, int(max_claims))))): + try: + job_row = await claim_next_job( + self._engine, + worker_id=str(self._worker_id), + lock_ttl_s=int(self._lock_ttl_s), + ) + except Exception as exc: + msg = redact_text(f"{type(exc).__name__}: {exc}") + log.warning("jobs_claim_failed err=%s", msg) + break + if job_row is None: + break + + async def _run(row: dict[str, Any]) -> None: + try: + await execute_claimed_job( + self._engine, + self._dispatcher, + job_row=row, + worker_id=str(self._worker_id), + ) + except Exception as exc: + msg = redact_text(f"{type(exc).__name__}: {exc}") + log.warning("job_execute_failed err=%s", msg) + + task = asyncio.create_task(_run(job_row)) + task.add_done_callback(self._on_task_done) + self._tasks.add(task) + claimed += 1 + + return int(claimed) + + async def shutdown(self) -> None: + self._stopping = True + if not self._tasks: + return + tasks = list(self._tasks) + for t in tasks: + t.cancel() + _ = await asyncio.gather(*tasks, return_exceptions=True) + self._tasks.clear() + + +async def poll_and_execute_jobs( + engine, + dispatcher: JobDispatcher, + *, + worker_id: str, + lock_ttl_s: int = DEFAULT_LOCK_TTL_S, + max_jobs: int = 10, +) -> int: + worker_id = (worker_id or "").strip() + if not worker_id: + raise ValueError("worker_id is required") + + ran = 0 + for _ in range(int(max_jobs)): + try: + job_row = await claim_next_job(engine, worker_id=worker_id, lock_ttl_s=int(lock_ttl_s)) + except Exception as exc: + msg = redact_text(f"{type(exc).__name__}: {exc}") + log.warning("jobs_claim_failed err=%s", msg) + break + if job_row is None: + break + + try: + await execute_claimed_job(engine, dispatcher, job_row=job_row, worker_id=worker_id) + except Exception as exc: + msg = redact_text(f"{type(exc).__name__}: {exc}") + log.warning("job_execute_failed err=%s", msg) + ran += 1 + return ran + + +async def _poll_once() -> None: + return None + + +async def run_worker( + *, + poll_interval_s: float = 1.0, + max_iterations: int | None = None, + on_tick: Callable[[], Awaitable[None]] | None = None, +) -> None: + stop_event = asyncio.Event() + _install_signal_handlers(stop_event) + + on_tick = on_tick or _poll_once + iterations = 0 + + while not stop_event.is_set(): + await on_tick() + + iterations += 1 + if max_iterations is not None and iterations >= max_iterations: + break + + if poll_interval_s > 0: + await asyncio.sleep(poll_interval_s) + + +async def main_async(*, max_iterations: int | None = None, poll_interval_s: float = 1.0) -> None: + configure_logging() + settings = load_settings() + + engine = create_engine(settings.database_url) + scheduler: _JobScheduler | None = None + try: + dispatcher = build_default_dispatcher(engine) + + base_url = (os.environ.get("EASY_PROXIES_BASE_URL") or "").strip() + raw_auto = (os.environ.get("EASY_PROXIES_AUTO_REFRESH") or "").strip().lower() + auto_refresh_disabled = raw_auto in {"0", "false", "no", "n", "off"} + + raw_ms = (os.environ.get("EASY_PROXIES_REFRESH_INTERVAL_MS") or "").strip() + raw_s = (os.environ.get("EASY_PROXIES_REFRESH_INTERVAL_SECONDS") or "").strip() + if not raw_s: + raw_s = (os.environ.get("EASY_PROXIES_REFRESH_INTERVAL_S") or "").strip() + + interval_s = 0.0 + if raw_ms: + try: + interval_s = float(raw_ms) / 1000.0 + except Exception: + interval_s = 0.0 + elif raw_s: + try: + interval_s = float(raw_s) + except Exception: + interval_s = 0.0 + + if auto_refresh_disabled: + interval_s = 0.0 + + conflict_policy = (os.environ.get("EASY_PROXIES_CONFLICT_POLICY") or "skip_non_easy_proxies").strip() + + host_override = (os.environ.get("EASY_PROXIES_HOST_OVERRIDE") or "").strip() or None + + raw_attach = (os.environ.get("EASY_PROXIES_AUTO_ATTACH") or "").strip().lower() + auto_attach_disabled = raw_attach in {"0", "false", "no", "n", "off"} + + attach_pool_id: int | None = None + raw_attach_pool = (os.environ.get("EASY_PROXIES_ATTACH_POOL_ID") or "").strip() + if raw_attach_pool: + try: + attach_pool_id = int(raw_attach_pool) + except Exception: + attach_pool_id = None + if attach_pool_id is not None and int(attach_pool_id) <= 0: + attach_pool_id = None + + attach_weight = _parse_int_env( + "EASY_PROXIES_ATTACH_WEIGHT", + default=1, + min_v=0, + max_v=1000, + ) + + raw_recompute = (os.environ.get("EASY_PROXIES_AUTO_RECOMPUTE_BINDINGS") or "").strip().lower() + recompute_disabled = raw_recompute in {"0", "false", "no", "n", "off"} + recompute_bindings = not recompute_disabled + + max_tokens_per_proxy = _parse_int_env( + "EASY_PROXIES_MAX_TOKENS_PER_PROXY", + default=2, + min_v=1, + max_v=1000, + ) + + raw_strict = (os.environ.get("EASY_PROXIES_BINDINGS_STRICT") or "").strip().lower() + strict = raw_strict in {"1", "true", "yes", "y", "on"} + + refresher = EasyProxiesAutoRefresher( + EasyProxiesAutoRefreshConfig( + base_url=base_url, + interval_s=interval_s, + conflict_policy=conflict_policy or "skip_non_easy_proxies", + host_override=host_override, + auto_attach=not auto_attach_disabled, + attach_pool_id=attach_pool_id, + attach_weight=int(attach_weight), + recompute_bindings=bool(recompute_bindings), + max_tokens_per_proxy=int(max_tokens_per_proxy), + strict=bool(strict), + ) + ) + if refresher.enabled: + log.info("easy_proxies_auto_refresh_enabled base_url=%s interval_s=%s", base_url, interval_s) + + worker_id = (os.environ.get("WORKER_ID") or f"pid{os.getpid()}").strip() + jobs_lock_ttl_s = _parse_int_env( + "WORKER_JOBS_LOCK_TTL_SECONDS", + default=int(DEFAULT_LOCK_TTL_S), + min_v=5, + max_v=3600, + ) + max_jobs_per_tick = _parse_int_env( + "WORKER_MAX_JOBS_PER_TICK", + default=10, + min_v=1, + max_v=1000, + ) + max_concurrency = _parse_int_env( + "WORKER_MAX_CONCURRENCY", + default=20, + min_v=1, + max_v=200, + ) + auto_concurrency = _parse_bool_env("WORKER_AUTO_CONCURRENCY", default=True) + auto_refresh_s = _parse_int_env( + "WORKER_AUTO_CONCURRENCY_REFRESH_SECONDS", + default=15, + min_v=1, + max_v=3600, + ) + try: + heartbeat_interval_s = float((os.environ.get("WORKER_HEARTBEAT_INTERVAL_SECONDS") or "10").strip() or "10") + except Exception: + heartbeat_interval_s = 10.0 + heartbeat_interval_s = max(1.0, min(float(heartbeat_interval_s), 300.0)) + last_heartbeat_m = 0.0 + last_auto_refresh_m = 0.0 + cached_enabled_tokens: int | None = None + cached_desired_concurrency = 1 + + scheduler = _JobScheduler(engine, dispatcher, worker_id=str(worker_id), lock_ttl_s=int(jobs_lock_ttl_s)) + + async def _on_tick() -> None: + nonlocal last_heartbeat_m + nonlocal last_auto_refresh_m + nonlocal cached_enabled_tokens + nonlocal cached_desired_concurrency + now_m = time.monotonic() + + if bool(auto_concurrency) and (now_m - last_auto_refresh_m) >= float(auto_refresh_s): + last_auto_refresh_m = now_m + cached_enabled_tokens = await _count_enabled_tokens(engine) + + cached_desired_concurrency = compute_desired_worker_concurrency( + auto_enabled=bool(auto_concurrency), + enabled_tokens=cached_enabled_tokens, + max_concurrency=int(max_concurrency), + ) + + if now_m - last_heartbeat_m >= heartbeat_interval_s: + last_heartbeat_m = now_m + try: + await set_runtime_setting( + engine, + key="worker.last_seen_at", + value={"at": iso_utc_ms(), "worker_id": worker_id, "pid": int(os.getpid())}, + description="worker heartbeat", + updated_by=f"worker:{worker_id}", + ) + await set_runtime_setting( + engine, + key="worker.concurrency", + value={ + "at": iso_utc_ms(), + "worker_id": worker_id, + "auto": bool(auto_concurrency), + "enabled_tokens": int(cached_enabled_tokens or 0), + "desired": int(cached_desired_concurrency), + "max": int(max_concurrency), + }, + description="worker concurrency", + updated_by=f"worker:{worker_id}", + ) + except Exception: + log.warning("worker_heartbeat_update_failed") + + await refresher.tick(engine) + + if scheduler is not None: + await scheduler.tick( + desired_concurrency=int(cached_desired_concurrency), + max_claims=int(max_jobs_per_tick), + ) + + log.info("worker_start env=%s", settings.app_env) + await run_worker( + max_iterations=max_iterations, + poll_interval_s=poll_interval_s, + on_tick=_on_tick, + ) + log.info("worker_stop") + finally: + if scheduler is not None: + try: + await scheduler.shutdown() + except Exception: + log.warning("worker_scheduler_shutdown_failed") + await engine.dispose() + + +def main(argv: list[str] | None = None) -> None: + _argv = argv or [] + _ = _argv + asyncio.run(main_async()) + + +if __name__ == "__main__": + main() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..be70763 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,7 @@ +import sys +from pathlib import Path + +BACKEND_DIR = Path(__file__).resolve().parents[1] +if str(BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(BACKEND_DIR)) + diff --git a/backend/tests/test_access_token_cache.py b/backend/tests/test_access_token_cache.py new file mode 100644 index 0000000..caa5977 --- /dev/null +++ b/backend/tests/test_access_token_cache.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import asyncio + +from app.pixiv.access_token_cache import AccessTokenCache +from app.pixiv.oauth import PixivOauthToken + + +def test_access_token_cache_reuses_token_until_expiry_margin() -> None: + now_box = {"now": 1000.0} + + def now() -> float: + return float(now_box["now"]) + + cache = AccessTokenCache(now=now, refresh_margin_s=10.0) + calls = {"n": 0} + + async def refresher() -> PixivOauthToken: + calls["n"] += 1 + return PixivOauthToken( + access_token=f"acc{calls['n']}", + token_type="bearer", + expires_in=100, + refresh_token=None, + scope=None, + user_id=None, + ) + + async def _run() -> None: + t1 = await cache.get_or_refresh("k1", refresher=refresher) + t2 = await cache.get_or_refresh("k1", refresher=refresher) + assert calls["n"] == 1 + assert t1.access_token == "acc1" + assert t2.access_token == "acc1" + + now_box["now"] = 1089.0 + t3 = await cache.get_or_refresh("k1", refresher=refresher) + assert calls["n"] == 1 + assert t3.access_token == "acc1" + + now_box["now"] = 1095.0 + t4 = await cache.get_or_refresh("k1", refresher=refresher) + assert calls["n"] == 2 + assert t4.access_token == "acc2" + + asyncio.run(_run()) + diff --git a/backend/tests/test_admin_api_keys.py b/backend/tests/test_admin_api_keys.py new file mode 100644 index 0000000..c5d60eb --- /dev/null +++ b/backend/tests/test_admin_api_keys.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_api_keys_create_list_disable_and_reject_duplicate_name(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_api_keys.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + create_resp = client.post( + "/admin/api/api-keys", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"name": "key1", "api_key": "k_" + ("a" * 40), "description": "d1"}, + ) + assert create_resp.status_code == 200 + body = create_resp.json() + assert body["ok"] is True + assert body["api_key_id"].isdigit() + assert body["hint"] + assert "api_key" not in body + + list_resp = client.get( + "/admin/api/api-keys", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert list_resp.status_code == 200 + list_body = list_resp.json() + assert list_body["ok"] is True + assert len(list_body["items"]) == 1 + item = list_body["items"][0] + assert item["id"] == body["api_key_id"] + assert item["name"] == "key1" + assert item["enabled"] is True + assert "key_hash" not in item + + disable_resp = client.put( + f"/admin/api/api-keys/{body['api_key_id']}", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"enabled": False}, + ) + assert disable_resp.status_code == 200 + disable_body = disable_resp.json() + assert disable_body["ok"] is True + + dup_resp = client.post( + "/admin/api/api-keys", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"name": "key1", "api_key": "k_" + ("b" * 40)}, + ) + assert dup_resp.status_code == 400 + dup_body = dup_resp.json() + assert dup_body["ok"] is False + assert dup_body["code"] == "BAD_REQUEST" + diff --git a/backend/tests/test_admin_audit_list.py b/backend/tests/test_admin_audit_list.py new file mode 100644 index 0000000..13f683f --- /dev/null +++ b/backend/tests/test_admin_audit_list.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.admin_audit import AdminAudit +from app.db.models.base import Base +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_audit_list_cursor_and_parses_detail_json(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_audit_list.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + AdminAudit( + actor="admin", + action="POST", + resource="/admin/api/proxy-pools", + record_id=None, + request_id="req_1", + ip="1.1.1.1", + user_agent="ua", + detail_json=json.dumps({"a": 1}, ensure_ascii=False, separators=(",", ":")), + ), + AdminAudit( + actor="admin", + action="PUT", + resource="/admin/api/proxies/endpoints/1", + record_id="1", + request_id="req_2", + ip=None, + user_agent=None, + detail_json=None, + ), + AdminAudit( + actor=None, + action="POST", + resource="/admin/api/login", + record_id=None, + request_id="req_3", + ip=None, + user_agent=None, + detail_json="{not_json", + ), + ] + ) + await session.commit() + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp1 = client.get( + "/admin/api/audit", + params={"limit": 2}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp1.status_code == 200 + body1 = resp1.json() + assert body1["ok"] is True + assert body1["request_id"] == "req_test" + assert resp1.headers["X-Request-Id"] == "req_test" + assert len(body1["items"]) == 2 + assert body1["next_cursor"].isdigit() + + first_id = int(body1["items"][0]["id"]) + second_id = int(body1["items"][1]["id"]) + assert first_id > second_id + assert body1["next_cursor"] == str(second_id) + + assert body1["items"][0]["detail_json"] == {"raw": "{not_json"} + assert body1["items"][1]["detail_json"] is None + + resp2 = client.get( + "/admin/api/audit", + params={"limit": 5, "cursor": body1["next_cursor"]}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp2.status_code == 200 + body2 = resp2.json() + assert body2["ok"] is True + assert len(body2["items"]) == 1 + assert body2["next_cursor"] == "" + assert body2["items"][0]["detail_json"] == {"a": 1} + + +def test_admin_audit_middleware_records_write_ops(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_audit_middleware.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxy-pools", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"name": "pool1", "description": None, "enabled": True}, + ) + assert resp.status_code == 200 + + async def _fetch_audit() -> tuple[int, str, str, str]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT id, action, resource, request_id FROM admin_audit ORDER BY id ASC", + ) + rows = result.fetchall() + assert len(rows) == 1 + return (int(rows[0][0]), str(rows[0][1]), str(rows[0][2]), str(rows[0][3])) + + audit_id, action, resource, request_id = asyncio.run(_fetch_audit()) + assert audit_id > 0 + assert action == "POST" + assert resource == "/admin/api/proxy-pools" + assert request_id == "req_test" diff --git a/backend/tests/test_admin_bindings_clear_override.py b/backend/tests/test_admin_bindings_clear_override.py new file mode 100644 index 0000000..0d46634 --- /dev/null +++ b/backend/tests/test_admin_bindings_clear_override.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_clear_binding_override(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_clear_binding_override.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + pool_id: int | None = None + binding_id: int | None = None + p1_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + token = PixivToken( + label="t1", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + pool = ProxyPool(name="pool1", description=None, enabled=1) + p1 = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + p2 = ProxyEndpoint( + scheme="socks5", + host="5.6.7.8", + port=1080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + session.add_all([token, pool, p1, p2]) + await session.commit() + await session.refresh(token) + await session.refresh(pool) + await session.refresh(p1) + await session.refresh(p2) + + session.add_all( + [ + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p1.id), enabled=1, weight=1), + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p2.id), enabled=1, weight=1), + ] + ) + await session.commit() + + binding = TokenProxyBinding( + token_id=int(token.id), + pool_id=int(pool.id), + primary_proxy_id=int(p1.id), + override_proxy_id=int(p2.id), + override_expires_at="2099-01-01T00:00:00.000Z", + ) + session.add(binding) + await session.commit() + await session.refresh(binding) + + nonlocal pool_id, binding_id, p1_id + pool_id = int(pool.id) + binding_id = int(binding.id) + p1_id = int(p1.id) + + asyncio.run(_seed()) + assert pool_id is not None + assert binding_id is not None + assert p1_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + clear_resp = client.post( + f"/admin/api/bindings/{binding_id}/clear-override", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert clear_resp.status_code == 200 + body1 = clear_resp.json() + assert body1["ok"] is True + assert body1["binding_id"] == str(binding_id) + assert body1["request_id"] == "req_test" + assert clear_resp.headers["X-Request-Id"] == "req_test" + + list_resp = client.get( + "/admin/api/bindings", + params={"pool_id": pool_id}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert list_resp.status_code == 200 + body2 = list_resp.json() + assert body2["ok"] is True + assert len(body2["items"]) == 1 + item = body2["items"][0] + assert item["override_proxy"] is None + assert item["effective_mode"] == "primary" + assert item["effective_proxy_id"] == str(p1_id) + diff --git a/backend/tests/test_admin_bindings_list.py b/backend/tests/test_admin_bindings_list.py new file mode 100644 index 0000000..4c66900 --- /dev/null +++ b/backend/tests/test_admin_bindings_list.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_list_bindings_filters_by_pool_id(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_list_bindings.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + pool_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + token = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + pool = ProxyPool(name="pool1", description=None, enabled=1) + p1 = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + p2 = ProxyEndpoint( + scheme="socks5", + host="5.6.7.8", + port=1080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + session.add_all([token, pool, p1, p2]) + await session.commit() + await session.refresh(token) + await session.refresh(pool) + await session.refresh(p1) + await session.refresh(p2) + + session.add( + TokenProxyBinding( + token_id=int(token.id), + pool_id=int(pool.id), + primary_proxy_id=int(p1.id), + override_proxy_id=int(p2.id), + override_expires_at="2099-01-01T00:00:00Z", + ) + ) + await session.commit() + + nonlocal pool_id + pool_id = int(pool.id) + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert pool_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.get( + "/admin/api/bindings", + params={"pool_id": pool_id}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + assert len(body["items"]) == 1 + assert body["items"][0]["pool"]["id"] == str(pool_id) + assert body["items"][0]["effective_mode"] == "override" + diff --git a/backend/tests/test_admin_bindings_override.py b/backend/tests/test_admin_bindings_override.py new file mode 100644 index 0000000..596f5aa --- /dev/null +++ b/backend/tests/test_admin_bindings_override.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_set_binding_override_and_effective_mode(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_set_binding_override.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + pool_id: int | None = None + binding_id: int | None = None + p2_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + token = PixivToken( + label="t1", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + pool = ProxyPool(name="pool1", description=None, enabled=1) + p1 = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + p2 = ProxyEndpoint( + scheme="socks5", + host="5.6.7.8", + port=1080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + session.add_all([token, pool, p1, p2]) + await session.commit() + await session.refresh(token) + await session.refresh(pool) + await session.refresh(p1) + await session.refresh(p2) + + session.add_all( + [ + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p1.id), enabled=1, weight=1), + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p2.id), enabled=1, weight=1), + ] + ) + await session.commit() + + binding = TokenProxyBinding( + token_id=int(token.id), + pool_id=int(pool.id), + primary_proxy_id=int(p1.id), + override_proxy_id=None, + override_expires_at=None, + ) + session.add(binding) + await session.commit() + await session.refresh(binding) + + nonlocal pool_id, binding_id, p2_id + pool_id = int(pool.id) + binding_id = int(binding.id) + p2_id = int(p2.id) + + asyncio.run(_seed()) + assert pool_id is not None + assert binding_id is not None + assert p2_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + override_resp = client.post( + f"/admin/api/bindings/{binding_id}/override", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"override_proxy_id": p2_id, "ttl_ms": 600000, "reason": "manual_override"}, + ) + assert override_resp.status_code == 200 + body1 = override_resp.json() + assert body1["ok"] is True + assert body1["binding_id"] == str(binding_id) + assert body1["override_proxy_id"] == str(p2_id) + assert body1["override_expires_at"] + assert body1["request_id"] == "req_test" + assert override_resp.headers["X-Request-Id"] == "req_test" + + list_resp = client.get( + "/admin/api/bindings", + params={"pool_id": pool_id}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert list_resp.status_code == 200 + body2 = list_resp.json() + assert body2["ok"] is True + assert len(body2["items"]) == 1 + item = body2["items"][0] + assert item["effective_mode"] == "override" + assert item["effective_proxy_id"] == str(p2_id) + assert item["override_proxy"]["id"] == str(p2_id) + + +def test_admin_set_binding_override_rejects_proxy_not_in_pool(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_set_binding_override_not_in_pool.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + binding_id: int | None = None + p3_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + token = PixivToken( + label="t1", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + pool = ProxyPool(name="pool1", description=None, enabled=1) + p1 = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + p2 = ProxyEndpoint( + scheme="socks5", + host="5.6.7.8", + port=1080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + p3 = ProxyEndpoint( + scheme="http", + host="9.9.9.9", + port=9999, + username="", + password_enc="", + enabled=1, + source="manual", + ) + session.add_all([token, pool, p1, p2, p3]) + await session.commit() + await session.refresh(token) + await session.refresh(pool) + await session.refresh(p1) + await session.refresh(p2) + await session.refresh(p3) + + session.add_all( + [ + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p1.id), enabled=1, weight=1), + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p2.id), enabled=1, weight=1), + ] + ) + await session.commit() + + binding = TokenProxyBinding( + token_id=int(token.id), + pool_id=int(pool.id), + primary_proxy_id=int(p1.id), + override_proxy_id=None, + override_expires_at=None, + ) + session.add(binding) + await session.commit() + await session.refresh(binding) + + nonlocal binding_id, p3_id + binding_id = int(binding.id) + p3_id = int(p3.id) + + asyncio.run(_seed()) + assert binding_id is not None + assert p3_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + f"/admin/api/bindings/{binding_id}/override", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"override_proxy_id": p3_id, "ttl_ms": 600000, "reason": "manual_override"}, + ) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" + diff --git a/backend/tests/test_admin_bindings_recompute.py b/backend/tests/test_admin_bindings_recompute.py new file mode 100644 index 0000000..2156528 --- /dev/null +++ b/backend/tests/test_admin_bindings_recompute.py @@ -0,0 +1,394 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_recompute_bindings_creates_rows_and_respects_capacity(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_recompute_bindings.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + token_ids: list[int] = [] + pool_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + pool = ProxyPool(name="pool1", description=None, enabled=1) + p1 = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + p2 = ProxyEndpoint( + scheme="socks5", + host="5.6.7.8", + port=1080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + t1 = PixivToken( + label="t1", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + t2 = PixivToken( + label="t2", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + t3 = PixivToken( + label="t3", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + session.add_all([pool, p1, p2, t1, t2, t3]) + await session.commit() + await session.refresh(pool) + await session.refresh(p1) + await session.refresh(p2) + await session.refresh(t1) + await session.refresh(t2) + await session.refresh(t3) + + session.add_all( + [ + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p1.id), enabled=1, weight=1), + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p2.id), enabled=1, weight=1), + ] + ) + await session.commit() + + nonlocal pool_id + pool_id = int(pool.id) + + nonlocal token_ids + token_ids = [int(t1.id), int(t2.id), int(t3.id)] + + asyncio.run(_seed()) + assert pool_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + recompute_resp = client.post( + "/admin/api/bindings/recompute", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"pool_id": pool_id, "max_tokens_per_proxy": 2}, + ) + assert recompute_resp.status_code == 200 + body1 = recompute_resp.json() + assert body1["ok"] is True + assert body1["pool_id"] == str(pool_id) + assert body1["recomputed"] == 3 + assert body1["request_id"] == "req_test" + assert recompute_resp.headers["X-Request-Id"] == "req_test" + + list_resp = client.get( + "/admin/api/bindings", + params={"pool_id": pool_id}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert list_resp.status_code == 200 + body2 = list_resp.json() + assert body2["ok"] is True + assert body2["request_id"] == "req_test" + assert list_resp.headers["X-Request-Id"] == "req_test" + assert len(body2["items"]) == 3 + + seen_tokens = {int(item["token"]["id"]) for item in body2["items"]} + assert seen_tokens == set(token_ids) + + by_proxy: dict[str, int] = {} + for item in body2["items"]: + assert item["effective_mode"] == "primary" + by_proxy[item["primary_proxy"]["id"]] = by_proxy.get(item["primary_proxy"]["id"], 0) + 1 + assert all(count <= 2 for count in by_proxy.values()) + + +def test_admin_recompute_bindings_rejects_insufficient_capacity(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_recompute_bindings_capacity.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + pool_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + pool = ProxyPool(name="pool1", description=None, enabled=1) + p1 = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + p2 = ProxyEndpoint( + scheme="socks5", + host="5.6.7.8", + port=1080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + tokens = [ + PixivToken( + label=f"t{i}", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + for i in range(5) + ] + session.add_all([pool, p1, p2, *tokens]) + await session.commit() + await session.refresh(pool) + await session.refresh(p1) + await session.refresh(p2) + + session.add_all( + [ + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p1.id), enabled=1, weight=1), + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p2.id), enabled=1, weight=1), + ] + ) + await session.commit() + + nonlocal pool_id + pool_id = int(pool.id) + + asyncio.run(_seed()) + assert pool_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/bindings/recompute", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"pool_id": pool_id, "max_tokens_per_proxy": 2}, + ) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" + + +def test_admin_recompute_bindings_respects_pool_endpoint_weight(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_recompute_bindings_weight.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + pool_id: int | None = None + endpoint_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + pool = ProxyPool(name="pool_weight", description=None, enabled=1) + p1 = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + tokens = [ + PixivToken( + label=f"t{i}", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + for i in range(5) + ] + session.add_all([pool, p1, *tokens]) + await session.commit() + await session.refresh(pool) + await session.refresh(p1) + + session.add(ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p1.id), enabled=1, weight=5)) + await session.commit() + + nonlocal pool_id, endpoint_id + pool_id = int(pool.id) + endpoint_id = int(p1.id) + + asyncio.run(_seed()) + assert pool_id is not None + assert endpoint_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/bindings/recompute", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"pool_id": pool_id, "max_tokens_per_proxy": 1}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["pool_id"] == str(pool_id) + assert body["recomputed"] == 5 + + list_resp = client.get( + "/admin/api/bindings", + params={"pool_id": pool_id}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert list_resp.status_code == 200 + items = list_resp.json()["items"] + assert len(items) == 5 + assert {int(it["primary_proxy"]["id"]) for it in items} == {int(endpoint_id)} + + +def test_admin_recompute_bindings_allows_over_capacity_when_strict_false(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_recompute_bindings_non_strict.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + pool_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + pool = ProxyPool(name="pool1", description=None, enabled=1) + p1 = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + p2 = ProxyEndpoint( + scheme="socks5", + host="5.6.7.8", + port=1080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + tokens = [ + PixivToken( + label=f"t{i}", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + for i in range(5) + ] + session.add_all([pool, p1, p2, *tokens]) + await session.commit() + await session.refresh(pool) + await session.refresh(p1) + await session.refresh(p2) + + session.add_all( + [ + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p1.id), enabled=1, weight=1), + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(p2.id), enabled=1, weight=1), + ] + ) + await session.commit() + + nonlocal pool_id + pool_id = int(pool.id) + + asyncio.run(_seed()) + assert pool_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/bindings/recompute", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"pool_id": pool_id, "max_tokens_per_proxy": 2, "strict": False}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["pool_id"] == str(pool_id) + assert body["recomputed"] == 5 + assert body["strict"] is False + assert int(body["capacity"]) == 4 + assert int(body["over_capacity_assigned"]) == 1 + + list_resp = client.get( + "/admin/api/bindings", + params={"pool_id": pool_id}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert list_resp.status_code == 200 + items = list_resp.json()["items"] + assert len(items) == 5 diff --git a/backend/tests/test_admin_hydration_runs_actions.py b/backend/tests/test_admin_hydration_runs_actions.py new file mode 100644 index 0000000..b4cfaec --- /dev/null +++ b/backend/tests/test_admin_hydration_runs_actions.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.hydration_runs import HydrationRun +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_hydration_run_pause_resume_cancel(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_hydration_run_actions.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + created = client.post( + "/admin/api/hydration-runs", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"type": "backfill", "criteria": {}}, + ) + assert created.status_code == 200 + body0 = created.json() + run_id = int(body0["hydration_run_id"]) + job_id = int(body0["job_id"]) + + paused = client.post( + f"/admin/api/hydration-runs/{run_id}/pause", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert paused.status_code == 200 + assert paused.json()["status"] == "paused" + + resumed = client.post( + f"/admin/api/hydration-runs/{run_id}/resume", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resumed.status_code == 200 + assert resumed.json()["status"] == "pending" + + canceled = client.post( + f"/admin/api/hydration-runs/{run_id}/cancel", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert canceled.status_code == 200 + assert canceled.json()["status"] == "canceled" + + async def _verify() -> None: + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + run = await session.get(HydrationRun, run_id) + assert run is not None + assert run.status == "canceled" + + job = await session.get(JobRow, job_id) + assert job is not None + assert job.status == "canceled" + + asyncio.run(_verify()) + diff --git a/backend/tests/test_admin_hydration_runs_create.py b/backend/tests/test_admin_hydration_runs_create.py new file mode 100644 index 0000000..dec63c9 --- /dev/null +++ b/backend/tests/test_admin_hydration_runs_create.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.hydration_runs import HydrationRun +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_create_hydration_run_creates_job(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_create_hydration_run.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/hydration-runs", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"type": "backfill", "criteria": {"missing": ["tags"]}}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["hydration_run_id"].isdigit() + assert body["job_id"].isdigit() + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + run_id = int(body["hydration_run_id"]) + job_id = int(body["job_id"]) + + async def _verify() -> None: + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + run = await session.get(HydrationRun, run_id) + assert run is not None + assert run.status == "pending" + assert run.type == "backfill" + assert json.loads(run.criteria_json or "{}") == {"missing": ["tags"]} + + job = await session.get(JobRow, job_id) + assert job is not None + assert job.status == "pending" + assert job.type == "hydrate_metadata" + assert job.ref_type == "hydration_run" + assert job.ref_id == str(run_id) + + asyncio.run(_verify()) + diff --git a/backend/tests/test_admin_hydration_runs_list_and_manual.py b/backend/tests/test_admin_hydration_runs_list_and_manual.py new file mode 100644 index 0000000..15c8676 --- /dev/null +++ b/backend/tests/test_admin_hydration_runs_list_and_manual.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import sqlalchemy as sa +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.hydration_runs import HydrationRun +from app.db.models.images import Image +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_list_hydration_runs_and_detail(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_hydration_runs_list.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + ids: dict[str, int] = {} + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + run1 = HydrationRun( + type="backfill", + status="running", + criteria_json=json.dumps({"missing": ["geometry", "tags"]}, separators=(",", ":"), ensure_ascii=False), + cursor_json=json.dumps({"image_id": 1001}, separators=(",", ":"), ensure_ascii=False), + total=None, + processed=12, + success=10, + failed=2, + started_at="2026-02-13T01:00:00Z", + finished_at=None, + last_error="sample_error", + updated_at="2026-02-13T01:10:00Z", + ) + run2 = HydrationRun( + type="manual", + status="completed", + criteria_json=json.dumps({}, separators=(",", ":"), ensure_ascii=False), + cursor_json=None, + total=1, + processed=1, + success=1, + failed=0, + started_at="2026-02-13T02:00:00Z", + finished_at="2026-02-13T02:00:10Z", + last_error=None, + updated_at="2026-02-13T02:00:10Z", + ) + session.add_all([run1, run2]) + await session.flush() + + job1 = JobRow( + type="hydrate_metadata", + status="running", + priority=0, + run_after=None, + attempt=1, + max_attempts=3, + payload_json=json.dumps({"hydration_run_id": int(run1.id)}, separators=(",", ":"), ensure_ascii=False), + last_error=None, + locked_by="worker_1", + locked_at="2026-02-13T01:00:01Z", + ref_type="hydration_run", + ref_id=str(int(run1.id)), + updated_at="2026-02-13T01:10:00Z", + ) + job2 = JobRow( + type="hydrate_metadata", + status="completed", + priority=0, + run_after=None, + attempt=1, + max_attempts=3, + payload_json=json.dumps({"hydration_run_id": int(run2.id)}, separators=(",", ":"), ensure_ascii=False), + last_error=None, + locked_by=None, + locked_at=None, + ref_type="hydration_run", + ref_id=str(int(run2.id)), + updated_at="2026-02-13T02:00:10Z", + ) + session.add_all([job1, job2]) + await session.commit() + + nonlocal ids + ids = {"run1": int(run1.id), "run2": int(run2.id)} + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + listed = client.get( + "/admin/api/hydration-runs", + params={"limit": 10}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_list"}, + ) + assert listed.status_code == 200 + body = listed.json() + assert body["ok"] is True + assert body["next_cursor"] == "" + assert body["request_id"] == "req_list" + assert isinstance(body["items"], list) + assert len(body["items"]) == 2 + + items_by_id = {str(item["id"]): item for item in body["items"]} + run1_item = items_by_id[str(ids["run1"])] + assert run1_item["criteria"] == {"missing": ["geometry", "tags"]} + assert run1_item["cursor"] == {"image_id": 1001} + assert run1_item["latest_job"]["status"] == "running" + + detail = client.get( + f"/admin/api/hydration-runs/{ids['run1']}", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_detail"}, + ) + assert detail.status_code == 200 + item = detail.json()["item"] + assert item["id"] == str(ids["run1"]) + assert item["status"] == "running" + assert item["latest_job"]["locked_by"] == "worker_1" + + +def test_admin_create_manual_hydration_job_by_image_or_illust(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_hydration_manual.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + seeded_image_id = 0 + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + image = Image( + illust_id=987654, + page_index=0, + ext="jpg", + original_url="https://i.pximg.net/img-original/img/2026/02/14/00/00/00/987654_p0.jpg", + proxy_path="/i/1.jpg", + random_key=0.1234, + status=1, + ) + session.add(image) + await session.commit() + await session.refresh(image) + + nonlocal seeded_image_id + seeded_image_id = int(image.id) + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + created_by_image = client.post( + "/admin/api/hydration-runs/manual", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_m1"}, + json={"image_id": seeded_image_id}, + ) + assert created_by_image.status_code == 200 + body1 = created_by_image.json() + assert body1["ok"] is True + assert body1["created"] is True + assert body1["illust_id"] == "987654" + assert body1["job_id"].isdigit() + + duplicate = client.post( + "/admin/api/hydration-runs/manual", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_m2"}, + json={"illust_id": 987654}, + ) + assert duplicate.status_code == 200 + body2 = duplicate.json() + assert body2["ok"] is True + assert body2["created"] is False + assert body2["illust_id"] == "987654" + + async def _verify() -> None: + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + rows = ( + ( + await session.execute( + sa.select(JobRow) + .where(JobRow.ref_type == "manual_hydrate", JobRow.ref_id == "987654") + .order_by(JobRow.id.asc()) + ) + ) + .scalars() + .all() + ) + assert len(rows) == 1 + payload = json.loads(rows[0].payload_json) + assert payload["illust_id"] == 987654 + assert rows[0].status == "pending" + + asyncio.run(_verify()) diff --git a/backend/tests/test_admin_hydration_stats.py b/backend/tests/test_admin_hydration_stats.py new file mode 100644 index 0000000..5733f63 --- /dev/null +++ b/backend/tests/test_admin_hydration_stats.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_summary_includes_hydration_missing_counts(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_summary_hydration.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img1 = Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.com/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + ) + img2 = Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.com/2.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + width=100, + height=200, + x_restrict=0, + ai_type=0, + illust_type=0, + user_id=123, + user_name="u", + title="t", + created_at_pixiv="2026-01-01T00:00:00Z", + bookmark_count=1, + view_count=2, + comment_count=3, + ) + session.add_all([img1, img2]) + await session.flush() + + tag = Tag(name="tag1", translated_name=None) + session.add(tag) + await session.flush() + + session.add(ImageTag(image_id=int(img2.id), tag_id=int(tag.id))) + await session.commit() + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.get( + "/admin/api/summary", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + + hydration = body["counts"]["hydration"] + assert hydration["enabled_images_total"] == 2 + missing = hydration["missing"] + assert missing["tags"] == 1 + assert missing["geometry"] == 1 + assert missing["r18"] == 1 + assert missing["ai"] == 1 + assert missing["illust_type"] == 1 + assert missing["user"] == 1 + assert missing["title"] == 1 + assert missing["created_at"] == 1 + assert missing["popularity"] == 1 diff --git a/backend/tests/test_admin_images_delete.py b/backend/tests/test_admin_images_delete.py new file mode 100644 index 0000000..4cb22c5 --- /dev/null +++ b/backend/tests/test_admin_images_delete.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import sqlalchemy as sa +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_images_delete_bulk_and_clear(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_images_delete.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + ids: dict[str, int] = {} + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img1 = Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.com/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + ) + img2 = Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.com/2.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + ) + session.add_all([img1, img2]) + await session.flush() + + tag = Tag(name="tag1", translated_name=None) + session.add(tag) + await session.flush() + session.add_all([ImageTag(image_id=int(img1.id), tag_id=int(tag.id)), ImageTag(image_id=int(img2.id), tag_id=int(tag.id))]) + + await session.commit() + ids["img1"] = int(img1.id) + ids["img2"] = int(img2.id) + ids["tag"] = int(tag.id) + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + headers = {"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"} + + with TestClient(app) as client: + # single delete + r0 = client.delete(f"/admin/api/images/{ids['img2']}", headers=headers) + assert r0.status_code == 200 + assert r0.json()["ok"] is True + assert r0.json()["image_id"] == str(ids["img2"]) + + # verify image removed, other remains + r0b = client.get("/admin/api/images", headers=headers) + assert r0b.status_code == 200 + assert [int(x["id"]) for x in r0b.json()["items"]] == [ids["img1"]] + + # bulk delete ignores duplicates and reports missing + r1 = client.post( + "/admin/api/images/bulk-delete", + headers=headers, + json={"image_ids": [ids["img1"], ids["img1"], 999999]}, + ) + assert r1.status_code == 200 + b1 = r1.json() + assert b1["ok"] is True + assert int(b1["requested"]) == 2 + assert int(b1["deleted"]) == 1 + assert int(b1["missing"]) == 1 + + r1b = client.get("/admin/api/images", headers=headers) + assert r1b.status_code == 200 + assert r1b.json()["items"] == [] + + # clear removes tags by default + r2 = client.post("/admin/api/images/clear", headers=headers, json={"confirm": True}) + assert r2.status_code == 200 + assert r2.json()["ok"] is True + + # verify tables are empty + Session = create_sessionmaker(app.state.engine) + + async def _check() -> tuple[int, int, int]: + async with Session() as session: + c_images = int((await session.execute(sa.select(sa.func.count()).select_from(Image))).scalar_one()) + c_links = int((await session.execute(sa.select(sa.func.count()).select_from(ImageTag))).scalar_one()) + c_tags = int((await session.execute(sa.select(sa.func.count()).select_from(Tag))).scalar_one()) + return c_images, c_links, c_tags + + c_images, c_links, c_tags = asyncio.run(_check()) + assert c_images == 0 + assert c_links == 0 + assert c_tags == 0 + diff --git a/backend/tests/test_admin_images_list_filters.py b/backend/tests/test_admin_images_list_filters.py new file mode 100644 index 0000000..b932943 --- /dev/null +++ b/backend/tests/test_admin_images_list_filters.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_images_list_missing_filters_and_cursor(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_images_list.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + ids: dict[str, int] = {} + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img1 = Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.com/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + width=None, + height=None, + x_restrict=None, + ai_type=None, + user_id=None, + title=None, + created_at_pixiv=None, + ) + img2 = Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.com/2.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + width=100, + height=200, + x_restrict=0, + ai_type=0, + illust_type=0, + user_id=123, + user_name="u", + title="t", + created_at_pixiv="2026-01-01T00:00:00Z", + bookmark_count=1, + view_count=2, + comment_count=3, + ) + session.add_all([img1, img2]) + await session.flush() + + tag = Tag(name="tag1", translated_name=None) + session.add(tag) + await session.flush() + session.add(ImageTag(image_id=int(img2.id), tag_id=int(tag.id))) + + await session.commit() + ids["img1"] = int(img1.id) + ids["img2"] = int(img2.id) + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + r0 = client.get("/admin/api/images", headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}) + assert r0.status_code == 200 + b0 = r0.json() + assert b0["ok"] is True + assert [int(x["id"]) for x in b0["items"]] == [ids["img2"], ids["img1"]] + + r1 = client.get( + "/admin/api/images", + params={"missing": "tags"}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert r1.status_code == 200 + b1 = r1.json() + assert len(b1["items"]) == 1 + assert int(b1["items"][0]["id"]) == ids["img1"] + + r2 = client.get( + "/admin/api/images", + params={"missing": "geometry"}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert r2.status_code == 200 + assert int(r2.json()["items"][0]["id"]) == ids["img1"] + + r2b = client.get( + "/admin/api/images", + params={"missing": "popularity"}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert r2b.status_code == 200 + b2b = r2b.json() + assert len(b2b["items"]) == 1 + assert int(b2b["items"][0]["id"]) == ids["img1"] + + r2c = client.get( + "/admin/api/images", + params={"missing": "illust_type"}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert r2c.status_code == 200 + b2c = r2c.json() + assert len(b2c["items"]) == 1 + assert int(b2c["items"][0]["id"]) == ids["img1"] + + r3 = client.get( + "/admin/api/images", + params={"limit": 1}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert r3.status_code == 200 + b3 = r3.json() + assert len(b3["items"]) == 1 + assert b3["next_cursor"] == str(ids["img2"]) + + r4 = client.get( + "/admin/api/images", + params={"limit": 1, "cursor": b3["next_cursor"]}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert r4.status_code == 200 + b4 = r4.json() + assert len(b4["items"]) == 1 + assert int(b4["items"][0]["id"]) == ids["img1"] + + bad = client.get( + "/admin/api/images", + params={"missing": "nope"}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert bad.status_code == 400 + assert bad.json()["ok"] is False diff --git a/backend/tests/test_admin_imports_get.py b/backend/tests/test_admin_imports_get.py new file mode 100644 index 0000000..e3f35a6 --- /dev/null +++ b/backend/tests/test_admin_imports_get.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_get_import_returns_item_and_job(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_get_import.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + text = "\n".join( + [ + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg", + "https://www.pixiv.net/artworks/12345678", + ] + ) + + create_resp = client.post( + "/admin/api/imports", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"text": text, "dry_run": False, "hydrate_on_import": False, "source": "manual"}, + ) + assert create_resp.status_code == 200 + import_id = int(create_resp.json()["import_id"]) + + get_resp = client.get( + f"/admin/api/imports/{import_id}", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert get_resp.status_code == 200 + body = get_resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert get_resp.headers["X-Request-Id"] == "req_test" + + item = body["item"] + assert item["import"]["id"] == str(import_id) + assert item["import"]["accepted"] == 1 + assert item["job"] is not None + assert item["job"]["id"].isdigit() + + +def test_admin_get_import_not_found_returns_404(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_get_import_not_found.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.get( + "/admin/api/imports/999", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 404 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "NOT_FOUND" + assert body["request_id"] == "req_test" + diff --git a/backend/tests/test_admin_imports_json.py b/backend/tests/test_admin_imports_json.py new file mode 100644 index 0000000..3de00c1 --- /dev/null +++ b/backend/tests/test_admin_imports_json.py @@ -0,0 +1,363 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app +from app.worker import build_default_dispatcher, poll_and_execute_jobs + + +def test_admin_imports_json_happy_path(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_imports.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + text = "\n".join( + [ + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg", + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg", + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p1.png", + "https://www.pixiv.net/artworks/12345678", + ] + ) + + resp = client.post( + "/admin/api/imports", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"text": text, "dry_run": False, "hydrate_on_import": False, "source": "manual"}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["accepted"] == 2 + assert body["deduped"] == 1 + assert len(body["errors"]) == 1 + assert body["import_id"].isdigit() + assert body["job_id"].isdigit() + executed_inline = bool(body.get("executed_inline")) + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + if not executed_inline: + async def _run_worker() -> None: + dispatcher = build_default_dispatcher(app.state.engine) + ran = await poll_and_execute_jobs(app.state.engine, dispatcher, worker_id="test-worker", max_jobs=10) + assert ran >= 1 + + asyncio.run(_run_worker()) + + async def _fetch_import_counts() -> tuple[int, int, int, int]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql("SELECT total, accepted, success, failed FROM imports") + row = result.fetchone() + assert row is not None + return (int(row[0]), int(row[1]), int(row[2]), int(row[3])) + + assert asyncio.run(_fetch_import_counts()) == (4, 2, 2, 1) + + +def test_admin_imports_multipart_happy_path(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_imports_multipart.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + text = "\n".join( + [ + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg", + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg", + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p1.png", + "https://www.pixiv.net/artworks/12345678", + ] + ) + + resp = client.post( + "/admin/api/imports", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + data={"dry_run": "false", "hydrate_on_import": "false", "source": "manual"}, + files={"file": ("urls.txt", text.encode("utf-8"), "text/plain")}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["accepted"] == 2 + assert body["deduped"] == 1 + assert len(body["errors"]) == 1 + assert body["import_id"].isdigit() + assert body["job_id"].isdigit() + executed_inline = bool(body.get("executed_inline")) + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + import_id = int(body["import_id"]) + + if not executed_inline: + async def _run_worker() -> None: + dispatcher = build_default_dispatcher(app.state.engine) + ran = await poll_and_execute_jobs(app.state.engine, dispatcher, worker_id="test-worker", max_jobs=10) + assert ran >= 1 + + asyncio.run(_run_worker()) + + async def _fetch_import_counts() -> tuple[int, int, int, int]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT total, accepted, success, failed FROM imports WHERE id = ?", + (import_id,), + ) + row = result.fetchone() + assert row is not None + return (int(row[0]), int(row[1]), int(row[2]), int(row[3])) + + assert asyncio.run(_fetch_import_counts()) == (4, 2, 2, 1) + + +def test_admin_imports_dry_run_preview_does_not_write_db(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_imports_dry_run.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + text = "\n".join( + [ + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg", + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg", + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p1.png", + "https://www.pixiv.net/artworks/12345678", + ] + ) + + resp = client.post( + "/admin/api/imports", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"text": text, "dry_run": True, "hydrate_on_import": False, "source": "manual"}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["accepted"] == 2 + assert body["deduped"] == 1 + assert len(body["errors"]) == 1 + assert len(body["preview"]) == 2 + assert body["import_id"] == "" + assert body["job_id"] == "" + assert body["request_id"] == "req_test" + + async def _count_images() -> int: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql("SELECT COUNT(*) FROM images") + return int(result.scalar_one()) + + assert asyncio.run(_count_images()) == 0 + + +def test_admin_imports_rollback_disable_and_delete(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_imports_rollback.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + text = "\n".join( + [ + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg", + "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p1.png", + ] + ) + + create_resp = client.post( + "/admin/api/imports", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"text": text, "dry_run": False, "hydrate_on_import": False, "source": "manual"}, + ) + assert create_resp.status_code == 200 + create_body = create_resp.json() + import_id = int(create_body["import_id"]) + executed_inline = bool(create_body.get("executed_inline")) + + if not executed_inline: + async def _run_worker() -> None: + dispatcher = build_default_dispatcher(app.state.engine) + ran = await poll_and_execute_jobs(app.state.engine, dispatcher, worker_id="test-worker", max_jobs=10) + assert ran >= 1 + + asyncio.run(_run_worker()) + + disable_resp = client.post( + f"/admin/api/imports/{import_id}/rollback", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"mode": "disable"}, + ) + assert disable_resp.status_code == 200 + assert disable_resp.json()["updated"] == 2 + + async def _count_status(status: int) -> int: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT COUNT(*) FROM images WHERE created_import_id = ? AND status = ?", + (import_id, status), + ) + return int(result.scalar_one()) + + assert asyncio.run(_count_status(2)) == 2 + + delete_resp = client.post( + f"/admin/api/imports/{import_id}/rollback", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"mode": "delete"}, + ) + assert delete_resp.status_code == 200 + assert delete_resp.json()["updated"] == 2 + assert asyncio.run(_count_status(4)) == 2 + + +def test_admin_imports_invalid_body_returns_400(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_imports_invalid_body.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/imports", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"text": "", "dry_run": False}, + ) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" + + +def test_admin_imports_rollback_not_found_returns_404(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_imports_rollback_not_found.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/imports/999/rollback", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"mode": "disable"}, + ) + assert resp.status_code == 404 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "NOT_FOUND" + assert body["request_id"] == "req_test" + + +def test_admin_imports_rollback_invalid_id_returns_400(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_imports_rollback_invalid_id.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/imports/0/rollback", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"mode": "disable"}, + ) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" diff --git a/backend/tests/test_admin_jobs_actions.py b/backend/tests/test_admin_jobs_actions.py new file mode 100644 index 0000000..d5e1304 --- /dev/null +++ b/backend/tests/test_admin_jobs_actions.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_job_retry_cancel_move_to_dlq(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_job_actions.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + ids: dict[str, int] = {} + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + j1 = JobRow( + type="import_images", + status="dlq", + priority=0, + run_after=None, + attempt=3, + max_attempts=3, + payload_json=json.dumps({"a": 1}, separators=(",", ":"), ensure_ascii=False), + last_error="err", + locked_by=None, + locked_at=None, + ref_type=None, + ref_id=None, + ) + j2 = JobRow( + type="proxy_probe", + status="pending", + priority=0, + run_after=None, + attempt=0, + max_attempts=3, + payload_json=json.dumps({"b": 2}, separators=(",", ":"), ensure_ascii=False), + last_error=None, + locked_by=None, + locked_at=None, + ref_type=None, + ref_id=None, + ) + j3 = JobRow( + type="hydrate_metadata", + status="failed", + priority=0, + run_after=None, + attempt=1, + max_attempts=3, + payload_json=json.dumps({"c": 3}, separators=(",", ":"), ensure_ascii=False), + last_error="boom", + locked_by=None, + locked_at=None, + ref_type=None, + ref_id=None, + ) + session.add_all([j1, j2, j3]) + await session.commit() + await session.refresh(j1) + await session.refresh(j2) + await session.refresh(j3) + + nonlocal ids + ids = {"retry": int(j1.id), "cancel": int(j2.id), "dlq": int(j3.id)} + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + r1 = client.post( + f"/admin/api/jobs/{ids['retry']}/retry", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert r1.status_code == 200 + b1 = r1.json() + assert b1["ok"] is True + assert b1["status"] == "pending" + assert b1["job_id"] == str(ids["retry"]) + + r2 = client.post( + f"/admin/api/jobs/{ids['cancel']}/cancel", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert r2.status_code == 200 + b2 = r2.json() + assert b2["ok"] is True + assert b2["status"] == "canceled" + assert b2["job_id"] == str(ids["cancel"]) + + r3 = client.post( + f"/admin/api/jobs/{ids['dlq']}/move-to-dlq", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert r3.status_code == 200 + b3 = r3.json() + assert b3["ok"] is True + assert b3["status"] == "dlq" + assert b3["job_id"] == str(ids["dlq"]) + + l = client.get( + "/admin/api/jobs", + params={"limit": 10}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert l.status_code == 200 + items = {int(it["id"]): it for it in l.json()["items"]} + assert items[ids["retry"]]["status"] == "pending" + assert items[ids["cancel"]]["status"] == "canceled" + assert items[ids["dlq"]]["status"] == "dlq" + diff --git a/backend/tests/test_admin_jobs_detail_and_actions.py b/backend/tests/test_admin_jobs_detail_and_actions.py new file mode 100644 index 0000000..ba96ecc --- /dev/null +++ b/backend/tests/test_admin_jobs_detail_and_actions.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_job_detail_returns_payload(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_job_detail.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + ids: dict[str, int] = {} + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + job = JobRow( + type="proxy_probe", + status="failed", + priority=0, + run_after=None, + attempt=1, + max_attempts=3, + payload_json=json.dumps({"a": 1}, separators=(",", ":"), ensure_ascii=False), + last_error="boom", + locked_by=None, + locked_at=None, + ref_type="test", + ref_id="r1", + ) + session.add(job) + await session.commit() + await session.refresh(job) + ids["job"] = int(job.id) + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.get( + f"/admin/api/jobs/{ids['job']}", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + assert body["item"]["id"] == str(ids["job"]) + assert body["item"]["status"] == "failed" + assert body["item"]["payload"] == {"a": 1} + assert body["item"]["payload_json"] == '{"a":1}' + + missing = client.get( + "/admin/api/jobs/999999", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert missing.status_code == 404 + assert missing.json()["ok"] is False + diff --git a/backend/tests/test_admin_jobs_list.py b/backend/tests/test_admin_jobs_list.py new file mode 100644 index 0000000..7f3343b --- /dev/null +++ b/backend/tests/test_admin_jobs_list.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_list_jobs_cursor_and_filters(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_list_jobs.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + JobRow( + type="import_images", + status="pending", + priority=0, + run_after=None, + attempt=0, + max_attempts=3, + payload_json=json.dumps({"a": 1}, separators=(",", ":"), ensure_ascii=False), + last_error=None, + locked_by=None, + locked_at=None, + ref_type=None, + ref_id=None, + ), + JobRow( + type="proxy_probe", + status="failed", + priority=0, + run_after=None, + attempt=1, + max_attempts=3, + payload_json=json.dumps({"b": 2}, separators=(",", ":"), ensure_ascii=False), + last_error="err", + locked_by=None, + locked_at=None, + ref_type=None, + ref_id=None, + ), + JobRow( + type="import_images", + status="completed", + priority=0, + run_after=None, + attempt=1, + max_attempts=3, + payload_json=json.dumps({"c": 3}, separators=(",", ":"), ensure_ascii=False), + last_error=None, + locked_by=None, + locked_at=None, + ref_type=None, + ref_id=None, + ), + ] + ) + await session.commit() + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp1 = client.get( + "/admin/api/jobs", + params={"limit": 2}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp1.status_code == 200 + body1 = resp1.json() + assert body1["ok"] is True + assert body1["request_id"] == "req_test" + assert resp1.headers["X-Request-Id"] == "req_test" + assert len(body1["items"]) == 2 + assert body1["next_cursor"].isdigit() + + first_id = int(body1["items"][0]["id"]) + second_id = int(body1["items"][1]["id"]) + assert first_id > second_id + assert body1["next_cursor"] == str(second_id) + + resp2 = client.get( + "/admin/api/jobs", + params={"limit": 2, "cursor": body1["next_cursor"]}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp2.status_code == 200 + body2 = resp2.json() + assert body2["ok"] is True + assert len(body2["items"]) == 1 + assert body2["next_cursor"] == "" + + resp3 = client.get( + "/admin/api/jobs", + params={"status": "failed"}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp3.status_code == 200 + body3 = resp3.json() + assert body3["ok"] is True + assert len(body3["items"]) == 1 + assert body3["items"][0]["status"] == "failed" + + resp4 = client.get( + "/admin/api/jobs", + params={"type": "import_images"}, + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp4.status_code == 200 + body4 = resp4.json() + assert body4["ok"] is True + assert len(body4["items"]) == 2 + assert {item["type"] for item in body4["items"]} == {"import_images"} + diff --git a/backend/tests/test_admin_jwt.py b/backend/tests/test_admin_jwt.py new file mode 100644 index 0000000..aa19915 --- /dev/null +++ b/backend/tests/test_admin_jwt.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import pytest + +from app.core.errors import ApiError, ErrorCode +from app.core.security import create_jwt, decode_jwt, require_admin + + +def test_admin_auth_jwt_roundtrip() -> None: + token = create_jwt(secret_key="secret", subject="admin", ttl_s=60) + claims = decode_jwt(token, secret_key="secret") + assert claims["sub"] == "admin" + assert isinstance(claims["iat"], int) + assert isinstance(claims["exp"], int) + + +def test_admin_auth_jwt_expired() -> None: + token = create_jwt(secret_key="secret", subject="admin", ttl_s=-1) + with pytest.raises(ValueError, match="expired"): + decode_jwt(token, secret_key="secret") + + +def test_admin_auth_jwt_invalid_signature() -> None: + token = create_jwt(secret_key="secret1", subject="admin", ttl_s=60) + with pytest.raises(ValueError, match="signature"): + decode_jwt(token, secret_key="secret2") + + +def test_admin_auth_require_admin_allows_valid_token() -> None: + token = create_jwt(secret_key="secret", subject="admin", ttl_s=60) + claims = require_admin( + {"Authorization": f"Bearer {token}"}, + secret_key="secret", + admin_username="admin", + ) + assert claims["sub"] == "admin" + + +def test_admin_auth_require_admin_rejects_missing_token() -> None: + with pytest.raises(ApiError) as excinfo: + require_admin({}, secret_key="secret", admin_username="admin") + assert excinfo.value.code == ErrorCode.UNAUTHORIZED + assert excinfo.value.status_code == 401 + + +def test_admin_auth_require_admin_rejects_wrong_subject() -> None: + token = create_jwt(secret_key="secret", subject="not-admin", ttl_s=60) + with pytest.raises(ApiError) as excinfo: + require_admin({"Authorization": f"Bearer {token}"}, secret_key="secret", admin_username="admin") + assert excinfo.value.code == ErrorCode.FORBIDDEN + assert excinfo.value.status_code == 403 diff --git a/backend/tests/test_admin_login.py b/backend/tests/test_admin_login.py new file mode 100644 index 0000000..b99fef0 --- /dev/null +++ b/backend/tests/test_admin_login.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_login_happy_path_returns_token(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_login.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("ADMIN_PASSWORD", "pass_test") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.post( + "/admin/api/login", + headers={"X-Request-Id": "req_test"}, + json={"username": "admin", "password": "pass_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + assert isinstance(body["token"], str) and body["token"] + assert "password" not in body + + +def test_admin_login_invalid_credentials_returns_401(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_login_invalid.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("ADMIN_PASSWORD", "pass_test") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.post( + "/admin/api/login", + headers={"X-Request-Id": "req_test"}, + json={"username": "admin", "password": "wrong"}, + ) + assert resp.status_code == 401 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "UNAUTHORIZED" + assert body["request_id"] == "req_test" + + +def test_admin_logout_happy_path(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_logout.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("ADMIN_PASSWORD", "pass_test") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + with TestClient(app) as client: + token = client.post( + "/admin/api/login", + headers={"X-Request-Id": "req_test"}, + json={"username": "admin", "password": "pass_test"}, + ).json()["token"] + + resp = client.post( + "/admin/api/logout", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + +def test_admin_logout_missing_token_returns_401(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_logout_missing.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("ADMIN_PASSWORD", "pass_test") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.post("/admin/api/logout", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 401 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "UNAUTHORIZED" + assert body["request_id"] == "req_test" diff --git a/backend/tests/test_admin_proxies_easy_proxies_import.py b/backend/tests/test_admin_proxies_easy_proxies_import.py new file mode 100644 index 0000000..19d843e --- /dev/null +++ b/backend/tests/test_admin_proxies_easy_proxies_import.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import httpx +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +from app.core.crypto import FieldEncryptor +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_easy_proxies_import_skip_non_easy_proxies(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_easy_proxies_import.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + base_url = "http://easy-proxies.test:9090" + easy_password = "easy_admin_password_secret" + easy_proxy_pass = "easy_proxy_pass_secret" + manual_pass = "manual_pass_secret" + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + app = create_app() + + async def _seed_manual() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="u", + password_enc=encryptor.encrypt_text(manual_pass), + enabled=1, + source="manual", + source_ref=None, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed_manual()) + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == f"{base_url}/api/auth": + data = json.loads((req.content or b"{}").decode("utf-8")) + assert data["password"] == easy_password + return httpx.Response(200, json={"token": "t"}) + if str(req.url) == f"{base_url}/api/export": + assert req.headers.get("Authorization") == "Bearer t" + return httpx.Response( + 200, + text="\n".join( + [ + f"http://u:{easy_proxy_pass}@1.2.3.4:8080", + "socks5://5.6.7.8:1080", + "", + ] + ), + ) + return httpx.Response(404) + + app.state.httpx_transport = httpx.MockTransport(handler) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxies/easy-proxies/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"base_url": base_url, "password": easy_password, "conflict_policy": "skip_non_easy_proxies"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["created"] == 1 + assert body["updated"] == 0 + assert body["skipped"] == 1 + + dumped = json.dumps(body, ensure_ascii=False) + assert easy_password not in dumped + assert easy_proxy_pass not in dumped + assert "password" not in dumped + + async def _fetch_password_and_source(host: str, port: int, username: str) -> tuple[str, str, str | None]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT password_enc, source, source_ref FROM proxy_endpoints WHERE host = ? AND port = ? AND username = ?", + (host, port, username), + ) + row = result.fetchone() + assert row is not None + return (str(row[0]), str(row[1]), str(row[2]) if row[2] is not None else None) + + enc_manual, source_manual, source_ref_manual = asyncio.run(_fetch_password_and_source("1.2.3.4", 8080, "u")) + assert source_manual == "manual" + assert source_ref_manual is None + assert encryptor.decrypt_text(enc_manual) == manual_pass + + enc_new, source_new, source_ref_new = asyncio.run(_fetch_password_and_source("5.6.7.8", 1080, "")) + assert source_new == "easy_proxies" + assert source_ref_new == base_url + assert enc_new == "" + + +def test_admin_easy_proxies_import_allows_missing_password(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_easy_proxies_import_no_password.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + + base_url = "http://easy-proxies.test:9090" + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == f"{base_url}/api/auth": + raise AssertionError("auth should not be called when password is missing") + if str(req.url) == f"{base_url}/api/export": + assert req.headers.get("Authorization") is None + return httpx.Response(200, text="http://1.2.3.4:8080\n") + return httpx.Response(404) + + app.state.httpx_transport = httpx.MockTransport(handler) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxies/easy-proxies/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"base_url": base_url, "conflict_policy": "overwrite"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["created"] == 1 + assert body["updated"] == 0 + assert body["skipped"] == 0 + + dumped = json.dumps(body, ensure_ascii=False) + assert "password" not in dumped + + +def test_admin_easy_proxies_import_clears_blacklist_on_update(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_easy_proxies_import_clear_blacklist.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + + base_url = "http://easy-proxies.test:9090" + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + ProxyEndpoint( + scheme="socks5", + host="5.6.7.8", + port=1080, + username="", + password_enc="", + enabled=1, + source="easy_proxies", + source_ref=base_url, + failure_count=3, + blacklisted_until="2099-01-01T00:00:00Z", + last_error="ConnectError: old", + ) + ) + await session.commit() + + asyncio.run(_seed()) + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == f"{base_url}/api/auth": + return httpx.Response(200, json={"token": "t"}) + if str(req.url) == f"{base_url}/api/export": + assert req.headers.get("Authorization") == "Bearer t" + return httpx.Response(200, text="socks5://5.6.7.8:1080\n") + return httpx.Response(404) + + app.state.httpx_transport = httpx.MockTransport(handler) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxies/easy-proxies/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"base_url": base_url, "password": "pw", "conflict_policy": "overwrite"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["created"] == 0 + assert body["updated"] == 1 + assert body["skipped"] == 0 + + async def _fetch_flags() -> tuple[int, str | None, str | None]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT failure_count, blacklisted_until, last_error FROM proxy_endpoints WHERE scheme='socks5' AND host='5.6.7.8' AND port=1080", + ) + row = result.fetchone() + assert row is not None + return int(row[0] or 0), str(row[1]) if row[1] is not None else None, str(row[2]) if row[2] is not None else None + + failure_count, blacklisted_until, last_error = asyncio.run(_fetch_flags()) + assert failure_count == 3 + assert blacklisted_until is None + assert last_error is None diff --git a/backend/tests/test_admin_proxies_import.py b/backend/tests/test_admin_proxies_import.py new file mode 100644 index 0000000..bf2fffb --- /dev/null +++ b/backend/tests/test_admin_proxies_import.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +from app.core.crypto import FieldEncryptor +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_import_proxies_encrypts_password_and_supports_overwrite(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_import_proxies.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + + password1 = "pass_SECRET_1" + password2 = "pass_SECRET_2" + + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxies/endpoints/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={ + "text": "\n".join( + [ + f"http://u:{password1}@1.2.3.4:8080", + "socks5://5.6.7.8:1080", + "not_a_proxy", + ] + ), + "source": "manual", + "conflict_policy": "skip", + }, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["created"] == 2 + assert body["updated"] == 0 + assert body["skipped"] == 0 + assert len(body["errors"]) == 1 + + dumped = json.dumps(body, ensure_ascii=False) + assert password1 not in dumped + assert "password" not in dumped + + async def _fetch_passwords() -> list[str]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT scheme, host, port, username, password_enc FROM proxy_endpoints ORDER BY id ASC" + ) + rows = result.fetchall() + assert rows is not None + out: list[str] = [] + for scheme, host, port, username, password_enc in rows: + _ = scheme, host, port, username + out.append(str(password_enc)) + return out + + encs = asyncio.run(_fetch_passwords()) + assert len(encs) == 2 + assert encryptor.decrypt_text(encs[0]) == password1 + assert encs[1] == "" + + resp2 = client.post( + "/admin/api/proxies/endpoints/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={ + "text": f"http://u:{password2}@1.2.3.4:8080", + "source": "manual", + "conflict_policy": "overwrite", + }, + ) + assert resp2.status_code == 200 + body2 = resp2.json() + assert body2["created"] == 0 + assert body2["updated"] == 1 + + async def _fetch_first_enc() -> str: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT password_enc FROM proxy_endpoints WHERE host = ? AND port = ? AND username = ?", + ("1.2.3.4", 8080, "u"), + ) + row = result.fetchone() + assert row is not None + return str(row[0]) + + enc = asyncio.run(_fetch_first_enc()) + assert encryptor.decrypt_text(enc) == password2 + + +def test_admin_import_proxies_auto_generates_field_encryption_key(tmp_path: Path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + + db_path = tmp_path / "admin_import_proxies_auto.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.delenv("FIELD_ENCRYPTION_KEY", raising=False) + monkeypatch.delenv("FIELD_ENCRYPTION_KEY_FILE", raising=False) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + key_path = tmp_path / "data" / "field_encryption_key" + assert key_path.exists() + field_key = key_path.read_text(encoding="utf-8").strip() + encryptor = FieldEncryptor.from_key(field_key) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + + password = "pass_SECRET_1" + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxies/endpoints/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={ + "text": f"http://u:{password}@1.2.3.4:8080", + "source": "manual", + "conflict_policy": "skip", + }, + ) + assert resp.status_code == 200 + + async def _fetch_first_enc() -> str: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT password_enc FROM proxy_endpoints WHERE host = ? AND port = ? AND username = ?", + ("1.2.3.4", 8080, "u"), + ) + row = result.fetchone() + assert row is not None + return str(row[0]) + + enc = asyncio.run(_fetch_first_enc()) + assert encryptor.decrypt_text(enc) == password diff --git a/backend/tests/test_admin_proxies_list.py b/backend/tests/test_admin_proxies_list.py new file mode 100644 index 0000000..4bc0c92 --- /dev/null +++ b/backend/tests/test_admin_proxies_list.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_list_proxy_endpoints_does_not_echo_password(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_list_proxies.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="u", + password_enc="enc_dummy", + enabled=1, + source="manual", + ), + ProxyEndpoint( + scheme="http", + host="9.9.9.9", + port=2323, + username="inliver", + password_enc="enc_dummy2", + enabled=1, + source="easy_proxies", + source_ref="http://user:pw_secret@easy.test:15666", + ), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.get( + "/admin/api/proxies/endpoints", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + assert len(body["items"]) == 2 + + dumped = json.dumps(body, ensure_ascii=False) + assert "enc_dummy" not in dumped + assert "enc_dummy2" not in dumped + assert "password" not in dumped + assert "pw_secret" not in dumped + + items = body["items"] + assert all("source" in item for item in items) + assert all("source_ref" in item for item in items) + + easy = next(i for i in items if i["source"] == "easy_proxies") + assert easy["source_ref"] == "http://easy.test:15666" diff --git a/backend/tests/test_admin_proxies_probe.py b/backend/tests/test_admin_proxies_probe.py new file mode 100644 index 0000000..8bbfe6f --- /dev/null +++ b/backend/tests/test_admin_proxies_probe.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_proxies_probe_enqueues_job(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_proxies_probe.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxies/probe", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["job_id"].isdigit() + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + job_id = int(body["job_id"]) + + async def _fetch_job() -> tuple[str, str]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT type, status FROM jobs WHERE id = ?", + (job_id,), + ) + row = result.fetchone() + assert row is not None + return (str(row[0]), str(row[1])) + + job_type, status = asyncio.run(_fetch_job()) + assert job_type == "proxy_probe" + assert status == "pending" + diff --git a/backend/tests/test_admin_proxies_reset_failures.py b/backend/tests/test_admin_proxies_reset_failures.py new file mode 100644 index 0000000..76f6551 --- /dev/null +++ b/backend/tests/test_admin_proxies_reset_failures.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_proxy_reset_failures_clears_blacklist(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_proxy_reset_failures.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", Fernet.generate_key().decode("ascii")) + + app = create_app() + + async def _seed() -> int: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + row = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="u", + password_enc="", + enabled=1, + source="manual", + source_ref=None, + failure_count=9, + blacklisted_until="2099-01-01T00:00:00Z", + last_fail_at="2099-01-01T00:00:00Z", + last_error="ConnectError: old", + ) + session.add(row) + await session.commit() + return int(row.id) + + endpoint_id = asyncio.run(_seed()) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + f"/admin/api/proxies/endpoints/{endpoint_id}/reset-failures", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["endpoint_id"] == str(endpoint_id) + + async def _fetch() -> tuple[int, str | None, str | None, str | None]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT failure_count, blacklisted_until, last_fail_at, last_error FROM proxy_endpoints WHERE id = ?", + (int(endpoint_id),), + ) + row = result.fetchone() + assert row is not None + return ( + int(row[0] or 0), + str(row[1]) if row[1] is not None else None, + str(row[2]) if row[2] is not None else None, + str(row[3]) if row[3] is not None else None, + ) + + failure_count, blacklisted_until, last_fail_at, last_error = asyncio.run(_fetch()) + assert failure_count == 0 + assert blacklisted_until is None + assert last_fail_at is None + assert last_error is None + diff --git a/backend/tests/test_admin_proxies_update_enabled_and_passwordless_import.py b/backend/tests/test_admin_proxies_update_enabled_and_passwordless_import.py new file mode 100644 index 0000000..40f9708 --- /dev/null +++ b/backend/tests/test_admin_proxies_update_enabled_and_passwordless_import.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_update_proxy_endpoint_enabled(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_update_proxy_enabled.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + async with app.state.engine.begin() as conn: + await conn.exec_driver_sql( + "INSERT INTO proxy_endpoints (scheme, host, port, username, password_enc, enabled, source) VALUES (?,?,?,?,?,?,?)", + ("http", "1.2.3.4", 8080, "", "", 1, "manual"), + ) + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.put( + "/admin/api/proxies/endpoints/1", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"enabled": False}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["endpoint_id"] == "1" + assert body["enabled"] is False + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + async def _fetch_enabled() -> int: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql("SELECT enabled FROM proxy_endpoints WHERE id = 1") + row = result.fetchone() + assert row is not None + return int(row[0]) + + assert asyncio.run(_fetch_enabled()) == 0 + + +def test_admin_import_passwordless_proxies_without_encryption_key(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_import_proxy_no_enc_key.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + app.state.settings = replace(app.state.settings, field_encryption_key="") + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxies/endpoints/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={ + "text": "\n".join( + [ + "http://1.2.3.4:8080", + "socks5://5.6.7.8:1080", + "", + ] + ), + "source": "manual", + "conflict_policy": "skip", + }, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["created"] == 2 + assert body["updated"] == 0 + assert body["skipped"] == 0 + assert body["errors"] == [] + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + resp2 = client.post( + "/admin/api/proxies/endpoints/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={ + "text": "http://u:pass_SECRET_1@9.9.9.9:8080", + "source": "manual", + "conflict_policy": "skip", + }, + ) + assert resp2.status_code == 200 + body2 = resp2.json() + assert body2["ok"] is True + assert body2["created"] == 0 + assert body2["updated"] == 0 + assert body2["skipped"] == 0 + assert len(body2["errors"]) == 1 + assert body2["errors"][0]["code"] == "encryption_not_configured" + + dumped = json.dumps(body2, ensure_ascii=False) + assert "pass_SECRET_1" not in dumped + + async def _count_endpoints() -> int: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql("SELECT COUNT(*) FROM proxy_endpoints") + return int(result.scalar_one()) + + assert asyncio.run(_count_endpoints()) == 2 diff --git a/backend/tests/test_admin_proxy_pools_create.py b/backend/tests/test_admin_proxy_pools_create.py new file mode 100644 index 0000000..32d720c --- /dev/null +++ b/backend/tests/test_admin_proxy_pools_create.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_create_proxy_pool_and_reject_duplicate_name(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_create_proxy_pool.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + create_resp = client.post( + "/admin/api/proxy-pools", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"name": "pool1", "description": "d1", "enabled": True}, + ) + assert create_resp.status_code == 200 + body1 = create_resp.json() + assert body1["ok"] is True + assert body1["pool_id"].isdigit() + assert body1["request_id"] == "req_test" + assert create_resp.headers["X-Request-Id"] == "req_test" + + dup_resp = client.post( + "/admin/api/proxy-pools", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"name": "pool1"}, + ) + assert dup_resp.status_code == 400 + body2 = dup_resp.json() + assert body2["ok"] is False + assert body2["code"] == "BAD_REQUEST" + assert body2["request_id"] == "req_test" + diff --git a/backend/tests/test_admin_proxy_pools_endpoints.py b/backend/tests/test_admin_proxy_pools_endpoints.py new file mode 100644 index 0000000..b22d1d8 --- /dev/null +++ b/backend/tests/test_admin_proxy_pools_endpoints.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import sqlalchemy as sa +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_set_proxy_pool_endpoints_replaces_membership(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_set_proxy_pool_endpoints.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + pool_id: int | None = None + ep1_id: int | None = None + ep2_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + pool = ProxyPool(name="pool1", description=None, enabled=1) + session.add(pool) + session.add_all( + [ + ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ), + ProxyEndpoint( + scheme="socks5", + host="5.6.7.8", + port=1080, + username="", + password_enc="", + enabled=1, + source="manual", + ), + ] + ) + await session.commit() + await session.refresh(pool) + nonlocal pool_id, ep1_id, ep2_id + pool_id = int(pool.id) + + eps = ( + ( + await session.execute( + sa.select(ProxyEndpoint).order_by(ProxyEndpoint.id.asc()) + ) + ) + .scalars() + .all() + ) + ep1_id = int(eps[0].id) + ep2_id = int(eps[1].id) + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert pool_id is not None and ep1_id is not None and ep2_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp1 = client.post( + f"/admin/api/proxy-pools/{pool_id}/endpoints", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"items": [{"endpoint_id": ep1_id, "enabled": True, "weight": 2}]}, + ) + assert resp1.status_code == 200 + body1 = resp1.json() + assert body1["created"] == 1 + assert body1["removed"] == 0 + + resp2 = client.post( + f"/admin/api/proxy-pools/{pool_id}/endpoints", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={ + "items": [ + {"endpoint_id": ep1_id, "enabled": False, "weight": 1}, + {"endpoint_id": ep2_id, "enabled": True, "weight": 3}, + ] + }, + ) + assert resp2.status_code == 200 + body2 = resp2.json() + assert body2["created"] == 1 + assert body2["updated"] == 1 + assert body2["removed"] == 0 + + resp3 = client.post( + f"/admin/api/proxy-pools/{pool_id}/endpoints", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"items": []}, + ) + assert resp3.status_code == 200 + assert resp3.json()["removed"] == 2 + + +def test_admin_set_proxy_pool_endpoints_unknown_endpoint_returns_400(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_set_proxy_pool_endpoints_unknown.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + pool_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + pool = ProxyPool(name="pool1", description=None, enabled=1) + session.add(pool) + await session.commit() + await session.refresh(pool) + nonlocal pool_id + pool_id = int(pool.id) + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert pool_id is not None + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + f"/admin/api/proxy-pools/{pool_id}/endpoints", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"items": [{"endpoint_id": 999, "enabled": True, "weight": 1}]}, + ) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" diff --git a/backend/tests/test_admin_proxy_pools_list.py b/backend/tests/test_admin_proxy_pools_list.py new file mode 100644 index 0000000..a728824 --- /dev/null +++ b/backend/tests/test_admin_proxy_pools_list.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.proxy_pools import ProxyPool +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_list_proxy_pools(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_list_proxy_pools.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add(ProxyPool(name="pool1", description="d1", enabled=1)) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.get( + "/admin/api/proxy-pools", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + assert len(body["items"]) == 1 + assert body["items"][0]["name"] == "pool1" + diff --git a/backend/tests/test_admin_proxy_pools_update.py b/backend/tests/test_admin_proxy_pools_update.py new file mode 100644 index 0000000..40c97e8 --- /dev/null +++ b/backend/tests/test_admin_proxy_pools_update.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_update_proxy_pool(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_update_proxy_pool.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + create_resp = client.post( + "/admin/api/proxy-pools", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"name": "pool1"}, + ) + pool_id = int(create_resp.json()["pool_id"]) + + update_resp = client.put( + f"/admin/api/proxy-pools/{pool_id}", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"name": "pool2", "enabled": False}, + ) + assert update_resp.status_code == 200 + body = update_resp.json() + assert body["ok"] is True + assert body["pool_id"] == str(pool_id) + assert body["request_id"] == "req_test" + + list_resp = client.get( + "/admin/api/proxy-pools", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + items = list_resp.json()["items"] + assert items[0]["name"] == "pool2" + assert items[0]["enabled"] is False + + +def test_admin_update_proxy_pool_not_found_returns_404(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_update_proxy_pool_not_found.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.put( + "/admin/api/proxy-pools/999", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"name": "poolx"}, + ) + assert resp.status_code == 404 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "NOT_FOUND" + assert body["request_id"] == "req_test" + diff --git a/backend/tests/test_admin_random_stats.py b/backend/tests/test_admin_random_stats.py new file mode 100644 index 0000000..cc32c49 --- /dev/null +++ b/backend/tests/test_admin_random_stats.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_random_stats_tracks_only_random_requests(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_random_stats.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + async with app.state.engine.begin() as conn: + await conn.exec_driver_sql( + """ +INSERT INTO images (illust_id,page_index,ext,original_url,proxy_path,random_key,status,x_restrict,width,height) +VALUES (123,0,'jpg','https://example.com/123.jpg','/i/123.jpg',0.5,1,0,100,100); +""".strip() + ) + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + # 1) success + resp_ok = client.get("/random?format=json", headers={"X-Request-Id": "req_r1"}) + assert resp_ok.status_code == 200 + assert resp_ok.json()["ok"] is True + + # 2) no match (filtered out) + resp_nm = client.get("/random?format=json&min_width=999999", headers={"X-Request-Id": "req_r2"}) + assert resp_nm.status_code == 404 + + # 3) stats should reflect 2 /random requests (admin calls are not counted) + resp = client.get( + "/admin/api/stats/random", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_stats"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_stats" + + stats = body["stats"] + assert int(stats["total_requests"]) >= 2 + assert int(stats["total_ok"]) >= 1 + assert int(stats["total_error"]) >= 1 + assert int(stats["in_flight"]) == 0 + assert int(stats["last_window_requests"]) >= 2 diff --git a/backend/tests/test_admin_request_logs_cleanup.py b/backend/tests/test_admin_request_logs_cleanup.py new file mode 100644 index 0000000..f53884f --- /dev/null +++ b/backend/tests/test_admin_request_logs_cleanup.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import sqlalchemy as sa +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.request_logs import RequestLog +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_request_logs_cleanup_dry_run_and_delete(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_request_logs_cleanup.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("ADMIN_PASSWORD", "pass_test") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + RequestLog( + created_at="2000-01-01T00:00:00.000Z", + request_id="req_old1", + method="GET", + route="/healthz", + status=200, + duration_ms=1, + ip="127.0.0.1", + user_agent="pytest", + sample_rate=1.0, + ), + RequestLog( + created_at="2000-01-02T00:00:00.000Z", + request_id="req_old2", + method="GET", + route="/healthz", + status=200, + duration_ms=1, + ip="127.0.0.1", + user_agent="pytest", + sample_rate=1.0, + ), + RequestLog( + created_at="2000-01-03T00:00:00.000Z", + request_id="req_old3", + method="GET", + route="/healthz", + status=200, + duration_ms=1, + ip="127.0.0.1", + user_agent="pytest", + sample_rate=1.0, + ), + RequestLog( + created_at="2099-01-01T00:00:00.000Z", + request_id="req_new", + method="GET", + route="/healthz", + status=200, + duration_ms=1, + ip="127.0.0.1", + user_agent="pytest", + sample_rate=1.0, + ), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + token = client.post( + "/admin/api/login", + headers={"X-Request-Id": "req_test"}, + json={"username": "admin", "password": "pass_test"}, + ).json()["token"] + + preview = client.post( + "/admin/api/maintenance/request-logs/cleanup", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"keep_days": 365, "max_delete_rows": 2, "chunk_size": 1, "dry_run": True}, + ) + assert preview.status_code == 200 + body_preview = preview.json() + assert body_preview["ok"] is True + assert body_preview["dry_run"] is True + assert body_preview["would_delete"] == 2 + assert body_preview["has_more"] is True + + cleanup1 = client.post( + "/admin/api/maintenance/request-logs/cleanup", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"keep_days": 365, "max_delete_rows": 2, "chunk_size": 1}, + ) + assert cleanup1.status_code == 200 + body1 = cleanup1.json() + assert body1["ok"] is True + assert body1["dry_run"] is False + assert body1["deleted"] == 2 + assert body1["has_more"] is True + + cleanup2 = client.post( + "/admin/api/maintenance/request-logs/cleanup", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"keep_days": 365, "max_delete_rows": 10, "chunk_size": 10}, + ) + assert cleanup2.status_code == 200 + body2 = cleanup2.json() + assert body2["deleted"] == 1 + assert body2["has_more"] is False + + async def _assert_remaining() -> None: + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + rows = (await session.execute(sa.select(RequestLog).order_by(RequestLog.id.asc()))).scalars().all() + assert len(rows) == 1 + assert rows[0].request_id == "req_new" + + await app.state.engine.dispose() + + asyncio.run(_assert_remaining()) + diff --git a/backend/tests/test_admin_settings_get.py b/backend/tests/test_admin_settings_get.py new file mode 100644 index 0000000..8cb977c --- /dev/null +++ b/backend/tests/test_admin_settings_get.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.runtime_settings import RuntimeSetting +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_get_settings_returns_defaults_and_runtime_overrides(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_get_settings.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate_and_seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + RuntimeSetting(key="proxy.enabled", value_json="true", description=None, updated_by=None), + RuntimeSetting( + key="proxy.route_mode", + value_json=json.dumps("allowlist", separators=(",", ":"), ensure_ascii=False), + description=None, + updated_by=None, + ), + RuntimeSetting( + key="proxy.allowlist_domains", + value_json=json.dumps(["i.pximg.net"], separators=(",", ":"), ensure_ascii=False), + description=None, + updated_by=None, + ), + ] + ) + await session.commit() + + asyncio.run(_migrate_and_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.get( + "/admin/api/settings", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + settings = body["settings"] + assert settings["proxy"]["enabled"] is True + assert settings["proxy"]["fail_closed"] is True + assert settings["proxy"]["route_mode"] == "allowlist" + assert settings["proxy"]["allowlist_domains"] == ["i.pximg.net"] + + assert settings["random"]["default_attempts"] == 3 + assert settings["random"]["default_r18_strict"] is True + assert settings["random"]["fail_cooldown_ms"] == 600000 + assert settings["random"]["strategy"] == "quality" + assert settings["random"]["quality_samples"] == 5 + + recommendation = settings["random"]["recommendation"] + assert recommendation["pick_mode"] == "weighted" + assert recommendation["temperature"] == 1.0 + assert recommendation["score_weights"]["bookmark"] == 4.0 + assert recommendation["score_weights"]["view"] == 0.5 + assert recommendation["multipliers"]["ai"] == 1.0 + assert recommendation["multipliers"]["manga"] == 1.0 diff --git a/backend/tests/test_admin_settings_put.py b/backend/tests/test_admin_settings_put.py new file mode 100644 index 0000000..f31fb34 --- /dev/null +++ b/backend/tests/test_admin_settings_put.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_put_settings_updates_and_get_reflects(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_put_settings.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + put_resp = client.put( + "/admin/api/settings", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={ + "settings": { + "proxy": { + "enabled": True, + "fail_closed": False, + "route_mode": "allowlist", + "allowlist_domains": ["i.pximg.net"], + }, + "random": { + "default_attempts": 5, + "default_r18_strict": False, + "fail_cooldown_ms": 12345, + "strategy": "random", + "quality_samples": 7, + "recommendation": { + "pick_mode": "best", + "temperature": 2.5, + "score_weights": {"bookmark": 10, "view": 1}, + "multipliers": {"ai": 0.5, "manga": 0}, + }, + }, + "security": {"hide_origin_url_in_public_json": False}, + "rate_limit": {"proxy_probe_concurrency": 7}, + } + }, + ) + assert put_resp.status_code == 200 + body1 = put_resp.json() + assert body1["ok"] is True + assert body1["updated"] >= 4 + assert body1["request_id"] == "req_test" + assert put_resp.headers["X-Request-Id"] == "req_test" + + get_resp = client.get( + "/admin/api/settings", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert get_resp.status_code == 200 + body2 = get_resp.json() + assert body2["ok"] is True + + settings = body2["settings"] + assert settings["proxy"]["enabled"] is True + assert settings["proxy"]["fail_closed"] is False + assert settings["proxy"]["route_mode"] == "allowlist" + assert settings["proxy"]["allowlist_domains"] == ["i.pximg.net"] + + assert settings["random"]["default_attempts"] == 5 + assert settings["random"]["default_r18_strict"] is False + assert settings["random"]["fail_cooldown_ms"] == 12345 + assert settings["random"]["strategy"] == "random" + assert settings["random"]["quality_samples"] == 7 + assert settings["random"]["recommendation"]["pick_mode"] == "best" + assert settings["random"]["recommendation"]["temperature"] == 2.5 + assert settings["random"]["recommendation"]["score_weights"]["bookmark"] == 10.0 + assert settings["random"]["recommendation"]["score_weights"]["view"] == 1.0 + assert settings["random"]["recommendation"]["multipliers"]["ai"] == 0.5 + assert settings["random"]["recommendation"]["multipliers"]["manga"] == 0.0 + + assert settings["security"]["hide_origin_url_in_public_json"] is False + assert settings["rate_limit"]["proxy_probe_concurrency"] == 7 + + +def test_admin_put_settings_rejects_invalid_route_mode(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_put_settings_bad_route_mode.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.put( + "/admin/api/settings", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"settings": {"proxy": {"route_mode": "nope"}}}, + ) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" + + +def test_admin_put_settings_rejects_invalid_random_recommendation(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_put_settings_bad_recommendation.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.put( + "/admin/api/settings", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"settings": {"random": {"recommendation": {"pick_mode": "nope"}}}}, + ) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" diff --git a/backend/tests/test_admin_tokens_create.py b/backend/tests/test_admin_tokens_create.py new file mode 100644 index 0000000..b0d1617 --- /dev/null +++ b/backend/tests/test_admin_tokens_create.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +from app.core.crypto import FieldEncryptor +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_admin_create_token_encrypts_and_does_not_echo_refresh_token(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_create_token.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + refresh_token = "rt_secret_123" + resp = client.post( + "/admin/api/tokens", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"label": "acc1", "refresh_token": refresh_token, "enabled": True, "weight": 1.0}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + assert body["token_id"].isdigit() + + dumped = json.dumps(body, ensure_ascii=False) + assert refresh_token not in dumped + assert "refresh_token" not in dumped + + token_id = int(body["token_id"]) + + async def _fetch_row() -> tuple[str, str]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT refresh_token_enc, refresh_token_masked FROM pixiv_tokens WHERE id = ?", + (token_id,), + ) + row = result.fetchone() + assert row is not None + return (str(row[0]), str(row[1])) + + enc, masked = asyncio.run(_fetch_row()) + assert masked == "***" + assert enc != refresh_token + assert FieldEncryptor.from_key(field_key).decrypt_text(enc) == refresh_token + + +def test_admin_create_token_auto_generates_field_encryption_key(tmp_path: Path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + + db_path = tmp_path / "admin_create_token_auto.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.delenv("FIELD_ENCRYPTION_KEY", raising=False) + monkeypatch.delenv("FIELD_ENCRYPTION_KEY_FILE", raising=False) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + key_path = tmp_path / "data" / "field_encryption_key" + assert key_path.exists() + field_key = key_path.read_text(encoding="utf-8").strip() + encryptor = FieldEncryptor.from_key(field_key) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + refresh_token = "rt_secret_123" + resp = client.post( + "/admin/api/tokens", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"label": "acc1", "refresh_token": refresh_token, "enabled": True, "weight": 1.0}, + ) + assert resp.status_code == 200 + + body = resp.json() + token_id = int(body["token_id"]) + + async def _fetch_enc() -> str: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT refresh_token_enc FROM pixiv_tokens WHERE id = ?", + (token_id,), + ) + row = result.fetchone() + assert row is not None + return str(row[0]) + + enc = asyncio.run(_fetch_enc()) + assert encryptor.decrypt_text(enc) == refresh_token diff --git a/backend/tests/test_admin_tokens_delete.py b/backend/tests/test_admin_tokens_delete.py new file mode 100644 index 0000000..aeab821 --- /dev/null +++ b/backend/tests/test_admin_tokens_delete.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_delete_token_removes_bindings(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_delete_token.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + token = PixivToken( + label="t1", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + session.add(token) + + endpoint = ProxyEndpoint( + scheme="http", + host="127.0.0.1", + port=8080, + username="u", + password_enc="", + enabled=1, + source="manual", + source_ref=None, + ) + session.add(endpoint) + + pool = ProxyPool(name="pool1", description=None, enabled=1) + session.add(pool) + await session.flush() + + session.add(ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(endpoint.id), enabled=1, weight=1)) + session.add( + TokenProxyBinding( + token_id=int(token.id), + pool_id=int(pool.id), + primary_proxy_id=int(endpoint.id), + override_proxy_id=None, + override_expires_at=None, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.delete( + "/admin/api/tokens/1", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["token_id"] == "1" + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + async def _counts() -> tuple[int, int]: + async with app.state.engine.connect() as conn: + t_count = int((await conn.exec_driver_sql("SELECT COUNT(*) FROM pixiv_tokens;")).scalar_one()) + b_count = int((await conn.exec_driver_sql("SELECT COUNT(*) FROM token_proxy_bindings;")).scalar_one()) + return t_count, b_count + + token_count, binding_count = asyncio.run(_counts()) + assert token_count == 0 + assert binding_count == 0 + diff --git a/backend/tests/test_admin_tokens_list.py b/backend/tests/test_admin_tokens_list.py new file mode 100644 index 0000000..9f56571 --- /dev/null +++ b/backend/tests/test_admin_tokens_list.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_list_tokens_does_not_echo_refresh_token(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_list_tokens.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + PixivToken( + label="acc1", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.get( + "/admin/api/tokens", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + assert body["items"][0]["refresh_token_masked"] == "***" + + dumped = json.dumps(body, ensure_ascii=False) + assert "enc_dummy" not in dumped + assert "refresh_token_enc" not in dumped diff --git a/backend/tests/test_admin_tokens_reset_failures.py b/backend/tests/test_admin_tokens_reset_failures.py new file mode 100644 index 0000000..584ec6c --- /dev/null +++ b/backend/tests/test_admin_tokens_reset_failures.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_token_reset_failures_clears_backoff_and_error_state(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_token_reset_failures.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + token_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + error_count=2, + backoff_until="2099-01-01T00:00:00Z", + last_fail_at="2099-01-01T00:00:00Z", + last_error_code="TOKEN_REFRESH_FAILED", + last_error_msg="oops", + ) + session.add(row) + await session.commit() + await session.refresh(row) + nonlocal token_id + token_id = int(row.id) + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert token_id is not None + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + f"/admin/api/tokens/{token_id}/reset-failures", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + + async def _fetch_state() -> tuple[int, str | None, str | None, str | None]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT error_count, backoff_until, last_error_code, last_error_msg FROM pixiv_tokens WHERE id = ?", + (token_id,), + ) + row = result.fetchone() + assert row is not None + return ( + int(row[0]), + str(row[1]) if row[1] is not None else None, + str(row[2]) if row[2] is not None else None, + str(row[3]) if row[3] is not None else None, + ) + + error_count, backoff_until, last_error_code, last_error_msg = asyncio.run(_fetch_state()) + assert error_count == 0 + assert backoff_until is None + assert last_error_code is None + assert last_error_msg is None + diff --git a/backend/tests/test_admin_tokens_test_refresh.py b/backend/tests/test_admin_tokens_test_refresh.py new file mode 100644 index 0000000..05129ac --- /dev/null +++ b/backend/tests/test_admin_tokens_test_refresh.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import httpx +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +from app.core.crypto import FieldEncryptor +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_token_test_refresh_success_rotates_refresh_token(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_token_test_refresh.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "client_id_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "client_secret_test") + monkeypatch.setenv("PIXIV_OAUTH_HASH_SECRET", "hash_secret_test") + + app = create_app() + token_id: int | None = None + + refresh_token = "rt_old" + rotated_refresh_token = "rt_new" + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text(refresh_token), + refresh_token_masked="***", + weight=1.0, + ) + session.add(row) + await session.commit() + await session.refresh(row) + nonlocal token_id + token_id = int(row.id) + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert token_id is not None + + def handler(req: httpx.Request) -> httpx.Response: + assert req.method == "POST" + assert str(req.url) == "https://oauth.secure.pixiv.net/auth/token" + body = (req.content or b"").decode("utf-8") + assert f"refresh_token={refresh_token}" in body + return httpx.Response( + 200, + json={ + "response": { + "access_token": "access_token_test", + "token_type": "bearer", + "expires_in": 3600, + "refresh_token": rotated_refresh_token, + "scope": "", + "user": {"id": 123}, + } + }, + ) + + app.state.httpx_transport = httpx.MockTransport(handler) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + f"/admin/api/tokens/{token_id}/test-refresh", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["expires_in"] == 3600 + assert body["user_id"] == "123" + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + dumped = json.dumps(body, ensure_ascii=False) + assert refresh_token not in dumped + assert rotated_refresh_token not in dumped + assert "access_token" not in dumped + + async def _fetch_refresh_token_enc() -> str: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT refresh_token_enc FROM pixiv_tokens WHERE id = ?", + (token_id,), + ) + row = result.fetchone() + assert row is not None + return str(row[0]) + + enc = asyncio.run(_fetch_refresh_token_enc()) + assert encryptor.decrypt_text(enc) == rotated_refresh_token + + +def test_admin_token_test_refresh_failure_returns_502_and_updates_backoff(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_token_test_refresh_fail.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "client_id_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "client_secret_test") + + app = create_app() + token_id: int | None = None + + refresh_token = "rt_old" + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text(refresh_token), + refresh_token_masked="***", + weight=1.0, + ) + session.add(row) + await session.commit() + await session.refresh(row) + nonlocal token_id + token_id = int(row.id) + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert token_id is not None + + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response(400, json={"error": "invalid_grant"}) + + app.state.httpx_transport = httpx.MockTransport(handler) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + f"/admin/api/tokens/{token_id}/test-refresh", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 502 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "TOKEN_REFRESH_FAILED" + assert body["request_id"] == "req_test" + + dumped = json.dumps(body, ensure_ascii=False) + assert refresh_token not in dumped + + async def _fetch_error_state() -> tuple[int, str | None]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT error_count, backoff_until FROM pixiv_tokens WHERE id = ?", + (token_id,), + ) + row = result.fetchone() + assert row is not None + return (int(row[0]), str(row[1]) if row[1] is not None else None) + + error_count, backoff_until = asyncio.run(_fetch_error_state()) + assert error_count == 1 + assert backoff_until is not None and backoff_until + diff --git a/backend/tests/test_admin_tokens_update.py b/backend/tests/test_admin_tokens_update.py new file mode 100644 index 0000000..f1612dd --- /dev/null +++ b/backend/tests/test_admin_tokens_update.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_admin_update_token_label_enabled_weight(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "admin_update_token.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + PixivToken( + label="old", + enabled=1, + refresh_token_enc="enc_dummy", + refresh_token_masked="***", + weight=1.0, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.put( + "/admin/api/tokens/1", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"label": "new", "enabled": False, "weight": 2.5}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["token_id"] == "1" + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + dumped = json.dumps(body, ensure_ascii=False) + assert "refresh_token" not in dumped + assert "enc_dummy" not in dumped + + async def _fetch_row() -> tuple[str | None, int, float, str]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + "SELECT label, enabled, weight, refresh_token_enc FROM pixiv_tokens WHERE id = 1;" + ) + row = result.fetchone() + assert row is not None + return (row[0], int(row[1]), float(row[2]), str(row[3])) + + label, enabled, weight, refresh_token_enc = asyncio.run(_fetch_row()) + assert label == "new" + assert enabled == 0 + assert weight == 2.5 + assert refresh_token_enc == "enc_dummy" + diff --git a/backend/tests/test_authors_list.py b/backend/tests/test_authors_list.py new file mode 100644 index 0000000..bab343b --- /dev/null +++ b/backend/tests/test_authors_list.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import asyncio +import sqlite3 +from pathlib import Path + +from fastapi.testclient import TestClient +import pytest + +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_authors_list_search_and_pagination(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "authors_list.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=100, + page_index=0, + ext="jpg", + original_url="https://example.test/100.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + user_id=1, + user_name="Alice", + ), + Image( + illust_id=101, + page_index=0, + ext="jpg", + original_url="https://example.test/101.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + user_id=1, + user_name="Alice", + ), + Image( + illust_id=102, + page_index=0, + ext="jpg", + original_url="https://example.test/102.jpg", + proxy_path="/i/3.jpg", + random_key=0.3, + user_id=2, + user_name="Bob", + ), + Image( + illust_id=103, + page_index=0, + ext="jpg", + original_url="https://example.test/103.jpg", + proxy_path="/i/4.jpg", + random_key=0.4, + user_id=None, + user_name=None, + ), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + page1 = client.get("/authors", params={"limit": 1}, headers={"X-Request-Id": "req_test"}) + assert page1.status_code == 200 + body1 = page1.json() + assert body1["ok"] is True + assert body1["request_id"] == "req_test" + assert page1.headers["X-Request-Id"] == "req_test" + assert [i["user_id"] for i in body1["items"]] == ["1"] + assert body1["items"][0]["count_images"] == 2 + assert body1["next_cursor"] == "1" + + page2 = client.get( + "/authors", + params={"limit": 1, "cursor": body1["next_cursor"]}, + headers={"X-Request-Id": "req_test"}, + ) + assert page2.status_code == 200 + body2 = page2.json() + assert [i["user_id"] for i in body2["items"]] == ["2"] + assert body2["next_cursor"] == "" + + search = client.get("/authors", params={"q": "Ali"}, headers={"X-Request-Id": "req_test"}) + assert search.status_code == 200 + body3 = search.json() + assert [i["user_id"] for i in body3["items"]] == ["1"] + + +def test_authors_list_invalid_cursor_returns_400(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "authors_list_invalid_cursor.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + await app.state.engine.dispose() + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.get("/authors", params={"cursor": "nope"}, headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" + + +def test_authors_list_uses_fts_when_available(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "authors_list_fts.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> bool: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=100, + page_index=0, + ext="jpg", + original_url="https://example.test/100.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + user_id=1, + user_name="Alice", + ) + ) + await session.commit() + + await app.state.engine.dispose() + + try: + con = sqlite3.connect(db_path) + try: + con.execute("CREATE VIRTUAL TABLE authors_fts USING fts5(user_name, tokenize='trigram');") + except sqlite3.OperationalError: + con.execute("CREATE VIRTUAL TABLE authors_fts USING fts5(user_name);") + con.execute("INSERT INTO authors_fts(rowid, user_name) VALUES (?,?);", (1, "Alicia")) + con.commit() + return True + except sqlite3.OperationalError: + return False + finally: + try: + con.close() + except Exception: + pass + + fts_ok = asyncio.run(_seed()) + if not fts_ok: + pytest.skip("fts5 not available in sqlite3 build") + + with TestClient(app) as client: + resp = client.get("/authors", params={"q": "Alicia"}, headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + body = resp.json() + assert [i["user_id"] for i in body["items"]] == ["1"] diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py new file mode 100644 index 0000000..e137814 --- /dev/null +++ b/backend/tests/test_config.py @@ -0,0 +1,48 @@ +import pytest + +from app.core.config import load_settings +from app.core.crypto import FieldEncryptor + + +def test_load_settings_dev_defaults() -> None: + s = load_settings({}) + assert s.app_env == "dev" + assert s.database_url + assert s.secret_key + assert s.admin_username == "admin" + assert s.admin_password == "admin" + + +def test_load_settings_prod_requires_secrets() -> None: + with pytest.raises(ValueError): + load_settings({"APP_ENV": "prod"}) + + +def test_load_settings_dev_auto_generates_field_encryption_key(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.delenv("FIELD_ENCRYPTION_KEY", raising=False) + monkeypatch.delenv("FIELD_ENCRYPTION_KEY_FILE", raising=False) + + s1 = load_settings() + assert s1.field_encryption_key + FieldEncryptor.from_key(s1.field_encryption_key) + + key_path = tmp_path / "data" / "field_encryption_key" + assert key_path.exists() + + s2 = load_settings() + assert s2.field_encryption_key == s1.field_encryption_key + + +def test_load_settings_dev_pixiv_oauth_empty_by_default(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.delenv("PIXIV_OAUTH_CLIENT_ID", raising=False) + monkeypatch.delenv("PIXIV_OAUTH_CLIENT_SECRET", raising=False) + monkeypatch.delenv("PIXIV_OAUTH_HASH_SECRET", raising=False) + + s = load_settings() + assert s.pixiv_oauth_client_id == "" + assert s.pixiv_oauth_client_secret == "" + assert s.pixiv_oauth_hash_secret == "" diff --git a/backend/tests/test_crypto.py b/backend/tests/test_crypto.py new file mode 100644 index 0000000..cc71de4 --- /dev/null +++ b/backend/tests/test_crypto.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import pytest +from cryptography.fernet import Fernet + +from app.core.crypto import FieldEncryptor, MASKED, mask_secret + + +def test_mask_secret() -> None: + assert mask_secret(None) == "" + assert mask_secret("") == "" + assert mask_secret(" ") == "" + assert mask_secret("secret") == MASKED + + +def test_encrypt_decrypt_roundtrip() -> None: + key = Fernet.generate_key().decode("utf-8") + crypto = FieldEncryptor.from_key(key) + + plaintext = "refresh_token_123" + token = crypto.encrypt_text(plaintext) + + assert isinstance(token, str) + assert token != plaintext + assert crypto.decrypt_text(token) == plaintext + + +def test_requires_key() -> None: + with pytest.raises(ValueError, match="FIELD_ENCRYPTION_KEY is required"): + FieldEncryptor.from_key("") + + +def test_invalid_key_rejected() -> None: + with pytest.raises(ValueError, match="Invalid FIELD_ENCRYPTION_KEY"): + FieldEncryptor.from_key("not-a-fernet-key") + + +def test_invalid_token_rejected() -> None: + key = Fernet.generate_key().decode("utf-8") + crypto = FieldEncryptor.from_key(key) + with pytest.raises(ValueError, match="Invalid encrypted value"): + crypto.decrypt_text("not-a-fernet-token") + diff --git a/backend/tests/test_e2e_minimal_import_random.py b/backend/tests/test_e2e_minimal_import_random.py new file mode 100644 index 0000000..e664f03 --- /dev/null +++ b/backend/tests/test_e2e_minimal_import_random.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app +from app.worker import build_default_dispatcher, poll_and_execute_jobs + + +def test_e2e_import_10_then_random_json_ok_and_no_match(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "e2e_minimal.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + + lines: list[str] = [] + for i in range(10): + illust_id = 12345670 + i + lines.append( + f"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/{illust_id}_p0.jpg" + ) + + with TestClient(app) as client: + import_resp = client.post( + "/admin/api/imports", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"text": "\n".join(lines), "dry_run": False, "hydrate_on_import": False, "source": "manual"}, + ) + assert import_resp.status_code == 200 + import_body = import_resp.json() + assert import_body["ok"] is True + assert import_body["accepted"] == 10 + assert import_body["deduped"] == 0 + assert import_body["request_id"] == "req_test" + + if not bool(import_body.get("executed_inline")): + async def _run_worker() -> None: + dispatcher = build_default_dispatcher(app.state.engine) + ran = await poll_and_execute_jobs(app.state.engine, dispatcher, worker_id="test-worker", max_jobs=10) + assert ran >= 1 + + asyncio.run(_run_worker()) + + ok_resp = client.get("/random", params={"format": "json", "attempts": 1, "r18_strict": 0}) + assert ok_resp.status_code == 200 + ok_body = ok_resp.json() + assert ok_body["ok"] is True + assert ok_body["code"] == "OK" + assert ok_body["data"]["image"]["id"].isdigit() + + nomatch_resp = client.get( + "/random", + params={"format": "json", "attempts": 1, "r18_strict": 0, "min_width": 999999}, + ) + assert nomatch_resp.status_code == 404 + nomatch_body = nomatch_resp.json() + assert nomatch_body["ok"] is False + assert nomatch_body["code"] == "NO_MATCH" diff --git a/backend/tests/test_easy_proxies_auto_refresh.py b/backend/tests/test_easy_proxies_auto_refresh.py new file mode 100644 index 0000000..d06728e --- /dev/null +++ b/backend/tests/test_easy_proxies_auto_refresh.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.easy_proxies.auto_refresh import EasyProxiesAutoRefreshConfig, EasyProxiesAutoRefresher + + +def test_easy_proxies_auto_refresher_enqueues_at_interval_and_skips_duplicates(tmp_path: Path) -> None: + db_path = tmp_path / "auto_refresh.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + now_box = {"now": 0.0} + + def now() -> float: + return float(now_box["now"]) + + engine = create_engine(db_url) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + refresher = EasyProxiesAutoRefresher( + EasyProxiesAutoRefreshConfig(base_url="http://easy-proxies:9090", interval_s=10.0), + now=now, + ) + + await refresher.tick(engine) + + Session = create_sessionmaker(engine) + async with Session() as session: + jobs = (await session.execute(select(JobRow))).scalars().all() + assert len(jobs) == 1 + payload = json.loads(jobs[0].payload_json) + assert payload["base_url"] == "http://easy-proxies:9090" + assert payload["conflict_policy"] == "skip_non_easy_proxies" + + now_box["now"] = 11.0 + await refresher.tick(engine) + async with Session() as session: + jobs = (await session.execute(select(JobRow))).scalars().all() + assert len(jobs) == 1 + + jobs[0].status = "completed" + await session.commit() + + now_box["now"] = 21.0 + await refresher.tick(engine) + async with Session() as session: + jobs = (await session.execute(select(JobRow).order_by(JobRow.id.asc()))).scalars().all() + assert len(jobs) == 2 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_easy_proxies_client.py b/backend/tests/test_easy_proxies_client.py new file mode 100644 index 0000000..0053ec7 --- /dev/null +++ b/backend/tests/test_easy_proxies_client.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import asyncio + +import httpx +import pytest + +from app.easy_proxies.client import EasyProxiesError, easy_proxies_auth, easy_proxies_export + + +def test_easy_proxies_auth_success_returns_token() -> None: + def handler(req: httpx.Request) -> httpx.Response: + assert req.method == "POST" + assert str(req.url) == "http://easy-proxies:9090/api/auth" + assert req.headers.get("content-type", "").startswith("application/json") + assert req.content == b'{\"password\":\"pw\"}' + return httpx.Response(200, json={"token": "tok"}) + + result = asyncio.run( + easy_proxies_auth( + base_url="http://easy-proxies:9090", + password="pw", + transport=httpx.MockTransport(handler), + ) + ) + assert result.token == "tok" + + +def test_easy_proxies_export_sends_bearer_token_header() -> None: + def handler(req: httpx.Request) -> httpx.Response: + assert req.method == "GET" + assert str(req.url) == "http://easy-proxies:9090/api/export" + assert req.headers.get("Authorization") == "Bearer tok" + return httpx.Response(200, headers={"Content-Type": "text/plain"}, content=b"http://1.2.3.4:1234\n\nsocks5://5.6.7.8:1080\n") + + out = asyncio.run( + easy_proxies_export( + base_url="http://easy-proxies:9090", + bearer_token="tok", + transport=httpx.MockTransport(handler), + ) + ) + assert out == ["http://1.2.3.4:1234", "socks5://5.6.7.8:1080"] + + +def test_easy_proxies_export_error_does_not_echo_token() -> None: + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response(401, json={"error": "unauthorized"}) + + with pytest.raises(EasyProxiesError) as excinfo: + asyncio.run( + easy_proxies_export( + base_url="http://easy-proxies:9090", + bearer_token="tok_secret", + transport=httpx.MockTransport(handler), + ) + ) + assert excinfo.value.status_code == 401 + assert "tok_secret" not in str(excinfo.value) + diff --git a/backend/tests/test_effective_settings.py b/backend/tests/test_effective_settings.py new file mode 100644 index 0000000..bc6c074 --- /dev/null +++ b/backend/tests/test_effective_settings.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.core.effective_settings import load_effective_settings +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.runtime_settings import RuntimeSetting +from app.db.session import create_sessionmaker + + +def test_effective_settings_merge_and_reload(tmp_path: Path) -> None: + db_path = tmp_path / "effective_settings.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + engine = create_engine(db_url) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + + base = await load_effective_settings(engine, env={"APP_ENV": "dev", "DATABASE_URL": db_url}) + assert base.runtime.proxy_enabled is False + + async with Session() as session: + session.add(RuntimeSetting(key="proxy.enabled", value_json="true", updated_by="tester")) + await session.commit() + + updated = await load_effective_settings(engine, env={"APP_ENV": "dev", "DATABASE_URL": db_url}) + assert updated.runtime.proxy_enabled is True + + async with Session() as session: + row = (await session.execute(select(RuntimeSetting).where(RuntimeSetting.key == "proxy.enabled"))).scalar_one() + row.value_json = "false" + await session.commit() + + updated2 = await load_effective_settings(engine, env={"APP_ENV": "dev", "DATABASE_URL": db_url}) + assert updated2.runtime.proxy_enabled is False + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_error_codes.py b/backend/tests/test_error_codes.py new file mode 100644 index 0000000..b9c108a --- /dev/null +++ b/backend/tests/test_error_codes.py @@ -0,0 +1,386 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import httpx +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +from app.core.errors import ApiError, ErrorCode +from app.core.http_stream import stream_url +from app.core.security import create_jwt +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.jobs.errors import JobDeferError +from app.jobs.handlers.hydrate_metadata import build_hydrate_metadata_handler +from app.main import create_app + + +def _references_error_code(name: str) -> bool: + backend_dir = Path(__file__).resolve().parents[1] + app_dir = backend_dir / "app" + errors_py = app_dir / "core" / "errors.py" + needle = f"ErrorCode.{name}" + for path in app_dir.rglob("*.py"): + if path == errors_py: + continue + if needle in path.read_text(encoding="utf-8", errors="ignore"): + return True + return False + + +def test_error_codes_random_json_success_is_http_200(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "error_codes_random_ok.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setattr("app.api.public.random.random.random", lambda: 0.0) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=12345678, + page_index=0, + ext="jpg", + original_url="https://example.test/ok.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + x_restrict=0, + ) + ) + await session.commit() + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp = client.get("/random", params={"format": "json", "attempts": 1, "r18_strict": 0}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["code"] == "OK" + + +def test_error_codes_random_json_no_match_is_http_404(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "error_codes_random_no_match.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setattr("app.api.public.random.random.random", lambda: 0.0) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=12345678, + page_index=0, + ext="jpg", + original_url="https://example.test/no_match.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + width=100, + height=100, + x_restrict=0, + ) + ) + await session.commit() + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp = client.get( + "/random", + params={ + "format": "json", + "attempts": 1, + "r18_strict": 0, + "min_width": 999999, + }, + ) + assert resp.status_code == 404 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "NO_MATCH" + + +def test_error_codes_bad_request_defined_and_used() -> None: + assert ErrorCode.BAD_REQUEST.value == "BAD_REQUEST" + assert _references_error_code("BAD_REQUEST") is True + + +def test_error_codes_unauthorized_defined_and_used() -> None: + assert ErrorCode.UNAUTHORIZED.value == "UNAUTHORIZED" + assert _references_error_code("UNAUTHORIZED") is True + + +def test_error_codes_forbidden_defined_and_used() -> None: + assert ErrorCode.FORBIDDEN.value == "FORBIDDEN" + assert _references_error_code("FORBIDDEN") is True + + +def test_error_codes_not_found_defined_and_used() -> None: + assert ErrorCode.NOT_FOUND.value == "NOT_FOUND" + assert _references_error_code("NOT_FOUND") is True + + +def test_error_codes_internal_error_defined_and_used() -> None: + assert ErrorCode.INTERNAL_ERROR.value == "INTERNAL_ERROR" + assert _references_error_code("INTERNAL_ERROR") is True + + +def test_error_codes_no_match_defined_and_used() -> None: + assert ErrorCode.NO_MATCH.value == "NO_MATCH" + assert _references_error_code("NO_MATCH") is True + + +def test_error_codes_upstream_stream_error_defined_and_used() -> None: + assert ErrorCode.UPSTREAM_STREAM_ERROR.value == "UPSTREAM_STREAM_ERROR" + assert _references_error_code("UPSTREAM_STREAM_ERROR") is True + + +def test_error_codes_upstream_403_defined_and_used() -> None: + assert ErrorCode.UPSTREAM_403.value == "UPSTREAM_403" + assert _references_error_code("UPSTREAM_403") is True + + +def test_error_codes_upstream_404_defined_and_used() -> None: + assert ErrorCode.UPSTREAM_404.value == "UPSTREAM_404" + assert _references_error_code("UPSTREAM_404") is True + + +def test_error_codes_upstream_rate_limit_defined_and_used() -> None: + assert ErrorCode.UPSTREAM_RATE_LIMIT.value == "UPSTREAM_RATE_LIMIT" + assert _references_error_code("UPSTREAM_RATE_LIMIT") is True + + +def test_error_codes_upstream_rate_limit_raised_on_429() -> None: + transport = httpx.MockTransport(lambda req: httpx.Response(429, request=req)) + + async def _run() -> None: + try: + await stream_url( + "https://example.test/429", + transport=transport, + cache_control="no-store", + ) + except ApiError as exc: + assert exc.code == ErrorCode.UPSTREAM_RATE_LIMIT + assert exc.status_code == 502 + else: + raise AssertionError("expected ApiError") + + asyncio.run(_run()) + + +def test_error_codes_invalid_upload_type_defined_and_used() -> None: + assert ErrorCode.INVALID_UPLOAD_TYPE.value == "INVALID_UPLOAD_TYPE" + assert _references_error_code("INVALID_UPLOAD_TYPE") is True + + +def test_error_codes_payload_too_large_defined_and_used() -> None: + assert ErrorCode.PAYLOAD_TOO_LARGE.value == "PAYLOAD_TOO_LARGE" + assert _references_error_code("PAYLOAD_TOO_LARGE") is True + + +def test_error_codes_payload_too_large_raised_on_import_multipart(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "error_codes_payload_too_large.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("IMPORT_MAX_BYTES", "1024") + + app = create_app() + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/imports", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + data={"dry_run": "true", "hydrate_on_import": "false", "source": "manual"}, + files={"file": ("urls.txt", b"x" * 1025, "text/plain")}, + ) + + assert resp.status_code == 413 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "PAYLOAD_TOO_LARGE" + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + +def test_error_codes_unsupported_url_defined_and_used() -> None: + assert ErrorCode.UNSUPPORTED_URL.value == "UNSUPPORTED_URL" + assert _references_error_code("UNSUPPORTED_URL") is True + + +def test_error_codes_unsupported_url_returned_in_import_errors(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "error_codes_unsupported_url.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/imports", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"text": "not-a-url", "dry_run": True, "hydrate_on_import": False, "source": "manual"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["accepted"] == 0 + assert body["errors"][0]["code"] == "UNSUPPORTED_URL" + + +def test_error_codes_token_refresh_failed_defined_and_used() -> None: + assert ErrorCode.TOKEN_REFRESH_FAILED.value == "TOKEN_REFRESH_FAILED" + assert _references_error_code("TOKEN_REFRESH_FAILED") is True + + +def test_error_codes_token_backoff_defined_and_used() -> None: + assert ErrorCode.TOKEN_BACKOFF.value == "TOKEN_BACKOFF" + assert _references_error_code("TOKEN_BACKOFF") is True + + +def test_error_codes_no_token_available_defined_and_used() -> None: + assert ErrorCode.NO_TOKEN_AVAILABLE.value == "NO_TOKEN_AVAILABLE" + assert _references_error_code("NO_TOKEN_AVAILABLE") is True + + +def test_error_codes_no_token_available_defers_hydrate_job(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "error_codes_no_token_available.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + field_key = Fernet.generate_key().decode("ascii") + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + monkeypatch.setenv("PIXIV_OAUTH_HASH_SECRET", "hsec_test") + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + handler = build_hydrate_metadata_handler(engine) + try: + await handler({"type": "hydrate_metadata", "payload_json": '{"illust_id": 123}', "ref_id": ""}) + except JobDeferError as exc: + assert ErrorCode.NO_TOKEN_AVAILABLE.value in str(exc) + else: + raise AssertionError("expected JobDeferError") + await engine.dispose() + + asyncio.run(_run()) + + +def test_error_codes_proxy_required_defined_and_used() -> None: + assert ErrorCode.PROXY_REQUIRED.value == "PROXY_REQUIRED" + assert _references_error_code("PROXY_REQUIRED") is True + + +def test_error_codes_proxy_required_raised_when_fail_closed_enabled_without_proxies( + tmp_path: Path, monkeypatch +) -> None: + db_path = tmp_path / "error_codes_proxy_required.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.put( + "/admin/api/settings", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + json={"settings": {"proxy": {"enabled": True, "fail_closed": True}}}, + ) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "PROXY_REQUIRED" + assert body["request_id"] == "req_test" + + +def test_error_codes_proxy_auth_failed_defined_and_used() -> None: + assert ErrorCode.PROXY_AUTH_FAILED.value == "PROXY_AUTH_FAILED" + assert _references_error_code("PROXY_AUTH_FAILED") is True + + +def test_error_codes_proxy_auth_failed_raised_on_proxy_407(monkeypatch) -> None: + async def fake_send(self, request: httpx.Request, **kwargs): # type: ignore[no-untyped-def] + raise httpx.ProxyError("407 Proxy Authentication Required") + + monkeypatch.setattr(httpx.AsyncClient, "send", fake_send, raising=True) + + async def _run() -> None: + try: + await stream_url("https://example.test/proxy", cache_control="no-store") + except ApiError as exc: + assert exc.code == ErrorCode.PROXY_AUTH_FAILED + assert exc.status_code == 502 + else: + raise AssertionError("expected ApiError") + + asyncio.run(_run()) + + +def test_error_codes_proxy_connect_failed_defined_and_used() -> None: + assert ErrorCode.PROXY_CONNECT_FAILED.value == "PROXY_CONNECT_FAILED" + assert _references_error_code("PROXY_CONNECT_FAILED") is True + + +def test_error_codes_proxy_connect_failed_raised_on_proxy_error(monkeypatch) -> None: + async def fake_send(self, request: httpx.Request, **kwargs): # type: ignore[no-untyped-def] + raise httpx.ProxyError("Proxy connect failed") + + monkeypatch.setattr(httpx.AsyncClient, "send", fake_send, raising=True) + + async def _run() -> None: + try: + await stream_url("https://example.test/proxy", cache_control="no-store") + except ApiError as exc: + assert exc.code == ErrorCode.PROXY_CONNECT_FAILED + assert exc.status_code == 502 + else: + raise AssertionError("expected ApiError") + + asyncio.run(_run()) diff --git a/backend/tests/test_error_schema.py b/backend/tests/test_error_schema.py new file mode 100644 index 0000000..ab1e5b9 --- /dev/null +++ b/backend/tests/test_error_schema.py @@ -0,0 +1,15 @@ +from app.core.errors import ErrorCode, UNKNOWN_REQUEST_ID, error_body + + +def test_error_body_shape() -> None: + body = error_body(code=ErrorCode.BAD_REQUEST, message="nope", request_id="req_123", details={"a": 1}) + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["message"] == "请求参数错误" + assert body["request_id"] == "req_123" + assert body["details"] == {"a": 1} + + +def test_error_body_request_id_default() -> None: + body = error_body(code=ErrorCode.INTERNAL_ERROR, message="boom", request_id=None, details=None) + assert body["request_id"] == UNKNOWN_REQUEST_ID diff --git a/backend/tests/test_failover_policy.py b/backend/tests/test_failover_policy.py new file mode 100644 index 0000000..30e0485 --- /dev/null +++ b/backend/tests/test_failover_policy.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import httpx + +from app.core.failover import ( + OutboundErrorKind, + classify_httpx_exception, + classify_pixiv_rate_limit, + pixiv_rate_limit_backoff_seconds, + proxy_override_ttl_seconds, + should_backoff_token, + should_override_proxy, +) + + +def test_classify_httpx_proxy_error_proxy_auth() -> None: + exc = httpx.ProxyError("407 Proxy Authentication Required") + assert classify_httpx_exception(exc) == OutboundErrorKind.PROXY_AUTH + + +def test_classify_httpx_proxy_error_proxy_connect() -> None: + exc = httpx.ProxyError("Proxy connect failed") + assert classify_httpx_exception(exc) == OutboundErrorKind.PROXY_CONNECT + + +def test_classify_pixiv_rate_limit_403_body_match() -> None: + kind = classify_pixiv_rate_limit(status_code=403, body_text="Rate Limit") + assert kind == OutboundErrorKind.PIXIV_RATE_LIMIT + + +def test_failover_decisions() -> None: + assert should_override_proxy(OutboundErrorKind.PROXY_CONNECT) is True + assert should_override_proxy(OutboundErrorKind.PROXY_AUTH) is True + assert should_override_proxy(OutboundErrorKind.PIXIV_RATE_LIMIT) is False + + assert should_backoff_token(OutboundErrorKind.PIXIV_RATE_LIMIT) is True + assert should_backoff_token(OutboundErrorKind.PROXY_CONNECT) is False + + +def test_proxy_override_ttl_schedule() -> None: + assert proxy_override_ttl_seconds(attempt=0) == 0 + assert proxy_override_ttl_seconds(attempt=1) == 20 * 60 + assert proxy_override_ttl_seconds(attempt=2) == 60 * 60 + + +def test_pixiv_rate_limit_backoff_schedule() -> None: + assert pixiv_rate_limit_backoff_seconds(attempt=0) == 0 + assert pixiv_rate_limit_backoff_seconds(attempt=1) == 60 + assert pixiv_rate_limit_backoff_seconds(attempt=2) == 5 * 60 + diff --git a/backend/tests/test_healthz.py b/backend/tests/test_healthz.py new file mode 100644 index 0000000..7a73b96 --- /dev/null +++ b/backend/tests/test_healthz.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.api.public.healthz import router as healthz_router +from app.db.engine import create_engine +from app.db.models.base import Base +from app.core.time import iso_utc_ms + + +def test_healthz_ok_includes_request_id() -> None: + app = FastAPI() + app.state.engine = create_engine("sqlite+aiosqlite:///:memory:") + app.include_router(healthz_router) + + client = TestClient(app) + resp = client.get("/healthz") + + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["db_ok"] is True + assert body["request_id"].startswith("req_") + assert resp.headers["X-Request-Id"] == body["request_id"] + + +def test_healthz_uses_request_id_header_if_provided() -> None: + app = FastAPI() + app.state.engine = create_engine("sqlite+aiosqlite:///:memory:") + app.include_router(healthz_router) + + client = TestClient(app) + resp = client.get("/healthz", headers={"X-Request-Id": "req_test"}) + + assert resp.status_code == 200 + body = resp.json() + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + + +def test_healthz_reports_db_down_if_no_engine() -> None: + app = FastAPI() + app.include_router(healthz_router) + + client = TestClient(app) + resp = client.get("/healthz", headers={"X-Request-Id": "req_test"}) + + assert resp.status_code == 503 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "INTERNAL_ERROR" + assert body["request_id"] == "req_test" + + +def test_healthz_reports_worker_and_queue_status_when_available(tmp_path: Path) -> None: + app = FastAPI() + db_path = tmp_path / "healthz_deps.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + engine = create_engine(db_url) + app.state.engine = engine + app.include_router(healthz_router) + + async def _migrate_and_seed() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + await conn.exec_driver_sql( + "INSERT INTO runtime_settings (key, value_json, description, updated_by) VALUES (?,?,?,?)", + ( + "worker.last_seen_at", + json.dumps({"at": iso_utc_ms(), "worker_id": "test", "pid": 1}), + "worker heartbeat", + "test", + ), + ) + await conn.exec_driver_sql( + "INSERT INTO jobs (type, status, payload_json) VALUES ('import_urls','pending','{}')" + ) + + asyncio.run(_migrate_and_seed()) + + client = TestClient(app) + resp = client.get("/healthz") + + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["db_ok"] is True + assert body["worker_ok"] is True + assert body["queue_ok"] is True + assert body["worker"]["last_seen_at"] + assert body["queue"]["counts"]["pending"] == 1 diff --git a/backend/tests/test_http_stream.py b/backend/tests/test_http_stream.py new file mode 100644 index 0000000..281eed1 --- /dev/null +++ b/backend/tests/test_http_stream.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import asyncio + +import httpx + +from app.core.http_stream import PIXIV_REFERER, stream_url + + +class _DummyStream(httpx.AsyncByteStream): + def __init__(self, chunks: list[bytes]) -> None: + self._chunks = chunks + self.closed = False + + async def __aiter__(self): + for chunk in self._chunks: + yield chunk + + async def aclose(self) -> None: + self.closed = True + + +class _BlockingStream(httpx.AsyncByteStream): + def __init__(self) -> None: + self.closed = False + self.started = asyncio.Event() + self.unblock = asyncio.Event() + + async def __aiter__(self): + yield b"first" + self.started.set() + await self.unblock.wait() + yield b"second" + + async def aclose(self) -> None: + self.closed = True + self.unblock.set() + + +def test_stream_url_uses_streaming(monkeypatch) -> None: + sent_stream_flag: bool | None = None + dummy_stream = _DummyStream([b"abc", b"def"]) + + async def fake_send(self, request: httpx.Request, **kwargs): # type: ignore[no-untyped-def] + nonlocal sent_stream_flag + sent_stream_flag = bool(kwargs.get("stream")) + return httpx.Response( + 200, + headers={"Content-Type": "application/octet-stream"}, + stream=dummy_stream, + request=request, + ) + + monkeypatch.setattr(httpx.AsyncClient, "send", fake_send, raising=True) + + async def _run() -> bytes: + resp = await stream_url("https://example.test/big.bin", cache_control="no-store") + chunks: list[bytes] = [] + async for chunk in resp.body_iterator: + chunks.append(chunk) + if resp.background is not None: + await resp.background() + return b"".join(chunks) + + body = asyncio.run(_run()) + assert sent_stream_flag is True + assert body == b"abcdef" + assert dummy_stream.closed is True + + +def test_stream_url_sets_pixiv_referer_header_by_default(monkeypatch) -> None: + seen_referer: str | None = None + dummy_stream = _DummyStream([b"x"]) + + async def fake_send(self, request: httpx.Request, **kwargs): # type: ignore[no-untyped-def] + nonlocal seen_referer + seen_referer = request.headers.get("Referer") + return httpx.Response( + 200, + headers={"Content-Type": "application/octet-stream"}, + stream=dummy_stream, + request=request, + ) + + monkeypatch.setattr(httpx.AsyncClient, "send", fake_send, raising=True) + + async def _run() -> None: + resp = await stream_url("https://example.test/x.bin", cache_control="no-store") + async for _ in resp.body_iterator: + pass + if resp.background is not None: + await resp.background() + + asyncio.run(_run()) + assert seen_referer == PIXIV_REFERER + + +def test_stream_url_closes_on_consumer_cancel(monkeypatch) -> None: + client_closed = False + orig_aclose = httpx.AsyncClient.aclose + blocking_stream: _BlockingStream | None = None + + async def fake_send(self, request: httpx.Request, **kwargs): # type: ignore[no-untyped-def] + assert blocking_stream is not None + return httpx.Response( + 200, + headers={"Content-Type": "application/octet-stream"}, + stream=blocking_stream, + request=request, + ) + + async def fake_aclose(self) -> None: # type: ignore[no-untyped-def] + nonlocal client_closed + client_closed = True + await orig_aclose(self) + + monkeypatch.setattr(httpx.AsyncClient, "send", fake_send, raising=True) + monkeypatch.setattr(httpx.AsyncClient, "aclose", fake_aclose, raising=True) + + async def _run() -> None: + nonlocal blocking_stream + blocking_stream = _BlockingStream() + resp = await stream_url("https://example.test/slow.bin", cache_control="no-store") + first_received = asyncio.Event() + + async def _consume() -> None: + async for _ in resp.body_iterator: + first_received.set() + await asyncio.sleep(3600) + + task = asyncio.create_task(_consume()) + await first_received.wait() + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + await asyncio.sleep(0) + + asyncio.run(_run()) + assert blocking_stream is not None + assert blocking_stream.closed is True + assert client_closed is True diff --git a/backend/tests/test_hydrate_persists_popularity_fields.py b/backend/tests/test_hydrate_persists_popularity_fields.py new file mode 100644 index 0000000..ee46e5e --- /dev/null +++ b/backend/tests/test_hydrate_persists_popularity_fields.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import httpx +import sqlalchemy as sa +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +from app.core.crypto import FieldEncryptor +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.models.jobs import JobRow +from app.db.models.pixiv_tokens import PixivToken +from app.db.session import create_sessionmaker +from app.jobs.claim import claim_next_job +from app.jobs.dispatch import JobDispatcher +from app.jobs.executor import execute_claimed_job +from app.jobs.handlers.hydrate_metadata import build_hydrate_metadata_handler +from app.main import create_app + + +def test_hydrate_persists_popularity_fields_and_public_api_outputs(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "hydrate_popularity.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + monkeypatch.setenv("PIXIV_OAUTH_HASH_SECRET", "hsec_test") + + refresh_token = "rt_test" + access_token = "at_test" + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == "https://oauth.secure.pixiv.net/auth/token": + body = (req.content or b"").decode("utf-8") + assert f"refresh_token={refresh_token}" in body + return httpx.Response( + 200, + json={ + "response": { + "access_token": access_token, + "token_type": "bearer", + "expires_in": 3600, + "refresh_token": refresh_token, + "scope": "", + "user": {"id": 123}, + } + }, + ) + + if str(req.url).startswith("https://app-api.pixiv.net/v1/illust/detail"): + assert req.headers.get("Authorization") == f"Bearer {access_token}" + return httpx.Response( + 200, + json={ + "illust": { + "id": 111, + "title": "title_test", + "user": {"id": 999, "name": "user_test"}, + "x_restrict": 0, + "illust_type": 0, + "illust_ai_type": 1, + "width": 1200, + "height": 800, + "create_date": "2020-01-01T00:00:00+00:00", + "page_count": 2, + "total_bookmarks": 123, + "total_view": 456, + "total_comments": 7, + "tags": [{"name": "tag1"}], + "meta_pages": [ + { + "image_urls": { + "original": "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p0.jpg" + } + }, + { + "image_urls": { + "original": "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p1.png" + } + }, + ], + } + }, + ) + + return httpx.Response(500, text="unexpected") + + transport = httpx.MockTransport(handler) + + app = create_app() + image0_id: int | None = None + + async def _run() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + + async with Session() as session: + token_row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text(refresh_token), + refresh_token_masked="***", + weight=1.0, + ) + session.add(token_row) + + job = JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps({"illust_id": 111}, ensure_ascii=False, separators=(",", ":")), + ) + session.add(job) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register("hydrate_metadata", build_hydrate_metadata_handler(app.state.engine, transport=transport)) + + claimed = await claim_next_job(app.state.engine, worker_id="w1") + assert claimed is not None + + transition = await execute_claimed_job(app.state.engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + images = ( + ( + await session.execute( + sa.select(Image).where(Image.illust_id == 111).order_by(Image.page_index.asc()) + ) + ) + .scalars() + .all() + ) + assert len(images) == 2 + assert images[0].page_index == 0 + assert images[1].page_index == 1 + for img in images: + assert img.bookmark_count == 123 + assert img.view_count == 456 + assert img.comment_count == 7 + assert img.illust_type == 0 + + nonlocal image0_id + image0_id = int(images[0].id) + + await app.state.engine.dispose() + + asyncio.run(_run()) + assert image0_id is not None + + with TestClient(app) as client: + resp = client.get("/images", params={"r18": 2}, headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + found = [ + it + for it in body["items"] + if int(it["illust_id"]) == 111 and int(it["page_index"]) == 0 + ] + assert found + item0 = found[0] + assert item0["bookmark_count"] == 123 + assert item0["view_count"] == 456 + assert item0["comment_count"] == 7 + + resp2 = client.get(f"/images/{image0_id}", headers={"X-Request-Id": "req_test2"}) + assert resp2.status_code == 200 + body2 = resp2.json() + assert body2["ok"] is True + assert body2["request_id"] == "req_test2" + item = body2["item"]["image"] + assert item["bookmark_count"] == 123 + assert item["view_count"] == 456 + assert item["comment_count"] == 7 diff --git a/backend/tests/test_image_proxy.py b/backend/tests/test_image_proxy.py new file mode 100644 index 0000000..6f1c69a --- /dev/null +++ b/backend/tests/test_image_proxy.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import httpx +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_image_proxy_streams_bytes(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "image_proxy.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + image_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img = Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + ) + session.add(img) + await session.commit() + await session.refresh(img) + nonlocal image_id + image_id = img.id + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert image_id is not None + + def handler(req: httpx.Request) -> httpx.Response: + assert req.headers.get("Referer") == "https://www.pixiv.net/" + return httpx.Response(200, headers={"Content-Type": "image/jpeg"}, content=b"img-bytes") + + app.state.httpx_transport = httpx.MockTransport(handler) + + with TestClient(app) as client: + resp = client.get(f"/i/{image_id}.jpg", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + assert resp.content == b"img-bytes" + assert resp.headers["Cache-Control"] == "public, max-age=31536000, immutable" + assert resp.headers["X-Request-Id"] == "req_test" + + +def test_image_proxy_range_passthrough_returns_206(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "image_proxy_range.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + image_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img = Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + ) + session.add(img) + await session.commit() + await session.refresh(img) + nonlocal image_id + image_id = img.id + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert image_id is not None + + def handler(req: httpx.Request) -> httpx.Response: + assert req.headers.get("Referer") == "https://www.pixiv.net/" + assert req.headers.get("Range") == "bytes=0-2" + return httpx.Response( + 206, + headers={ + "Content-Type": "image/jpeg", + "Content-Length": "3", + "Accept-Ranges": "bytes", + "Content-Range": "bytes 0-2/6", + }, + content=b"abc", + ) + + app.state.httpx_transport = httpx.MockTransport(handler) + + with TestClient(app) as client: + resp = client.get( + f"/i/{image_id}.jpg", + headers={"X-Request-Id": "req_test", "Range": "bytes=0-2"}, + ) + assert resp.status_code == 206 + assert resp.content == b"abc" + assert resp.headers["Cache-Control"] == "public, max-age=31536000, immutable" + assert resp.headers["Accept-Ranges"] == "bytes" + assert resp.headers["Content-Range"] == "bytes 0-2/6" + assert resp.headers["X-Request-Id"] == "req_test" + + +def test_image_proxy_ext_mismatch_returns_404(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "image_proxy_mismatch.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + image_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img = Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + ) + session.add(img) + await session.commit() + await session.refresh(img) + nonlocal image_id + image_id = img.id + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert image_id is not None + + with TestClient(app) as client: + resp = client.get(f"/i/{image_id}.png", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 404 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "NOT_FOUND" + assert body["request_id"] == "req_test" + + +def test_image_proxy_invalid_ext_returns_400(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "image_proxy_invalid_ext.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp = client.get("/i/1.exe", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" + + +def test_image_proxy_upstream_404_marks_failure(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "image_proxy_upstream_404.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + image_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img = Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + ) + session.add(img) + await session.commit() + await session.refresh(img) + nonlocal image_id + image_id = img.id + + asyncio.run(_seed()) + assert image_id is not None + + def handler(req: httpx.Request) -> httpx.Response: + assert req.headers.get("Referer") == "https://www.pixiv.net/" + return httpx.Response(404, headers={"Content-Type": "text/plain"}, content=b"not found") + + app.state.httpx_transport = httpx.MockTransport(handler) + + with TestClient(app) as client: + resp = client.get(f"/i/{image_id}.jpg", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 502 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "UPSTREAM_404" + assert body["request_id"] == "req_test" + + async def _get_row() -> Image: + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + row = await session.get(Image, int(image_id)) + assert row is not None + return row + + row = asyncio.run(_get_row()) + assert int(row.fail_count) == 1 + assert row.last_fail_at is not None + assert row.last_error_code == "UPSTREAM_404" diff --git a/backend/tests/test_images_get.py b/backend/tests/test_images_get.py new file mode 100644 index 0000000..20ff926 --- /dev/null +++ b/backend/tests/test_images_get.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_images_get_returns_item_and_tags(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "images_get.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + image_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img = Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + x_restrict=0, + ) + t1 = Tag(name="b") + t2 = Tag(name="a") + session.add_all([img, t1, t2]) + await session.commit() + await session.refresh(img) + await session.refresh(t1) + await session.refresh(t2) + session.add_all([ImageTag(image_id=int(img.id), tag_id=int(t1.id)), ImageTag(image_id=int(img.id), tag_id=int(t2.id))]) + await session.commit() + nonlocal image_id + image_id = int(img.id) + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert image_id is not None + + with TestClient(app) as client: + resp = client.get(f"/images/{image_id}", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["request_id"] == "req_test" + assert body["item"]["image"]["id"] == str(image_id) + assert body["item"]["tags"] == ["a", "b"] + + +def test_images_get_missing_returns_404(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "images_get_missing.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + await app.state.engine.dispose() + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.get("/images/999", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 404 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "NOT_FOUND" + assert body["request_id"] == "req_test" + + +def test_images_get_invalid_id_returns_400(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "images_get_invalid.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + with TestClient(app) as client: + resp = client.get("/images/0", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" + diff --git a/backend/tests/test_images_list.py b/backend/tests/test_images_list.py new file mode 100644 index 0000000..44e9558 --- /dev/null +++ b/backend/tests/test_images_list.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_images_list_pagination_and_filters(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "images_list.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + ids: dict[str, int] = {} + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img1 = Image( + illust_id=100, + page_index=0, + ext="jpg", + original_url="https://example.test/100.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + ai_type=0, + user_id=1, + user_name="u1", + ) + img2 = Image( + illust_id=101, + page_index=0, + ext="jpg", + original_url="https://example.test/101.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=1, + ai_type=1, + user_id=2, + user_name="u2", + ) + img3 = Image( + illust_id=102, + page_index=0, + ext="jpg", + original_url="https://example.test/102.jpg", + proxy_path="/i/3.jpg", + random_key=0.3, + x_restrict=0, + ai_type=1, + user_id=3, + user_name="u3", + ) + session.add_all([img1, img2, img3]) + + t_cat = Tag(name="cat") + t_dog = Tag(name="dog") + session.add_all([t_cat, t_dog]) + + await session.commit() + await session.refresh(img1) + await session.refresh(img2) + await session.refresh(img3) + await session.refresh(t_cat) + + session.add(ImageTag(image_id=int(img3.id), tag_id=int(t_cat.id))) + await session.commit() + + ids["img1"] = int(img1.id) + ids["img2"] = int(img2.id) + ids["img3"] = int(img3.id) + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + page1 = client.get("/images", params={"limit": 2, "r18": 2}, headers={"X-Request-Id": "req_test"}) + assert page1.status_code == 200 + body1 = page1.json() + assert body1["ok"] is True + assert body1["request_id"] == "req_test" + assert len(body1["items"]) == 2 + assert body1["next_cursor"].isdigit() + + # Ordered by id DESC + assert int(body1["items"][0]["id"]) == ids["img3"] + assert int(body1["items"][1]["id"]) == ids["img2"] + + page2 = client.get( + "/images", + params={"limit": 2, "cursor": body1["next_cursor"], "r18": 2}, + ) + assert page2.status_code == 200 + body2 = page2.json() + assert len(body2["items"]) == 1 + assert int(body2["items"][0]["id"]) == ids["img1"] + assert body2["next_cursor"] == "" + + only_ai0 = client.get("/images", params={"r18": 2, "ai_type": 0}) + assert only_ai0.status_code == 200 + items_ai0 = only_ai0.json()["items"] + assert len(items_ai0) == 1 + assert items_ai0[0]["ai_type"] == 0 + + only_cat = client.get("/images", params={"r18": 2, "included_tags": "cat"}) + assert only_cat.status_code == 200 + items_cat = only_cat.json()["items"] + assert len(items_cat) == 1 + assert int(items_cat[0]["id"]) == ids["img3"] + + +def test_images_list_invalid_limit_returns_400(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "images_list_invalid_limit.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + await app.state.engine.dispose() + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.get("/images", params={"limit": 0}, headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" + diff --git a/backend/tests/test_images_upsert.py b/backend/tests/test_images_upsert.py new file mode 100644 index 0000000..a31634e --- /dev/null +++ b/backend/tests/test_images_upsert.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.images_upsert import upsert_image_by_illust_page +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker + + +def test_images_upsert_by_illust_page(tmp_path: Path) -> None: + db_path = tmp_path / "images_upsert.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + + async with Session() as session: + image_id1 = await upsert_image_by_illust_page( + session, + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/old.jpg", + proxy_path="/i/123_p0.jpg", + random_key=0.1, + created_import_id=None, + ) + await session.commit() + + async with Session() as session: + image_id2 = await upsert_image_by_illust_page( + session, + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/new.jpg", + proxy_path="/i/123_p0.jpg", + random_key=0.2, + created_import_id=None, + ) + await session.commit() + + assert image_id1 == image_id2 + + async with Session() as session: + imgs = (await session.execute(select(Image))).scalars().all() + assert len(imgs) == 1 + assert imgs[0].original_url == "https://example.test/new.jpg" + assert imgs[0].random_key == 0.1 + + await engine.dispose() + + asyncio.run(_run()) diff --git a/backend/tests/test_imgproxy_signing.py b/backend/tests/test_imgproxy_signing.py new file mode 100644 index 0000000..8374be7 --- /dev/null +++ b/backend/tests/test_imgproxy_signing.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import asyncio +import base64 +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.imgproxy import ImgproxyConfig, sign_path +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_imgproxy_signature_matches_docs_example() -> None: + cfg = ImgproxyConfig( + base_url="http://imgproxy.example.com", + key=bytes.fromhex("736563726574"), + salt=bytes.fromhex("68656C6C6F"), + max_dim=2048, + default_options="rs:fit:2048:2048", + url_chunk_size=16, + ) + + path = "/rs:fill:300:400:0/g:sm/aHR0cDovL2V4YW1w/bGUuY29tL2ltYWdl/cy9jdXJpb3NpdHku/anBn.png" + assert sign_path(cfg, path) == "oKfUtW34Dvo2BGQehJFR4Nr0_rIjOtdtzJ3QFsUcXH8" + + +def test_random_json_imgproxy_uses_proxy_source_when_origin_hidden(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "imgproxy_random.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("IMGPROXY_BASE_URL", "http://imgproxy.example.com") + monkeypatch.setenv("IMGPROXY_KEY", "736563726574") + monkeypatch.setenv("IMGPROXY_SALT", "68656C6C6F") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + x_restrict=0, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + origin_b64 = base64.urlsafe_b64encode(b"https://example.test/origin.jpg").rstrip(b"=").decode("ascii") + origin_b64_chunked = "/".join(origin_b64[i : i + 16] for i in range(0, len(origin_b64), 16)) + + with TestClient(app) as client: + resp = client.get("/random?format=json", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + body = resp.json() + assert body["data"]["urls"]["origin"] is None + imgproxy_url = body["data"]["urls"]["imgproxy"] + assert isinstance(imgproxy_url, str) + assert imgproxy_url.startswith("http://imgproxy.example.com/") + assert "/rs:fit:" in imgproxy_url + assert origin_b64 not in imgproxy_url + assert origin_b64_chunked not in imgproxy_url + + image_id = body["data"]["image"]["id"] + proxy_source = f"http://testserver/i/{image_id}.jpg".encode("utf-8") + proxy_b64 = base64.urlsafe_b64encode(proxy_source).rstrip(b"=").decode("ascii") + proxy_b64_chunked = "/".join(proxy_b64[i : i + 16] for i in range(0, len(proxy_b64), 16)) + assert proxy_b64_chunked in imgproxy_url diff --git a/backend/tests/test_job_dispatch.py b/backend/tests/test_job_dispatch.py new file mode 100644 index 0000000..e711550 --- /dev/null +++ b/backend/tests/test_job_dispatch.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from app.jobs.dispatch import JobDispatcher + + +def test_job_dispatch_calls_registered_handler() -> None: + dispatcher = JobDispatcher() + called: list[int] = [] + + async def _handler(job): + called.append(int(job["id"])) + + dispatcher.register("noop", _handler) + asyncio.run(dispatcher.dispatch({"id": 1, "type": "noop"})) + assert called == [1] + + +def test_job_dispatch_unknown_type_rejected() -> None: + dispatcher = JobDispatcher() + with pytest.raises(ValueError, match="Unknown job type"): + asyncio.run(dispatcher.dispatch({"id": 1, "type": "unknown"})) + + +def test_job_dispatch_missing_type_rejected() -> None: + dispatcher = JobDispatcher() + with pytest.raises(ValueError, match="Unknown job type"): + asyncio.run(dispatcher.dispatch({"id": 1})) + diff --git a/backend/tests/test_job_fsm.py b/backend/tests/test_job_fsm.py new file mode 100644 index 0000000..b72b524 --- /dev/null +++ b/backend/tests/test_job_fsm.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from app.jobs.backoff import backoff_seconds +from app.jobs.model import Job, JobStatus, on_job_failure, on_job_success + + +def test_job_fsm_backoff_schedule() -> None: + assert backoff_seconds(0) == 0 + assert backoff_seconds(1) == 5 + assert backoff_seconds(2) == 30 + assert backoff_seconds(3) == 120 + assert backoff_seconds(4) == 600 + assert backoff_seconds(5) == 1800 + assert backoff_seconds(6) >= 1800 + + +def test_job_fsm_failure_transitions_to_failed_and_sets_run_after() -> None: + now = datetime(2026, 2, 10, 0, 0, 0, tzinfo=timezone.utc) + job = Job(id=1, status=JobStatus.RUNNING, attempt=0, max_attempts=3) + t = on_job_failure(job, error="boom", now=now) + assert t.status == JobStatus.FAILED + assert t.attempt == 1 + assert t.run_after == "2026-02-10T00:00:05.000Z" + assert t.locked_by is None + assert t.locked_at is None + + +def test_job_fsm_failure_transitions_to_dlq_at_max_attempts() -> None: + now = datetime(2026, 2, 10, 0, 0, 0, tzinfo=timezone.utc) + job = Job(id=1, status=JobStatus.RUNNING, attempt=2, max_attempts=3) + t = on_job_failure(job, error="boom", now=now) + assert t.status == JobStatus.DLQ + assert t.attempt == 3 + assert t.run_after is None + + +def test_job_fsm_success_transitions_to_completed() -> None: + now = datetime(2026, 2, 10, 0, 0, 0, tzinfo=timezone.utc) + job = Job(id=1, status=JobStatus.RUNNING, attempt=1, max_attempts=3) + t = on_job_success(job, now=now) + assert t.status == JobStatus.COMPLETED + assert t.run_after is None + assert t.last_error is None + diff --git a/backend/tests/test_job_handler_easy_proxies_import.py b/backend/tests/test_job_handler_easy_proxies_import.py new file mode 100644 index 0000000..5216246 --- /dev/null +++ b/backend/tests/test_job_handler_easy_proxies_import.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import httpx +import sqlalchemy as sa +from cryptography.fernet import Fernet + +from app.core.crypto import FieldEncryptor +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.jobs import JobRow +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.session import create_sessionmaker +from app.jobs.claim import claim_next_job +from app.jobs.dispatch import JobDispatcher +from app.jobs.executor import execute_claimed_job +from app.jobs.handlers.easy_proxies_import import build_easy_proxies_import_handler + + +def _sqlite_url(db_path: Path) -> str: + return "sqlite+aiosqlite:///" + db_path.as_posix() + + +def test_job_handler_easy_proxies_import_skips_non_easy_sources_by_default(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_easy_proxies_import.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + base_url = "http://easy-proxies:9090" + password = "pw" + + old_password = "old" + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == f"{base_url}/api/auth": + return httpx.Response(200, json={"token": "t"}) + if str(req.url) == f"{base_url}/api/export": + assert req.headers.get("Authorization") == "Bearer t" + return httpx.Response( + 200, + text="\n".join( + [ + "http://u:p@1.2.3.4:8080", + "http://2.2.2.2:8081", + "not_a_uri", + "", + ] + ), + ) + return httpx.Response(500, text="unexpected") + + transport = httpx.MockTransport(handler) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add( + ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="u", + password_enc=encryptor.encrypt_text(old_password), + enabled=1, + source="manual", + source_ref=None, + ) + ) + session.add( + JobRow( + type="easy_proxies_import", + status="pending", + payload_json=json.dumps( + { + "base_url": base_url, + "password": password, + "conflict_policy": "skip_non_easy_proxies", + }, + ensure_ascii=False, + separators=(",", ":"), + ), + ) + ) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register( + "easy_proxies_import", + build_easy_proxies_import_handler(engine, transport=transport), + ) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + assert claimed["type"] == "easy_proxies_import" + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + endpoints = ( + (await session.execute(sa.select(ProxyEndpoint).order_by(ProxyEndpoint.id.asc()))) + .scalars() + .all() + ) + assert len(endpoints) == 2 + + existing = endpoints[0] + assert existing.source == "manual" + assert encryptor.decrypt_text(str(existing.password_enc)) == old_password + + created = endpoints[1] + assert created.source == "easy_proxies" + assert created.source_ref == base_url + assert created.host == "2.2.2.2" + assert int(created.port) == 8081 + + await engine.dispose() + + asyncio.run(_run()) + + +def test_job_handler_easy_proxies_import_uses_env_password_when_missing(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_easy_proxies_import_env_password.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + + base_url = "http://easy-proxies:9090" + env_password = "pw_env_secret" + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("EASY_PROXIES_PASSWORD", env_password) + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == f"{base_url}/api/auth": + body = json.loads((req.content or b"{}").decode("utf-8")) + assert body.get("password") == env_password + return httpx.Response(200, json={"token": "t"}) + if str(req.url) == f"{base_url}/api/export": + assert req.headers.get("Authorization") == "Bearer t" + return httpx.Response(200, text="http://2.2.2.2:8081\n") + return httpx.Response(500, text="unexpected") + + transport = httpx.MockTransport(handler) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add( + JobRow( + type="easy_proxies_import", + status="pending", + payload_json=json.dumps( + {"base_url": base_url, "conflict_policy": "overwrite"}, + ensure_ascii=False, + separators=(",", ":"), + ), + ) + ) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register( + "easy_proxies_import", + build_easy_proxies_import_handler(engine, transport=transport), + ) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + assert claimed["type"] == "easy_proxies_import" + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + endpoints = ( + (await session.execute(sa.select(ProxyEndpoint).order_by(ProxyEndpoint.id.asc()))) + .scalars() + .all() + ) + assert len(endpoints) == 1 + created = endpoints[0] + assert created.source == "easy_proxies" + assert created.source_ref == base_url + assert created.host == "2.2.2.2" + assert int(created.port) == 8081 + + await engine.dispose() + + asyncio.run(_run()) + + +def test_job_handler_easy_proxies_import_rewrites_placeholder_host(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_easy_proxies_import_rewrite_host.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + base_url = "http://easy-proxies:9090" + password = "pw" + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == f"{base_url}/api/auth": + return httpx.Response(200, json={"token": "t"}) + if str(req.url) == f"{base_url}/api/export": + assert req.headers.get("Authorization") == "Bearer t" + return httpx.Response(200, text="http://u:p@0.0.0.0:8080\n") + return httpx.Response(500, text="unexpected") + + transport = httpx.MockTransport(handler) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add( + ProxyEndpoint( + scheme="http", + host="0.0.0.0", + port=8080, + username="u", + password_enc=encryptor.encrypt_text("old"), + enabled=1, + source="easy_proxies", + source_ref=base_url, + failure_count=7, + blacklisted_until="2099-01-01T00:00:00Z", + last_error="ConnectError: old", + ) + ) + session.add( + JobRow( + type="easy_proxies_import", + status="pending", + payload_json=json.dumps( + { + "base_url": base_url, + "password": password, + "conflict_policy": "skip_non_easy_proxies", + }, + ensure_ascii=False, + separators=(",", ":"), + ), + ) + ) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register( + "easy_proxies_import", + build_easy_proxies_import_handler(engine, transport=transport), + ) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + endpoints = ( + (await session.execute(sa.select(ProxyEndpoint).order_by(ProxyEndpoint.id.asc()))) + .scalars() + .all() + ) + assert len(endpoints) == 1 + row = endpoints[0] + assert row.host == "easy-proxies" + assert int(row.port) == 8080 + assert row.blacklisted_until is None + assert row.last_error is None + + await engine.dispose() + + asyncio.run(_run()) diff --git a/backend/tests/test_job_handler_heal_url.py b/backend/tests/test_job_handler_heal_url.py new file mode 100644 index 0000000..28c9eeb --- /dev/null +++ b/backend/tests/test_job_handler_heal_url.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import httpx +import sqlalchemy as sa +from cryptography.fernet import Fernet + +from app.core.crypto import FieldEncryptor +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.models.jobs import JobRow +from app.db.models.pixiv_tokens import PixivToken +from app.db.session import create_sessionmaker +from app.jobs.claim import claim_next_job +from app.jobs.dispatch import JobDispatcher +from app.jobs.executor import execute_claimed_job +from app.jobs.handlers.heal_url import build_heal_url_handler + + +def _sqlite_url(db_path: Path) -> str: + return "sqlite+aiosqlite:///" + db_path.as_posix() + + +def test_job_handler_heal_url_marks_broken_images_active_after_hydrate(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_heal_url.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + monkeypatch.setenv("PIXIV_OAUTH_HASH_SECRET", "hsec_test") + + refresh_token = "rt_old" + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == "https://oauth.secure.pixiv.net/auth/token": + return httpx.Response( + 200, + json={ + "response": { + "access_token": "at_test", + "token_type": "bearer", + "expires_in": 3600, + "refresh_token": None, + "scope": "", + "user": {"id": 123}, + } + }, + ) + if str(req.url).startswith("https://app-api.pixiv.net/v1/illust/detail"): + return httpx.Response( + 200, + json={ + "illust": { + "id": 333, + "title": "title_test", + "user": {"id": 1, "name": "u"}, + "x_restrict": 0, + "illust_ai_type": 0, + "width": 100, + "height": 200, + "create_date": "2020-01-01T00:00:00+00:00", + "page_count": 1, + "tags": [], + "meta_single_page": { + "original_image_url": "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/333_p0.jpg" + }, + } + }, + ) + return httpx.Response(500, text="unexpected") + + transport = httpx.MockTransport(handler) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + + async with Session() as session: + session.add( + PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text(refresh_token), + refresh_token_masked="***", + weight=1.0, + ) + ) + + img = Image( + illust_id=333, + page_index=0, + ext="jpg", + original_url="https://i.pximg.net/img-original/old/333_p0.jpg", + proxy_path="", + random_key=0.5, + status=3, + last_error_code="UPSTREAM_404", + last_error_msg="old_error", + ) + session.add(img) + await session.flush() + img.proxy_path = f"/i/{int(img.id)}.jpg" + + session.add( + JobRow( + type="heal_url", + status="pending", + payload_json=json.dumps({"illust_id": 333, "trigger": "upstream_404"}, ensure_ascii=False, separators=(",", ":")), + ) + ) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register("heal_url", build_heal_url_handler(engine, transport=transport)) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + assert claimed["type"] == "heal_url" + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + img2 = ( + ( + await session.execute( + sa.select(Image).where(Image.illust_id == 333).where(Image.page_index == 0) + ) + ) + .scalars() + .one() + ) + assert img2.status == 1 + assert img2.last_ok_at is not None and img2.last_ok_at + assert img2.last_error_code is None + assert img2.last_error_msg is None + assert img2.original_url.endswith("333_p0.jpg") + assert img2.proxy_path == f"/i/{int(img2.id)}.jpg" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_job_handler_hydrate_metadata.py b/backend/tests/test_job_handler_hydrate_metadata.py new file mode 100644 index 0000000..dba06ed --- /dev/null +++ b/backend/tests/test_job_handler_hydrate_metadata.py @@ -0,0 +1,697 @@ +from __future__ import annotations + +import asyncio +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import httpx +import sqlalchemy as sa +from cryptography.fernet import Fernet + +from app.core.crypto import FieldEncryptor +from app.core.runtime_settings import set_runtime_setting +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.models.imports import Import +from app.db.models.jobs import JobRow +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.models.tags import Tag +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker +from app.db.tags_get import get_tag_names_for_image +from app.jobs.claim import claim_next_job +from app.jobs.dispatch import JobDispatcher +from app.jobs.executor import execute_claimed_job +from app.jobs.handlers.hydrate_metadata import build_hydrate_metadata_handler +from app.pixiv.oauth import PixivOauthToken + + +def _sqlite_url(db_path: Path) -> str: + return "sqlite+aiosqlite:///" + db_path.as_posix() + + +def _parse_iso_utc_ms(value: str) -> datetime: + raw = value.strip() + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + dt = datetime.fromisoformat(raw) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def test_job_handler_hydrate_metadata_happy_path_updates_images_and_tags(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_hydrate_metadata.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + monkeypatch.setenv("PIXIV_OAUTH_HASH_SECRET", "hsec_test") + + refresh_token = "rt_old" + access_token = "at_test" + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == "https://oauth.secure.pixiv.net/auth/token": + body = (req.content or b"").decode("utf-8") + assert f"refresh_token={refresh_token}" in body + return httpx.Response( + 200, + json={ + "response": { + "access_token": access_token, + "token_type": "bearer", + "expires_in": 3600, + "refresh_token": "rt_rotated", + "scope": "", + "user": {"id": 123}, + } + }, + ) + + if str(req.url).startswith("https://app-api.pixiv.net/v1/illust/detail"): + assert req.headers.get("Authorization") == f"Bearer {access_token}" + return httpx.Response( + 200, + json={ + "illust": { + "id": 111, + "title": "title_test", + "user": {"id": 999, "name": "user_test"}, + "x_restrict": 0, + "illust_type": 0, + "illust_ai_type": 1, + "width": 1200, + "height": 800, + "create_date": "2020-01-01T00:00:00+00:00", + "page_count": 2, + "tags": [ + {"name": "tag1", "translated_name": "t1"}, + {"name": "tag2"}, + ], + "meta_pages": [ + { + "image_urls": { + "original": "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p0.jpg" + } + }, + { + "image_urls": { + "original": "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p1.png" + } + }, + ], + } + }, + ) + + return httpx.Response(500, text="unexpected") + + transport = httpx.MockTransport(handler) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + + async with Session() as session: + token_row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text(refresh_token), + refresh_token_masked="***", + weight=1.0, + ) + session.add(token_row) + + imp = Import(created_by="admin", source="manual") + session.add(imp) + await session.commit() + await session.refresh(token_row) + await session.refresh(imp) + + img0 = Image( + illust_id=111, + page_index=0, + ext="jpg", + original_url="https://i.pximg.net/img-original/old/111_p0.jpg", + proxy_path="", + random_key=0.123, + created_import_id=int(imp.id), + ) + session.add(img0) + await session.flush() + img0.proxy_path = f"/i/{int(img0.id)}.jpg" + + job = JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps({"illust_id": 111, "reason": "import"}, ensure_ascii=False, separators=(",", ":")), + ref_type="import", + ref_id=f"{int(imp.id)}:111", + ) + session.add(job) + await session.commit() + + existing_image_id = int(img0.id) + import_id = int(imp.id) + + dispatcher = JobDispatcher() + dispatcher.register("hydrate_metadata", build_hydrate_metadata_handler(engine, transport=transport)) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + images = ( + ( + await session.execute( + sa.select(Image).where(Image.illust_id == 111).order_by(Image.page_index.asc()) + ) + ) + .scalars() + .all() + ) + assert len(images) == 2 + assert int(images[0].id) == existing_image_id + assert images[0].ext == "jpg" + assert images[1].ext == "png" + assert images[0].proxy_path == f"/i/{int(images[0].id)}.jpg" + assert images[1].proxy_path == f"/i/{int(images[1].id)}.png" + assert images[0].created_import_id == import_id + assert images[1].created_import_id == import_id + + assert images[0].width == 1200 + assert images[0].height == 800 + assert images[0].orientation == 2 + assert images[0].aspect_ratio and abs(float(images[0].aspect_ratio) - 1.5) < 1e-6 + assert images[0].x_restrict == 0 + assert images[0].ai_type == 1 + assert images[0].illust_type == 0 + assert images[0].user_id == 999 + assert images[0].user_name == "user_test" + assert images[0].title == "title_test" + assert images[0].created_at_pixiv == "2020-01-01T00:00:00Z" + + tags = ((await session.execute(sa.select(Tag).order_by(Tag.name.asc()))).scalars().all()) + assert [t.name for t in tags] == ["tag1", "tag2"] + assert {t.name: t.translated_name for t in tags} == {"tag1": "t1", "tag2": None} + + names0 = await get_tag_names_for_image(session, image_id=int(images[0].id)) + names1 = await get_tag_names_for_image(session, image_id=int(images[1].id)) + assert names0 == ["tag1", "tag2"] + assert names1 == ["tag1", "tag2"] + + token_db = await session.get(PixivToken, int(token_row.id)) + assert token_db is not None + assert int(token_db.error_count or 0) == 0 + assert token_db.backoff_until is None + assert token_db.last_ok_at is not None and token_db.last_ok_at + + await engine.dispose() + + asyncio.run(_run()) + + +def test_job_handler_hydrate_metadata_missing_illust_id_moves_to_dlq(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_hydrate_metadata_missing_illust_id.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add( + JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps({}, ensure_ascii=False, separators=(",", ":")), + ) + ) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register("hydrate_metadata", build_hydrate_metadata_handler(engine, transport=httpx.MockTransport(lambda r: httpx.Response(500)))) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "dlq" + + async with Session() as session: + row = ( + ( + await session.execute( + sa.select(JobRow).where(JobRow.id == int(claimed["id"])) + ) + ) + .scalars() + .one() + ) + assert row.status == "dlq" + + await engine.dispose() + + asyncio.run(_run()) + + +def test_job_handler_hydrate_metadata_rate_limit_defers_job_and_updates_token(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_hydrate_metadata_rate_limit.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + + refresh_token = "rt_old" + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == "https://oauth.secure.pixiv.net/auth/token": + return httpx.Response( + 200, + json={ + "response": { + "access_token": "at_test", + "token_type": "bearer", + "expires_in": 3600, + "refresh_token": None, + "scope": "", + "user": {"id": 123}, + } + }, + ) + if str(req.url).startswith("https://app-api.pixiv.net/v1/illust/detail"): + return httpx.Response(403, text="Rate Limit") + return httpx.Response(500, text="unexpected") + + transport = httpx.MockTransport(handler) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + token_row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text(refresh_token), + refresh_token_masked="***", + weight=1.0, + ) + session.add(token_row) + session.add( + JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps({"illust_id": 222}, ensure_ascii=False, separators=(",", ":")), + ) + ) + await session.commit() + await session.refresh(token_row) + token_id = int(token_row.id) + + dispatcher = JobDispatcher() + dispatcher.register("hydrate_metadata", build_hydrate_metadata_handler(engine, transport=transport)) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "failed" + assert transition.run_after is not None and transition.run_after + assert transition.attempt == 0 + + async with Session() as session: + job_row = await session.get(JobRow, int(claimed["id"])) + assert job_row is not None + assert job_row.status == "failed" + assert int(job_row.attempt) == 0 + assert job_row.run_after is not None and job_row.run_after + + token_db = await session.get(PixivToken, token_id) + assert token_db is not None + assert int(token_db.error_count or 0) == 1 + assert token_db.backoff_until is not None and token_db.backoff_until + assert token_db.last_error_code == "TOKEN_BACKOFF" + + now = datetime.now(timezone.utc) + run_after_dt = _parse_iso_utc_ms(str(job_row.run_after)) + delta_s = (run_after_dt - now).total_seconds() + assert 10.0 < delta_s < 180.0 + + await engine.dispose() + + asyncio.run(_run()) + + +def test_job_handler_hydrate_metadata_refresh_network_error_defers_without_token_backoff( + tmp_path: Path, monkeypatch +) -> None: + db_path = tmp_path / "handler_hydrate_metadata_refresh_network_error.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == "https://oauth.secure.pixiv.net/auth/token": + raise httpx.ConnectError("connect failed", request=req) + return httpx.Response(500, text="unexpected") + + transport = httpx.MockTransport(handler) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + token_row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text("rt_test"), + refresh_token_masked="***", + weight=1.0, + ) + session.add(token_row) + session.add( + JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps({"illust_id": 333}, ensure_ascii=False, separators=(",", ":")), + ) + ) + await session.commit() + await session.refresh(token_row) + token_id = int(token_row.id) + + dispatcher = JobDispatcher() + dispatcher.register("hydrate_metadata", build_hydrate_metadata_handler(engine, transport=transport)) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "failed" + assert transition.run_after is not None and transition.run_after + assert transition.attempt == 0 + + async with Session() as session: + job_row = await session.get(JobRow, int(claimed["id"])) + assert job_row is not None + assert job_row.status == "failed" + assert int(job_row.attempt) == 0 + assert job_row.run_after is not None and job_row.run_after + assert job_row.last_error is not None + assert "PROXY_CONNECT_FAILED" in str(job_row.last_error) + assert "NO_TOKEN_AVAILABLE" not in str(job_row.last_error) + + token_db = await session.get(PixivToken, token_id) + assert token_db is not None + assert int(token_db.error_count or 0) == 0 + assert token_db.backoff_until is None + + await engine.dispose() + + asyncio.run(_run()) + + +def test_job_handler_hydrate_metadata_proxy_success_updates_endpoint_and_override(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_hydrate_metadata_proxy_success.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + + refresh_proxies: list[str | None] = [] + detail_proxies: list[str | None] = [] + + async def fake_refresh_access_token(*, refresh_token: str, config, transport=None, proxy=None, **kwargs): # type: ignore[no-untyped-def] + refresh_proxies.append(proxy) + return PixivOauthToken( + access_token="at_test", + token_type="bearer", + expires_in=3600, + refresh_token=None, + scope="", + user_id="123", + ) + + class FakeAsyncClient: + def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] + detail_proxies.append(kwargs.get("proxy")) + + async def __aenter__(self): # type: ignore[no-untyped-def] + return self + + async def __aexit__(self, exc_type, exc, tb): # type: ignore[no-untyped-def] + return False + + async def get(self, url, **kwargs): # type: ignore[no-untyped-def] + req = httpx.Request("GET", url) + return httpx.Response( + 200, + request=req, + json={ + "illust": { + "id": 444, + "title": "t", + "user": {"id": 1, "name": "u"}, + "x_restrict": 0, + "illust_ai_type": 0, + "width": 100, + "height": 100, + "create_date": "2020-01-01T00:00:00+00:00", + "page_count": 1, + "tags": [], + "meta_single_page": { + "original_image_url": "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/444_p0.jpg" + }, + } + }, + ) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + pool = ProxyPool(name="p1", enabled=1) + session.add(pool) + await session.flush() + pool_id = int(pool.id) + + endpoint = ProxyEndpoint(scheme="http", host="proxy.test", port=1234, username="", password_enc="") + session.add(endpoint) + await session.flush() + endpoint_id = int(endpoint.id) + + session.add(ProxyPoolEndpoint(pool_id=pool_id, endpoint_id=endpoint_id, enabled=1, weight=1)) + + token_row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text("rt_test"), + refresh_token_masked="***", + weight=1.0, + ) + session.add(token_row) + await session.flush() + + session.add( + TokenProxyBinding( + token_id=int(token_row.id), + pool_id=pool_id, + primary_proxy_id=endpoint_id, + ) + ) + session.add( + JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps({"illust_id": 444}, ensure_ascii=False, separators=(",", ":")), + ) + ) + await session.commit() + token_id = int(token_row.id) + + await set_runtime_setting(engine, key="proxy.enabled", value=True, updated_by="test") + await set_runtime_setting(engine, key="proxy.default_pool_id", value=pool_id, updated_by="test") + + import app.jobs.handlers.hydrate_metadata as hm + + monkeypatch.setattr(hm, "refresh_access_token", fake_refresh_access_token, raising=True) + monkeypatch.setattr(hm.httpx, "AsyncClient", FakeAsyncClient, raising=True) + + dispatcher = JobDispatcher() + dispatcher.register("hydrate_metadata", build_hydrate_metadata_handler(engine, transport=httpx.MockTransport(lambda r: httpx.Response(500)))) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + token_db = await session.get(PixivToken, token_id) + assert token_db is not None + assert int(token_db.error_count or 0) == 0 + assert token_db.backoff_until is None + + ep_db = await session.get(ProxyEndpoint, endpoint_id) + assert ep_db is not None + assert int(ep_db.success_count or 0) >= 1 + assert ep_db.last_ok_at is not None and ep_db.last_ok_at + assert ep_db.blacklisted_until is None + + binding = ( + ( + await session.execute( + sa.select(TokenProxyBinding) + .where(TokenProxyBinding.token_id == token_id, TokenProxyBinding.pool_id == pool_id) + .limit(1) + ) + ) + .scalars() + .one() + ) + assert int(binding.override_proxy_id or 0) == endpoint_id + assert binding.override_expires_at is not None and binding.override_expires_at + assert _parse_iso_utc_ms(str(binding.override_expires_at)) > datetime.now(timezone.utc) + + assert refresh_proxies and refresh_proxies[0] + assert detail_proxies and detail_proxies[0] + + await engine.dispose() + + asyncio.run(_run()) + + +def test_job_handler_hydrate_metadata_proxy_required_defers_until_next_available_at(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_hydrate_metadata_proxy_required_next_available.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + + future_dt = datetime.now(timezone.utc) + timedelta(seconds=60) + next_available_at = future_dt.isoformat(timespec="milliseconds").replace("+00:00", "Z") + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + pool = ProxyPool(name="p1", enabled=1) + session.add(pool) + await session.flush() + pool_id = int(pool.id) + + endpoint = ProxyEndpoint( + scheme="http", + host="proxy.test", + port=1234, + username="", + password_enc="", + blacklisted_until=next_available_at, + ) + session.add(endpoint) + await session.flush() + session.add(ProxyPoolEndpoint(pool_id=pool_id, endpoint_id=int(endpoint.id), enabled=1, weight=1)) + + token_row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text("rt_test"), + refresh_token_masked="***", + weight=1.0, + ) + session.add(token_row) + + session.add( + JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps({"illust_id": 555}, ensure_ascii=False, separators=(",", ":")), + ) + ) + await session.commit() + + await set_runtime_setting(engine, key="proxy.enabled", value=True, updated_by="test") + await set_runtime_setting(engine, key="proxy.fail_closed", value=True, updated_by="test") + await set_runtime_setting(engine, key="proxy.default_pool_id", value=pool_id, updated_by="test") + + dispatcher = JobDispatcher() + dispatcher.register( + "hydrate_metadata", + build_hydrate_metadata_handler(engine, transport=httpx.MockTransport(lambda r: httpx.Response(500))), + ) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "failed" + assert transition.run_after == next_available_at + + async with Session() as session: + job_row = await session.get(JobRow, int(claimed["id"])) + assert job_row is not None + assert job_row.run_after == next_available_at + assert job_row.last_error is not None + assert "PROXY_REQUIRED" in str(job_row.last_error) + + await engine.dispose() + + asyncio.run(_run()) diff --git a/backend/tests/test_job_handler_hydration_run_backfill.py b/backend/tests/test_job_handler_hydration_run_backfill.py new file mode 100644 index 0000000..a8cd12d --- /dev/null +++ b/backend/tests/test_job_handler_hydration_run_backfill.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import asyncio +import json +from datetime import datetime, timezone +from pathlib import Path + +import httpx +import sqlalchemy as sa +from cryptography.fernet import Fernet + +from app.core.crypto import FieldEncryptor +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.hydration_runs import HydrationRun +from app.db.models.images import Image +from app.db.models.jobs import JobRow +from app.db.models.pixiv_tokens import PixivToken +from app.db.session import create_sessionmaker +from app.jobs.claim import claim_next_job +from app.jobs.dispatch import JobDispatcher +from app.jobs.executor import execute_claimed_job +from app.jobs.handlers.hydrate_metadata import build_hydrate_metadata_handler + + +def _sqlite_url(db_path: Path) -> str: + return "sqlite+aiosqlite:///" + db_path.as_posix() + + +def test_hydration_run_backfill_job_processes_missing_metadata(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_hydration_run_backfill.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + encryptor = FieldEncryptor.from_key(field_key) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "cid_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "csec_test") + monkeypatch.setenv("PIXIV_OAUTH_HASH_SECRET", "hsec_test") + + refresh_token = "rt_old" + access_token = "at_test" + illust_id = 111 + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == "https://oauth.secure.pixiv.net/auth/token": + body = (req.content or b"").decode("utf-8") + assert f"refresh_token={refresh_token}" in body + return httpx.Response( + 200, + json={ + "response": { + "access_token": access_token, + "token_type": "bearer", + "expires_in": 3600, + "refresh_token": "rt_rotated", + "scope": "", + "user": {"id": 123}, + } + }, + ) + + if str(req.url).startswith("https://app-api.pixiv.net/v1/illust/detail"): + assert req.headers.get("Authorization") == f"Bearer {access_token}" + return httpx.Response( + 200, + json={ + "illust": { + "id": illust_id, + "title": "title_test", + "user": {"id": 999, "name": "user_test"}, + "x_restrict": 0, + "illust_ai_type": 1, + "width": 1200, + "height": 800, + "create_date": "2020-01-01T00:00:00+00:00", + "page_count": 1, + "tags": [ + {"name": "tag1", "translated_name": "t1"}, + {"name": "tag2"}, + ], + "meta_single_page": { + "original_image_url": "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p0.jpg" + }, + } + }, + ) + + return httpx.Response(500, text="unexpected") + + transport = httpx.MockTransport(handler) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + token = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc=encryptor.encrypt_text(refresh_token), + refresh_token_masked="***", + weight=1.0, + ) + session.add(token) + await session.flush() + + img = Image( + illust_id=int(illust_id), + page_index=0, + ext="jpg", + original_url="https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p0.jpg", + proxy_path="/i/1.jpg", + random_key=0.123, + width=None, + height=None, + aspect_ratio=None, + orientation=None, + x_restrict=None, + ai_type=None, + user_id=None, + user_name=None, + title=None, + created_at_pixiv=None, + status=1, + fail_count=0, + last_fail_at=None, + last_ok_at=None, + last_error_code=None, + last_error_msg=None, + created_import_id=None, + ) + session.add(img) + await session.flush() + + run = HydrationRun(type="backfill", status="pending", criteria_json=json.dumps({"missing": ["r18"]}), cursor_json="{}") + session.add(run) + await session.flush() + + job = JobRow( + type="hydrate_metadata", + status="pending", + payload_json=json.dumps({"hydration_run_id": int(run.id), "criteria": {"missing": ["r18"]}}), + last_error=None, + priority=0, + run_after=None, + attempt=0, + max_attempts=3, + locked_by=None, + locked_at=None, + ref_type="hydration_run", + ref_id=str(int(run.id)), + ) + session.add(job) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register("hydrate_metadata", build_hydrate_metadata_handler(engine, transport=transport)) + + claimed = await claim_next_job(engine, worker_id="test-worker") + assert claimed is not None + assert str(claimed.get("type")) == "hydrate_metadata" + + await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="test-worker") + + async with Session() as session: + run2 = await session.get(HydrationRun, int(run.id)) + assert run2 is not None + assert run2.status == "completed" + assert int(run2.processed or 0) == 1 + assert int(run2.success or 0) == 1 + assert int(run2.failed or 0) == 0 + assert run2.started_at is not None + assert run2.finished_at is not None + + job2 = await session.get(JobRow, int(job.id)) + assert job2 is not None + assert job2.status == "completed" + + img2 = ( + (await session.execute(sa.select(Image).where(Image.illust_id == int(illust_id), Image.page_index == 0))) + .scalars() + .first() + ) + assert img2 is not None + assert img2.width == 1200 + assert img2.height == 800 + assert img2.x_restrict == 0 + assert img2.user_id == 999 + assert (img2.title or "") == "title_test" + assert img2.created_at_pixiv is not None + + asyncio.run(_run()) + diff --git a/backend/tests/test_job_handler_import_images.py b/backend/tests/test_job_handler_import_images.py new file mode 100644 index 0000000..a06bfc8 --- /dev/null +++ b/backend/tests/test_job_handler_import_images.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import sqlalchemy as sa + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.models.imports import Import +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.jobs.claim import claim_next_job +from app.jobs.dispatch import JobDispatcher +from app.jobs.executor import execute_claimed_job +from app.jobs.handlers.import_images import build_import_images_handler + + +def _sqlite_url(db_path: Path) -> str: + return "sqlite+aiosqlite:///" + db_path.as_posix() + + +def test_job_handler_import_images_happy_path_and_enqueues_hydrate(tmp_path: Path) -> None: + db_path = tmp_path / "handler_import_images.db" + engine = create_engine(_sqlite_url(db_path)) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + imp = Import(created_by="admin", source="manual") + session.add(imp) + await session.commit() + await session.refresh(imp) + + payload = { + "import_id": int(imp.id), + "hydrate_on_import": True, + "text_lines": [ + "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p0.jpg", + "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p1.jpg", + "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p0.jpg", + "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/222_p0.png", + "https://example.com/not_pximg.jpg", + ], + } + job = JobRow( + type="import_images", + status="pending", + payload_json=json.dumps(payload, ensure_ascii=False, separators=(",", ":")), + ref_type="import", + ref_id=str(int(imp.id)), + ) + session.add(job) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register("import_images", build_import_images_handler(engine)) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + assert claimed["type"] == "import_images" + assert claimed["status"] == "running" + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + imp2 = await session.get(Import, int(payload["import_id"])) + assert imp2 is not None + assert int(imp2.total) == 5 + assert int(imp2.accepted) == 3 + assert int(imp2.success) == 3 + assert int(imp2.failed) == 1 + + detail = json.loads(imp2.detail_json or "{}") + assert detail["deduped"] == 1 + assert len(detail["errors"]) == 1 + + images = ( + (await session.execute(sa.select(Image).order_by(Image.id.asc()))).scalars().all() + ) + assert len(images) == 3 + for img in images: + assert img.proxy_path.startswith("/i/") + assert img.created_import_id == int(payload["import_id"]) + + hydrate_jobs = ( + ( + await session.execute( + sa.select(JobRow) + .where(JobRow.type == "hydrate_metadata") + .where(JobRow.ref_type == "import") + .order_by(JobRow.id.asc()) + ) + ) + .scalars() + .all() + ) + assert len(hydrate_jobs) == 2 + ref_ids = {j.ref_id for j in hydrate_jobs} + assert ref_ids == {f"{imp2.id}:111", f"{imp2.id}:222"} + + import_job = ( + (await session.execute(sa.select(JobRow).where(JobRow.type == "import_images"))) + .scalars() + .first() + ) + assert import_job is not None + assert import_job.status == "completed" + assert import_job.locked_by is None + assert import_job.locked_at is None + + await engine.dispose() + + asyncio.run(_run()) + + +def test_job_handler_import_images_supports_file_ref(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_import_images_file_ref.db" + db_url = _sqlite_url(db_path) + engine = create_engine(db_url) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + payload_dir = tmp_path / "imports_payloads" + payload_dir.mkdir(parents=True, exist_ok=True) + payload_path = payload_dir / "urls.txt" + payload_path.write_text( + "\n".join( + [ + "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p0.jpg", + "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/111_p0.jpg", + "https://i.pximg.net/img-original/img/2020/01/01/00/00/00/222_p0.png", + "https://example.com/not_pximg.jpg", + ] + ), + encoding="utf-8", + ) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + imp = Import(created_by="admin", source="manual") + session.add(imp) + await session.commit() + await session.refresh(imp) + + payload = { + "import_id": int(imp.id), + "hydrate_on_import": False, + "file_ref": "imports_payloads/urls.txt", + } + session.add( + JobRow( + type="import_images", + status="pending", + payload_json=json.dumps(payload, ensure_ascii=False, separators=(",", ":")), + ref_type="import", + ref_id=str(int(imp.id)), + ) + ) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register("import_images", build_import_images_handler(engine)) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + imp2 = await session.get(Import, int(imp.id)) + assert imp2 is not None + assert int(imp2.total) == 4 + assert int(imp2.accepted) == 2 + assert int(imp2.success) == 2 + assert int(imp2.failed) == 1 + + images = ((await session.execute(sa.select(Image))).scalars().all()) + assert len(images) == 2 + + assert payload_path.exists() is False + + await engine.dispose() + + asyncio.run(_run()) + + +def test_job_handler_import_images_missing_import_id_moves_to_dlq(tmp_path: Path) -> None: + db_path = tmp_path / "handler_import_images_missing_import_id.db" + engine = create_engine(_sqlite_url(db_path)) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add( + JobRow( + type="import_images", + status="pending", + payload_json=json.dumps({}, ensure_ascii=False, separators=(",", ":")), + ) + ) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register("import_images", build_import_images_handler(engine)) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "dlq" + assert transition.attempt == 1 + assert transition.run_after is None + assert transition.last_error + + async with Session() as session: + row = ( + ( + await session.execute( + sa.select(JobRow).where(JobRow.id == int(claimed["id"])) + ) + ) + .scalars() + .one() + ) + assert row.status == "dlq" + assert int(row.attempt) == 1 + + await engine.dispose() + + asyncio.run(_run()) diff --git a/backend/tests/test_job_handler_proxy_probe.py b/backend/tests/test_job_handler_proxy_probe.py new file mode 100644 index 0000000..1c8e741 --- /dev/null +++ b/backend/tests/test_job_handler_proxy_probe.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import sqlalchemy as sa +from cryptography.fernet import Fernet + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.jobs import JobRow +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.session import create_sessionmaker +from app.jobs.claim import claim_next_job +from app.jobs.dispatch import JobDispatcher +from app.jobs.executor import execute_claimed_job +from app.jobs.handlers.proxy_probe import ProbeConfig, ProbeResult, ProbeTarget, build_proxy_probe_handler + + +def _sqlite_url(db_path: Path) -> str: + return "sqlite+aiosqlite:///" + db_path.as_posix() + + +def test_job_handler_proxy_probe_updates_endpoint_health(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "handler_proxy_probe.db" + engine = create_engine(_sqlite_url(db_path)) + + field_key = Fernet.generate_key().decode("ascii") + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + async def prober(target: ProbeTarget, _cfg: ProbeConfig) -> ProbeResult: + if int(target.endpoint_id) == 1: + return ProbeResult(endpoint_id=1, ok=True, latency_ms=12.5) + return ProbeResult(endpoint_id=int(target.endpoint_id), ok=False, latency_ms=33.3, error="boom") + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add( + ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + ) + ) + session.add( + ProxyEndpoint( + scheme="http", + host="2.2.2.2", + port=8081, + username="", + password_enc="", + enabled=1, + source="manual", + ) + ) + session.add( + JobRow( + type="proxy_probe", + status="pending", + payload_json=json.dumps({"scope": "all"}, ensure_ascii=False, separators=(",", ":")), + ref_type="proxy_probe", + ref_id="all", + ) + ) + await session.commit() + + dispatcher = JobDispatcher() + dispatcher.register("proxy_probe", build_proxy_probe_handler(engine, prober=prober)) + + claimed = await claim_next_job(engine, worker_id="w1") + assert claimed is not None + assert claimed["type"] == "proxy_probe" + + transition = await execute_claimed_job(engine, dispatcher, job_row=claimed, worker_id="w1") + assert transition is not None + assert transition.status.value == "completed" + + async with Session() as session: + endpoints = ( + (await session.execute(sa.select(ProxyEndpoint).order_by(ProxyEndpoint.id.asc()))) + .scalars() + .all() + ) + assert len(endpoints) == 2 + + ok_ep = endpoints[0] + fail_ep = endpoints[1] + + assert int(ok_ep.success_count or 0) == 1 + assert ok_ep.last_ok_at is not None and ok_ep.last_ok_at + assert ok_ep.last_error is None + assert ok_ep.blacklisted_until is None + assert ok_ep.last_latency_ms is not None + + assert int(fail_ep.failure_count or 0) == 1 + assert fail_ep.last_fail_at is not None and fail_ep.last_fail_at + assert fail_ep.last_error is not None and "boom" in str(fail_ep.last_error) + assert fail_ep.last_latency_ms is not None + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_jobs_claim.py b/backend/tests/test_jobs_claim.py new file mode 100644 index 0000000..5d7ae8c --- /dev/null +++ b/backend/tests/test_jobs_claim.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from app.db.engine import create_engine +from app.jobs.claim import claim_next_job, renew_job_lock + + +def _sqlite_url(db_path: Path) -> str: + return "sqlite+aiosqlite:///" + db_path.as_posix() + + +def test_jobs_claim_multi_worker_no_double_claim(tmp_path: Path) -> None: + db_path = tmp_path / "jobs.db" + engine = create_engine(_sqlite_url(db_path)) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.exec_driver_sql( + """ +CREATE TABLE jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT, + updated_at TEXT, + type TEXT NOT NULL, + status TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + run_after TEXT, + attempt INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 3, + payload_json TEXT NOT NULL, + last_error TEXT, + locked_by TEXT, + locked_at TEXT +); +""".strip() + ) + await conn.exec_driver_sql( + """ +INSERT INTO jobs (type,status,priority,run_after,payload_json,created_at,updated_at) +VALUES ('noop','pending',0,NULL,'{}','2026-02-10T00:00:00.000Z','2026-02-10T00:00:00.000Z'); +""".strip() + ) + + start = asyncio.Event() + + async def _claim(worker_id: str): + await start.wait() + return await claim_next_job(engine, worker_id=worker_id) + + t1 = asyncio.create_task(_claim("w1")) + t2 = asyncio.create_task(_claim("w2")) + start.set() + r1, r2 = await asyncio.gather(t1, t2) + + assert (r1 is None) != (r2 is None) + winner = r1 or r2 + assert winner is not None + assert winner["status"] == "running" + assert winner["locked_by"] in {"w1", "w2"} + + async with engine.connect() as conn: + row = (await conn.exec_driver_sql("SELECT status, locked_by FROM jobs WHERE id=1")).one() + assert row[0] == "running" + assert row[1] == winner["locked_by"] + + await engine.dispose() + + asyncio.run(_run()) + + +def test_jobs_claim_ttl_reclaim_running_job(tmp_path: Path) -> None: + db_path = tmp_path / "jobs2.db" + engine = create_engine(_sqlite_url(db_path)) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.exec_driver_sql( + """ +CREATE TABLE jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT, + updated_at TEXT, + type TEXT NOT NULL, + status TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + run_after TEXT, + attempt INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 3, + payload_json TEXT NOT NULL, + last_error TEXT, + locked_by TEXT, + locked_at TEXT +); +""".strip() + ) + + await conn.exec_driver_sql( + """ +INSERT INTO jobs (type,status,priority,run_after,payload_json,locked_by,locked_at,created_at,updated_at) +VALUES ('noop','running',0,NULL,'{}','w1','2026-02-10T00:00:00.000Z','2026-02-10T00:00:00.000Z','2026-02-10T00:00:00.000Z'); +""".strip() + ) + + now = datetime(2026, 2, 10, 0, 10, 0, tzinfo=timezone.utc) + job = await claim_next_job(engine, worker_id="w2", lock_ttl_s=60, now=now) + assert job is not None + assert job["id"] == 1 + assert job["locked_by"] == "w2" + assert job["status"] == "running" + + await engine.dispose() + + asyncio.run(_run()) + + +def test_jobs_claim_heartbeat_renew(tmp_path: Path) -> None: + db_path = tmp_path / "jobs3.db" + engine = create_engine(_sqlite_url(db_path)) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.exec_driver_sql( + """ +CREATE TABLE jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT, + updated_at TEXT, + type TEXT NOT NULL, + status TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + run_after TEXT, + attempt INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 3, + payload_json TEXT NOT NULL, + last_error TEXT, + locked_by TEXT, + locked_at TEXT +); +""".strip() + ) + + await conn.exec_driver_sql( + """ +INSERT INTO jobs (type,status,priority,run_after,payload_json,locked_by,locked_at,created_at,updated_at) +VALUES ('noop','running',0,NULL,'{}','w1','2026-02-10T00:00:00.000Z','2026-02-10T00:00:00.000Z','2026-02-10T00:00:00.000Z'); +""".strip() + ) + + now = datetime(2026, 2, 10, 0, 0, 30, tzinfo=timezone.utc) + assert await renew_job_lock(engine, job_id=1, worker_id="w1", now=now) is True + assert await renew_job_lock(engine, job_id=1, worker_id="w2", now=now + timedelta(seconds=1)) is False + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_jobs_defer_on_sqlite_busy.py b/backend/tests/test_jobs_defer_on_sqlite_busy.py new file mode 100644 index 0000000..4d6400d --- /dev/null +++ b/backend/tests/test_jobs_defer_on_sqlite_busy.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import asyncio +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +from app.db.models.base import Base +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.jobs.dispatch import JobDispatcher +from app.jobs.executor import execute_claimed_job +from app.main import create_app + + +def test_execute_job_defers_on_sqlite_busy(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "jobs_defer_busy.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + engine = app.state.engine + + async def _seed() -> dict: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = JobRow( + type="dummy_busy", + status="running", + payload_json="{}", + last_error=None, + priority=0, + run_after=None, + attempt=0, + max_attempts=3, + locked_by="w1", + locked_at="2026-02-21T00:00:00.000Z", + ) + session.add(row) + await session.commit() + await session.refresh(row) + + return { + "id": int(row.id), + "type": row.type, + "status": row.status, + "payload_json": row.payload_json, + "attempt": int(row.attempt), + "max_attempts": int(row.max_attempts), + "run_after": row.run_after, + "last_error": row.last_error, + "locked_by": row.locked_by, + "locked_at": row.locked_at, + } + + job_row = asyncio.run(_seed()) + + dispatcher = JobDispatcher() + + @dispatcher.handler("dummy_busy") + async def _handler(_job: dict) -> None: + raise sqlite3.OperationalError("database is locked") + + now = datetime(2026, 2, 21, 0, 0, 0, tzinfo=timezone.utc) + + async def _run() -> None: + transition = await execute_claimed_job(engine, dispatcher, job_row=job_row, worker_id="w1", now=now) + assert transition is not None + assert transition.attempt == 0 + assert transition.status.value == "failed" + assert transition.run_after is not None + assert transition.last_error is not None + assert "database is locked" in transition.last_error.lower() + + asyncio.run(_run()) + diff --git a/backend/tests/test_legacy_single.py b/backend/tests/test_legacy_single.py new file mode 100644 index 0000000..3de5c85 --- /dev/null +++ b/backend/tests/test_legacy_single.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import httpx +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_legacy_single_streams_bytes(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "legacy_single.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + def handler(req: httpx.Request) -> httpx.Response: + assert req.headers.get("Referer") == "https://www.pixiv.net/" + return httpx.Response(200, headers={"Content-Type": "image/jpeg"}, content=b"img-bytes") + + app.state.httpx_transport = httpx.MockTransport(handler) + + with TestClient(app) as client: + resp = client.get("/123.jpg", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + assert resp.content == b"img-bytes" + assert resp.headers["Cache-Control"] == "public, max-age=31536000, immutable" + assert resp.headers["X-Request-Id"] == "req_test" + + +def test_legacy_single_ext_mismatch_returns_404(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "legacy_single_mismatch.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp = client.get("/123.png", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 404 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "NOT_FOUND" + assert body["request_id"] == "req_test" + + +def test_legacy_multi_one_based_page(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "legacy_multi.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin_p0.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + ), + Image( + illust_id=123, + page_index=1, + ext="png", + original_url="https://example.test/origin_p1.png", + proxy_path="/i/2.png", + random_key=0.6, + ), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + def handler(req: httpx.Request) -> httpx.Response: + assert req.headers.get("Referer") == "https://www.pixiv.net/" + if str(req.url) == "https://example.test/origin_p1.png": + return httpx.Response(200, headers={"Content-Type": "image/png"}, content=b"png-bytes") + return httpx.Response(404, content=b"") + + app.state.httpx_transport = httpx.MockTransport(handler) + + with TestClient(app) as client: + resp = client.get("/123-2.png", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + assert resp.content == b"png-bytes" + assert resp.headers["Cache-Control"] == "public, max-age=31536000, immutable" + assert resp.headers["X-Request-Id"] == "req_test" + + +def test_legacy_multi_invalid_page_returns_400(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "legacy_multi_invalid_page.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + await app.state.engine.dispose() + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.get("/123-0.jpg", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" diff --git a/backend/tests/test_log_redaction.py b/backend/tests/test_log_redaction.py new file mode 100644 index 0000000..365b048 --- /dev/null +++ b/backend/tests/test_log_redaction.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import io +import logging + +from app.core.logging import RedactFilter +from app.core.redact import REDACTED, redact_any, redact_text + + +def test_redact_proxy_uri_password_with_at() -> None: + raw = "http://user:pa@ss@1.2.3.4:2323" + redacted = redact_text(raw) + assert "pa@ss" not in redacted + assert redacted == "http://user:***@1.2.3.4:2323" + + +def test_redact_proxy_uri_password_when_embedded_in_text_and_punctuated() -> None: + raw = "ProxyError: cannot connect to http://user:pa@ss@1.2.3.4:2323, retry" + redacted = redact_text(raw) + assert "pa@ss" not in redacted + assert "http://user:***@1.2.3.4:2323," in redacted + + +def test_redact_multiple_proxy_uris_in_text() -> None: + raw = "p1=http://u:p@1.1.1.1:1 p2=socks5://a:b@2.2.2.2:2" + redacted = redact_text(raw) + assert "u:p@" not in redacted + assert "a:b@" not in redacted + assert "http://u:***@1.1.1.1:1" in redacted + assert "socks5://a:***@2.2.2.2:2" in redacted + + +def test_redact_bearer_token() -> None: + raw = "Authorization: Bearer abc.def.ghi" + redacted = redact_text(raw) + assert "abc.def.ghi" not in redacted + assert "Bearer ***" in redacted + + +def test_redact_mapping_sensitive_keys() -> None: + raw = {"refresh_token": "secret", "nested": {"password": "p"}, "ok": 1} + redacted = redact_any(raw) + assert redacted["refresh_token"] == REDACTED + assert redacted["nested"]["password"] == REDACTED + assert redacted["ok"] == 1 + + +def test_redact_refresh_token_query_param() -> None: + raw = "GET /auth/token?refresh_token=abc123&x=1" + redacted = redact_text(raw) + assert "abc123" not in redacted + assert "refresh_token=***" in redacted + + +def test_logging_filter_redacts_output() -> None: + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter("%(message)s")) + logger = logging.getLogger("redact_test") + logger.handlers = [handler] + logger.setLevel(logging.INFO) + logger.propagate = False + logger.addFilter(RedactFilter()) + + logger.info("Authorization: Bearer %s", "supersecret") + out = stream.getvalue() + assert "supersecret" not in out + assert "***" in out diff --git a/backend/tests/test_metrics.py b/backend/tests/test_metrics.py new file mode 100644 index 0000000..fe3761f --- /dev/null +++ b/backend/tests/test_metrics.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import asyncio +import re +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_metrics_requires_admin_auth(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "metrics_auth.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.get("/metrics", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 401 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "UNAUTHORIZED" + assert body["request_id"] == "req_test" + + +def test_metrics_exposes_random_jobs_and_proxy_metrics(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "metrics_basic.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + async def _migrate_and_seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + await conn.exec_driver_sql( + "INSERT INTO jobs (type, status, payload_json) VALUES ('import_urls','pending','{}')" + ) + await conn.exec_driver_sql( + "INSERT INTO proxy_endpoints (scheme, host, port) VALUES ('http','example.com',8080)" + ) + + asyncio.run(_migrate_and_seed()) + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.get("/metrics", headers={"Authorization": f"Bearer {token}"}) + assert resp.status_code == 200 + + text = resp.text + assert "new_pixiv_random_requests_total" in text + assert "new_pixiv_random_no_match_total" in text + assert "new_pixiv_random_opportunistic_hydrate_enqueued_total" in text + assert "new_pixiv_upstream_stream_errors_total" in text + assert "new_pixiv_jobs_claim_total" in text + assert "new_pixiv_jobs_failed_total" in text + assert "new_pixiv_token_refresh_fail_total" in text + assert "new_pixiv_jobs_status_count" in text + assert "new_pixiv_proxy_endpoints_state_count" in text + assert "new_pixiv_proxy_probe_latency_ms" in text + + assert re.search(r'new_pixiv_jobs_status_count\{status=\"pending\"\}\s+1(\.0+)?\b', text) + assert re.search(r'new_pixiv_proxy_endpoints_state_count\{state=\"enabled\"\}\s+1(\.0+)?\b', text) diff --git a/backend/tests/test_models_admin_audit.py b/backend/tests/test_models_admin_audit.py new file mode 100644 index 0000000..e75ebde --- /dev/null +++ b/backend/tests/test_models_admin_audit.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.admin_audit import AdminAudit +from app.db.models.base import Base +from app.db.session import create_sessionmaker + + +def test_models_admin_audit_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_admin_audit.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = AdminAudit(actor="admin", action="login", resource="auth", record_id="1", request_id="req_test") + session.add(row) + await session.commit() + await session.refresh(row) + assert row.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(AdminAudit))).scalars().first() + assert fetched is not None + assert fetched.action == "login" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_hydration_runs.py b/backend/tests/test_models_hydration_runs.py new file mode 100644 index 0000000..b3330ba --- /dev/null +++ b/backend/tests/test_models_hydration_runs.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.hydration_runs import HydrationRun +from app.db.session import create_sessionmaker + + +def test_models_hydration_runs_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_hydration_runs.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = HydrationRun(type="backfill", status="pending", criteria_json="{}", cursor_json="{}") + session.add(row) + await session.commit() + await session.refresh(row) + assert row.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(HydrationRun))).scalars().first() + assert fetched is not None + assert fetched.type == "backfill" + assert fetched.status == "pending" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_image_tags.py b/backend/tests/test_models_image_tags.py new file mode 100644 index 0000000..00ef57e --- /dev/null +++ b/backend/tests/test_models_image_tags.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.imports import Import +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker + + +def test_models_image_tags_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_image_tags.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + imp = Import(created_by="tester", source="manual") + session.add(imp) + await session.flush() + + img = Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/original.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + created_import_id=imp.id, + ) + session.add(img) + + tag = Tag(name="tag1") + session.add(tag) + await session.flush() + + session.add(ImageTag(image_id=img.id, tag_id=tag.id)) + await session.commit() + + async with Session() as session: + link = (await session.execute(select(ImageTag))).scalars().first() + assert link is not None + assert link.image_id == 1 + assert link.tag_id == 1 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_images.py b/backend/tests/test_models_images.py new file mode 100644 index 0000000..64abcdc --- /dev/null +++ b/backend/tests/test_models_images.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.models.imports import Import +from app.db.session import create_sessionmaker + + +def test_models_images_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_images.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + + async with Session() as session: + imp = Import(created_by="tester", source="manual") + session.add(imp) + await session.flush() + + img = Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/original.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + created_import_id=imp.id, + ) + session.add(img) + await session.commit() + await session.refresh(img) + assert img.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(Image))).scalars().first() + assert fetched is not None + assert fetched.illust_id == 1 + assert fetched.page_index == 0 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_imports.py b/backend/tests/test_models_imports.py new file mode 100644 index 0000000..3838a29 --- /dev/null +++ b/backend/tests/test_models_imports.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.imports import Import +from app.db.session import create_sessionmaker + + +def test_models_imports_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = Import(created_by="tester", source="manual", detail_json="{}") + session.add(row) + await session.commit() + await session.refresh(row) + assert row.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(Import))).scalars().first() + assert fetched is not None + assert fetched.id == 1 + assert fetched.created_by == "tester" + assert fetched.source == "manual" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_jobs.py b/backend/tests/test_models_jobs.py new file mode 100644 index 0000000..8f6e592 --- /dev/null +++ b/backend/tests/test_models_jobs.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker + + +def test_models_jobs_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_jobs.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = JobRow(type="noop", status="pending", payload_json="{}") + session.add(row) + await session.commit() + await session.refresh(row) + assert row.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(JobRow))).scalars().first() + assert fetched is not None + assert fetched.type == "noop" + assert fetched.status == "pending" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_pixiv_tokens.py b/backend/tests/test_models_pixiv_tokens.py new file mode 100644 index 0000000..8aa6ca1 --- /dev/null +++ b/backend/tests/test_models_pixiv_tokens.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.session import create_sessionmaker + + +def test_models_pixiv_tokens_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_pixiv_tokens.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = PixivToken( + label="acc1", + enabled=1, + refresh_token_enc="enc", + refresh_token_masked="***", + weight=1.0, + ) + session.add(row) + await session.commit() + await session.refresh(row) + assert row.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(PixivToken))).scalars().first() + assert fetched is not None + assert fetched.label == "acc1" + assert fetched.refresh_token_enc == "enc" + assert fetched.refresh_token_masked == "***" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_proxy_endpoints.py b/backend/tests/test_models_proxy_endpoints.py new file mode 100644 index 0000000..0a2d42b --- /dev/null +++ b/backend/tests/test_models_proxy_endpoints.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.session import create_sessionmaker + + +def test_models_proxy_endpoints_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_proxy_endpoints.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = ProxyEndpoint(scheme="http", host="1.2.3.4", port=8080, username="u", password_enc="enc", enabled=1) + session.add(row) + await session.commit() + await session.refresh(row) + assert row.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(ProxyEndpoint))).scalars().first() + assert fetched is not None + assert fetched.host == "1.2.3.4" + assert fetched.port == 8080 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_proxy_pool_endpoints.py b/backend/tests/test_models_proxy_pool_endpoints.py new file mode 100644 index 0000000..ab60f1b --- /dev/null +++ b/backend/tests/test_models_proxy_pool_endpoints.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.session import create_sessionmaker + + +def test_models_proxy_pool_endpoints_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_proxy_pool_endpoints.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + pool = ProxyPool(name="pool1", enabled=1) + session.add(pool) + + endpoint = ProxyEndpoint(scheme="http", host="1.2.3.4", port=8080, username="u", password_enc="enc", enabled=1) + session.add(endpoint) + + await session.flush() + session.add(ProxyPoolEndpoint(pool_id=pool.id, endpoint_id=endpoint.id, enabled=1, weight=1)) + await session.commit() + + async with Session() as session: + row = (await session.execute(select(ProxyPoolEndpoint))).scalars().first() + assert row is not None + assert row.pool_id == 1 + assert row.endpoint_id == 1 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_proxy_pools.py b/backend/tests/test_models_proxy_pools.py new file mode 100644 index 0000000..26ee9f6 --- /dev/null +++ b/backend/tests/test_models_proxy_pools.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.proxy_pools import ProxyPool +from app.db.session import create_sessionmaker + + +def test_models_proxy_pools_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_proxy_pools.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = ProxyPool(name="pool1", description="desc", enabled=1) + session.add(row) + await session.commit() + await session.refresh(row) + assert row.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(ProxyPool))).scalars().first() + assert fetched is not None + assert fetched.name == "pool1" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_request_logs.py b/backend/tests/test_models_request_logs.py new file mode 100644 index 0000000..9183e41 --- /dev/null +++ b/backend/tests/test_models_request_logs.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.request_logs import RequestLog +from app.db.session import create_sessionmaker + + +def test_models_request_logs_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_request_logs.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = RequestLog( + request_id="req_test", + method="GET", + route="/healthz", + status=200, + duration_ms=12, + ip="127.0.0.1", + user_agent="pytest", + sample_rate=1.0, + ) + session.add(row) + await session.commit() + await session.refresh(row) + assert row.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(RequestLog))).scalars().first() + assert fetched is not None + assert fetched.route == "/healthz" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_runtime_settings.py b/backend/tests/test_models_runtime_settings.py new file mode 100644 index 0000000..c620a32 --- /dev/null +++ b/backend/tests/test_models_runtime_settings.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.runtime_settings import RuntimeSetting +from app.db.session import create_sessionmaker + + +def test_models_runtime_settings_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_runtime_settings.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + row = RuntimeSetting(key="k1", value_json="{}", description="d", updated_by="tester") + session.add(row) + await session.commit() + + async with Session() as session: + fetched = (await session.execute(select(RuntimeSetting))).scalars().first() + assert fetched is not None + assert fetched.key == "k1" + assert fetched.value_json == "{}" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_tags.py b/backend/tests/test_models_tags.py new file mode 100644 index 0000000..c121ee9 --- /dev/null +++ b/backend/tests/test_models_tags.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker + + +def test_models_tags_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_tags.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + t = Tag(name="tag1", translated_name="标签1") + session.add(t) + await session.commit() + await session.refresh(t) + assert t.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(Tag))).scalars().first() + assert fetched is not None + assert fetched.name == "tag1" + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_models_token_proxy_bindings.py b/backend/tests/test_models_token_proxy_bindings.py new file mode 100644 index 0000000..041ce88 --- /dev/null +++ b/backend/tests/test_models_token_proxy_bindings.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from sqlalchemy import select + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.pixiv_tokens import PixivToken +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.models.token_proxy_bindings import TokenProxyBinding +from app.db.session import create_sessionmaker + + +def test_models_token_proxy_bindings_insert_select(tmp_path: Path) -> None: + db_path = tmp_path / "orm_token_proxy_bindings.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + token = PixivToken(refresh_token_enc="enc", refresh_token_masked="***", enabled=1, weight=1.0) + pool = ProxyPool(name="pool1", enabled=1) + ep1 = ProxyEndpoint(scheme="http", host="1.2.3.4", port=8080, username="u", password_enc="enc", enabled=1) + ep2 = ProxyEndpoint(scheme="http", host="1.2.3.5", port=8080, username="u", password_enc="enc", enabled=1) + session.add_all([token, pool, ep1, ep2]) + await session.flush() + + binding = TokenProxyBinding( + token_id=token.id, + pool_id=pool.id, + primary_proxy_id=ep1.id, + override_proxy_id=ep2.id, + override_expires_at="2026-02-10T00:00:00.000Z", + ) + session.add(binding) + await session.commit() + await session.refresh(binding) + assert binding.id == 1 + + async with Session() as session: + fetched = (await session.execute(select(TokenProxyBinding))).scalars().first() + assert fetched is not None + assert fetched.token_id == 1 + assert fetched.pool_id == 1 + assert fetched.primary_proxy_id == 1 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_opportunistic_hydrate.py b/backend/tests/test_opportunistic_hydrate.py new file mode 100644 index 0000000..3b8ac3e --- /dev/null +++ b/backend/tests/test_opportunistic_hydrate.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import httpx +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.main import create_app + + +def test_random_opportunistically_enqueues_hydrate_metadata_once(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "opportunistic_hydrate.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "client_id_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "client_secret_test") + + app = create_app() + + illust_id = 123 + + async def _migrate_and_seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + await conn.exec_driver_sql( + """ + INSERT INTO images (illust_id, page_index, ext, original_url, proxy_path, random_key, x_restrict) + VALUES (?, 0, 'jpg', 'https://example.com/x.jpg', '/i/1.jpg', 0.5, 0); + """.strip(), + (illust_id,), + ) + + asyncio.run(_migrate_and_seed()) + + def _count_jobs() -> int: + async def _op() -> int: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + """ + SELECT COUNT(*) FROM jobs + WHERE type='hydrate_metadata' + AND ref_type='opportunistic_hydrate' + AND ref_id=? + AND status IN ('pending','running'); + """.strip(), + (str(illust_id),), + ) + return int(result.scalar_one()) + + return asyncio.run(_op()) + + def _fetch_job_fields() -> tuple[int, int, str]: + async def _op() -> tuple[int, int, str]: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + """ + SELECT priority, status, payload_json FROM jobs + WHERE type='hydrate_metadata' AND ref_type='opportunistic_hydrate' AND ref_id=? + ORDER BY id DESC LIMIT 1; + """.strip(), + (str(illust_id),), + ) + row = result.fetchone() + assert row is not None + return (int(row[0]), 1 if str(row[1]) == "pending" else 0, str(row[2])) + + return asyncio.run(_op()) + + with TestClient(app) as client: + resp1 = client.get("/random?format=simple_json", headers={"X-Request-Id": "req_test"}) + assert resp1.status_code == 200 + body = resp1.json() + assert body["ok"] is True + assert body["data"]["image"]["illust_id"] == str(illust_id) + + assert _count_jobs() == 1 + priority, is_pending, payload_json = _fetch_job_fields() + assert priority < 0 + assert is_pending == 1 + payload = json.loads(payload_json) + assert payload["illust_id"] == illust_id + assert payload["reason"] == "random" + + with TestClient(app) as client: + resp2 = client.get("/random?format=simple_json", headers={"X-Request-Id": "req_test2"}) + assert resp2.status_code == 200 + + assert _count_jobs() == 1 + + async def _mark_running() -> None: + async with app.state.engine.begin() as conn: + await conn.exec_driver_sql( + """ + UPDATE jobs SET status='running' + WHERE type='hydrate_metadata' AND ref_type='opportunistic_hydrate' AND ref_id=?; + """.strip(), + (str(illust_id),), + ) + + asyncio.run(_mark_running()) + + with TestClient(app) as client: + resp3 = client.get("/random?format=simple_json", headers={"X-Request-Id": "req_test3"}) + assert resp3.status_code == 200 + + assert _count_jobs() == 1 + + +def test_image_proxy_opportunistically_enqueues_hydrate_metadata_once(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "opportunistic_hydrate_image_proxy.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_ID", "client_id_test") + monkeypatch.setenv("PIXIV_OAUTH_CLIENT_SECRET", "client_secret_test") + + app = create_app() + + illust_id = 123 + + async def _migrate_and_seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + await conn.exec_driver_sql( + """ + INSERT INTO images (illust_id, page_index, ext, original_url, proxy_path, random_key, x_restrict) + VALUES (?, 0, 'jpg', 'https://example.com/x.jpg', '/i/1.jpg', 0.5, 0); + """.strip(), + (illust_id,), + ) + + asyncio.run(_migrate_and_seed()) + + def handler(req: httpx.Request) -> httpx.Response: + assert req.headers.get("Referer") == "https://www.pixiv.net/" + return httpx.Response(200, headers={"Content-Type": "image/jpeg"}, content=b"img-bytes") + + app.state.httpx_transport = httpx.MockTransport(handler) + + def _count_jobs() -> int: + async def _op() -> int: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + """ + SELECT COUNT(*) FROM jobs + WHERE type='hydrate_metadata' + AND ref_type='opportunistic_hydrate' + AND ref_id=? + AND status IN ('pending','running'); + """.strip(), + (str(illust_id),), + ) + return int(result.scalar_one()) + + return asyncio.run(_op()) + + def _fetch_payload_reason() -> str: + async def _op() -> str: + async with app.state.engine.connect() as conn: + result = await conn.exec_driver_sql( + """ + SELECT payload_json FROM jobs + WHERE type='hydrate_metadata' AND ref_type='opportunistic_hydrate' AND ref_id=? + ORDER BY id DESC LIMIT 1; + """.strip(), + (str(illust_id),), + ) + row = result.fetchone() + assert row is not None + payload = json.loads(str(row[0] or "{}")) + return str(payload.get("reason") or "") + + return asyncio.run(_op()) + + with TestClient(app) as client: + resp1 = client.get("/i/1.jpg", headers={"X-Request-Id": "req_test"}) + assert resp1.status_code == 200 + assert resp1.content == b"img-bytes" + + assert _count_jobs() == 1 + assert _fetch_payload_reason() == "image_proxy" + + with TestClient(app) as client: + resp2 = client.get("/i/1.jpg", headers={"X-Request-Id": "req_test2"}) + assert resp2.status_code == 200 + + assert _count_jobs() == 1 diff --git a/backend/tests/test_pixiv_oauth.py b/backend/tests/test_pixiv_oauth.py new file mode 100644 index 0000000..46007d7 --- /dev/null +++ b/backend/tests/test_pixiv_oauth.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import asyncio +import hashlib +from urllib.parse import parse_qs + +import httpx +import pytest + +from app.pixiv.oauth import PixivOauthConfig, PixivOauthError, refresh_access_token + + +def test_refresh_access_token_success_builds_expected_request() -> None: + client_time = "2026-02-10T00:00:00+00:00" + hash_secret = "hash_secret_test" + refresh_token = "dummy_refresh_token" + + config = PixivOauthConfig( + client_id="client_id_test", + client_secret="client_secret_test", + hash_secret=hash_secret, + ) + + def handler(req: httpx.Request) -> httpx.Response: + assert req.method == "POST" + assert str(req.url) == "https://oauth.secure.pixiv.net/auth/token" + + assert req.headers.get("User-Agent") == config.user_agent + assert req.headers.get("X-Client-Time") == client_time + + expected_hash = hashlib.md5((client_time + hash_secret).encode("utf-8")).hexdigest() + assert req.headers.get("X-Client-Hash") == expected_hash + + body = (req.content or b"").decode("utf-8") + form = parse_qs(body) + assert form["grant_type"] == ["refresh_token"] + assert form["refresh_token"] == [refresh_token] + assert form["client_id"] == [config.client_id] + assert form["client_secret"] == [config.client_secret] + assert form["get_secure_url"] == ["1"] + assert form["include_policy"] == ["1"] + + return httpx.Response( + 200, + json={ + "response": { + "access_token": "access_token_test", + "token_type": "bearer", + "expires_in": 3600, + "refresh_token": "refresh_token_rotated", + "scope": "", + "user": {"id": 123}, + } + }, + ) + + token = asyncio.run( + refresh_access_token( + refresh_token=refresh_token, + config=config, + transport=httpx.MockTransport(handler), + client_time=client_time, + ) + ) + + assert token.access_token == "access_token_test" + assert token.token_type == "bearer" + assert token.expires_in == 3600 + assert token.refresh_token == "refresh_token_rotated" + assert token.scope is None + assert token.user_id == "123" + + +def test_refresh_access_token_error_does_not_echo_refresh_token() -> None: + refresh_token = "dummy_refresh_token" + + config = PixivOauthConfig( + client_id="client_id_test", + client_secret="client_secret_test", + hash_secret="hash_secret_test", + ) + + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response(400, json={"error": "invalid_grant"}) + + with pytest.raises(PixivOauthError) as excinfo: + asyncio.run( + refresh_access_token( + refresh_token=refresh_token, + config=config, + transport=httpx.MockTransport(handler), + client_time="2026-02-10T00:00:00+00:00", + ) + ) + + assert excinfo.value.status_code == 400 + assert refresh_token not in str(excinfo.value) + diff --git a/backend/tests/test_pixiv_urls.py b/backend/tests/test_pixiv_urls.py new file mode 100644 index 0000000..dcc1739 --- /dev/null +++ b/backend/tests/test_pixiv_urls.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import pytest + +from app.core.pixiv_urls import parse_pixiv_original_url + + +def test_parse_pixiv_original_url_ok() -> None: + u = "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg" + parsed = parse_pixiv_original_url(u) + assert parsed.illust_id == 12345678 + assert parsed.page_index == 0 + assert parsed.ext == "jpg" + + +def test_parse_pixiv_original_url_ok_master1200_suffix() -> None: + u = "https://i.pximg.net/img-master/img/2023/01/01/00/00/00/12345678_p1_master1200.png" + parsed = parse_pixiv_original_url(u) + assert parsed.illust_id == 12345678 + assert parsed.page_index == 1 + assert parsed.ext == "png" + + +def test_parse_pixiv_original_url_ok_ugoira0() -> None: + u = "https://i.pximg.net/img-original/img/2014/07/01/21/17/59/44439242_ugoira0.jpg" + parsed = parse_pixiv_original_url(u) + assert parsed.illust_id == 44439242 + assert parsed.page_index == 0 + assert parsed.ext == "jpg" + + +def test_parse_pixiv_original_url_ignores_query_and_whitespace() -> None: + u = " https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p12.PNG?foo=bar \n" + parsed = parse_pixiv_original_url(u) + assert parsed.illust_id == 12345678 + assert parsed.page_index == 12 + assert parsed.ext == "png" + + +def test_parse_pixiv_original_url_rejects_non_pximg() -> None: + with pytest.raises(ValueError, match="unsupported host"): + parse_pixiv_original_url("https://www.pixiv.net/artworks/12345678") + + +def test_parse_pixiv_original_url_rejects_wrong_pattern() -> None: + with pytest.raises(ValueError, match="unsupported pixiv original url"): + parse_pixiv_original_url("https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678.jpg") + + +def test_parse_pixiv_original_url_rejects_unsupported_ext() -> None: + with pytest.raises(ValueError, match="unsupported ext"): + parse_pixiv_original_url("https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.bmp") diff --git a/backend/tests/test_proxy_pool_routing.py b/backend/tests/test_proxy_pool_routing.py new file mode 100644 index 0000000..2f036c8 --- /dev/null +++ b/backend/tests/test_proxy_pool_routing.py @@ -0,0 +1,456 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest + +from app.core.errors import ApiError, ErrorCode +from app.core.proxy_routing import resolve_pool_id_for_host, select_proxy_uri_for_url, should_use_proxy_for_host +from app.core.runtime_settings import load_runtime_config +from app.db.models.base import Base +from app.db.models.proxy_endpoints import ProxyEndpoint +from app.db.models.proxy_pool_endpoints import ProxyPoolEndpoint +from app.db.models.proxy_pools import ProxyPool +from app.db.models.runtime_settings import RuntimeSetting +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_proxy_route_pools_suffix_matching_prefers_longest() -> None: + class _Runtime: + proxy_route_pools = {"pximg.net": 2, "i.pximg.net": 1} + proxy_default_pool_id = None + + pool = resolve_pool_id_for_host(_Runtime(), host="i.pximg.net") # type: ignore[arg-type] + assert pool == 1 + + +def test_select_proxy_uri_for_url_uses_configured_pool(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "proxy_pool_routing.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + pool = ProxyPool(name="pool1", description=None, enabled=1) + session.add(pool) + await session.flush() + + ep = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + source_ref=None, + ) + session.add(ep) + await session.flush() + + session.add(ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(ep.id), enabled=1, weight=3)) + + session.add_all( + [ + RuntimeSetting(key="proxy.enabled", value_json="true", description=None, updated_by=None), + RuntimeSetting(key="proxy.fail_closed", value_json="true", description=None, updated_by=None), + RuntimeSetting( + key="proxy.route_mode", + value_json=json.dumps("all", separators=(",", ":"), ensure_ascii=False), + description=None, + updated_by=None, + ), + RuntimeSetting( + key="proxy.default_pool_id", + value_json=str(int(pool.id)), + description=None, + updated_by=None, + ), + RuntimeSetting( + key="proxy.route_pools", + value_json=json.dumps({"i.pximg.net": int(pool.id)}, separators=(",", ":"), ensure_ascii=False), + description=None, + updated_by=None, + ), + ] + ) + await session.commit() + + asyncio.run(_seed()) + + async def _run() -> str: + runtime = await load_runtime_config(app.state.engine) + assert should_use_proxy_for_host(runtime, host="i.pximg.net") is True + picked = await select_proxy_uri_for_url( + app.state.engine, + app.state.settings, + runtime, + url="https://i.pximg.net/img-original/img/2020/01/01/00/00/00/12345678_p0.jpg", + ) + assert picked is not None + return picked.uri + + uri = asyncio.run(_run()) + assert uri == "http://1.2.3.4:8080" + + +def test_select_proxy_uri_for_url_falls_back_when_preferred_pool_empty(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "proxy_pool_routing_fallback.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> tuple[int, int]: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + empty_pool = ProxyPool(name="empty", description=None, enabled=1) + ok_pool = ProxyPool(name="ok", description=None, enabled=1) + session.add_all([empty_pool, ok_pool]) + await session.flush() + + ep = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + source_ref=None, + ) + session.add(ep) + await session.flush() + session.add(ProxyPoolEndpoint(pool_id=int(ok_pool.id), endpoint_id=int(ep.id), enabled=1, weight=1)) + + session.add_all( + [ + RuntimeSetting(key="proxy.enabled", value_json="true", description=None, updated_by=None), + RuntimeSetting(key="proxy.fail_closed", value_json="true", description=None, updated_by=None), + RuntimeSetting( + key="proxy.route_mode", + value_json=json.dumps("all", separators=(",", ":"), ensure_ascii=False), + description=None, + updated_by=None, + ), + RuntimeSetting( + key="proxy.default_pool_id", + value_json=str(int(empty_pool.id)), + description=None, + updated_by=None, + ), + ] + ) + await session.commit() + return int(empty_pool.id), int(ok_pool.id) + + empty_pool_id, ok_pool_id = asyncio.run(_seed()) + assert empty_pool_id > 0 + assert ok_pool_id > 0 + + async def _run() -> str: + runtime = await load_runtime_config(app.state.engine) + picked = await select_proxy_uri_for_url( + app.state.engine, + app.state.settings, + runtime, + url="https://i.pximg.net/img-original/img/2020/01/01/00/00/00/12345678_p0.jpg", + ) + assert picked is not None + return picked.uri + + uri = asyncio.run(_run()) + assert uri == "http://1.2.3.4:8080" + + +def test_select_proxy_uri_for_url_proxy_required_includes_pool_stats_when_all_blacklisted(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "proxy_pool_routing_blacklisted.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + blacklisted_until = "2999-01-01T00:00:00.000Z" + + async def _seed() -> int: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + pool = ProxyPool(name="pool1", description=None, enabled=1) + session.add(pool) + await session.flush() + + ep = ProxyEndpoint( + scheme="http", + host="1.2.3.4", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + source_ref=None, + blacklisted_until=blacklisted_until, + ) + session.add(ep) + await session.flush() + + session.add(ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(ep.id), enabled=1, weight=1)) + + session.add_all( + [ + RuntimeSetting(key="proxy.enabled", value_json="true", description=None, updated_by=None), + RuntimeSetting(key="proxy.fail_closed", value_json="true", description=None, updated_by=None), + RuntimeSetting( + key="proxy.route_mode", + value_json=json.dumps("all", separators=(",", ":"), ensure_ascii=False), + description=None, + updated_by=None, + ), + RuntimeSetting( + key="proxy.default_pool_id", + value_json=str(int(pool.id)), + description=None, + updated_by=None, + ), + ] + ) + await session.commit() + return int(pool.id) + + pool_id = asyncio.run(_seed()) + + async def _run() -> None: + runtime = await load_runtime_config(app.state.engine) + with pytest.raises(ApiError) as ei: + await select_proxy_uri_for_url( + app.state.engine, + app.state.settings, + runtime, + url="https://i.pximg.net/img-original/img/2020/01/01/00/00/00/12345678_p0.jpg", + ) + exc = ei.value + assert exc.code == ErrorCode.PROXY_REQUIRED + assert exc.status_code == 502 + assert isinstance(exc.details, dict) + assert exc.details.get("reason") == "all_endpoints_blacklisted" + assert exc.details.get("pool_id") == pool_id + assert exc.details.get("endpoints_total") == 1 + assert exc.details.get("endpoints_eligible") == 0 + assert exc.details.get("next_available_at") == blacklisted_until + + asyncio.run(_run()) + + +def test_select_proxy_uri_for_url_prefers_last_ok_over_fail(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "proxy_pool_routing_prefers_ok.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + pool = ProxyPool(name="pool1", description=None, enabled=1) + session.add(pool) + await session.flush() + + ok_ep = ProxyEndpoint( + scheme="http", + host="1.1.1.1", + port=8080, + username="", + password_enc="", + enabled=1, + source="manual", + source_ref=None, + last_ok_at="2026-02-18T00:00:00Z", + last_fail_at=None, + ) + fail_ep = ProxyEndpoint( + scheme="http", + host="2.2.2.2", + port=8081, + username="", + password_enc="", + enabled=1, + source="manual", + source_ref=None, + last_ok_at=None, + last_fail_at="2026-02-18T01:00:00Z", + ) + session.add_all([ok_ep, fail_ep]) + await session.flush() + + session.add_all( + [ + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(ok_ep.id), enabled=1, weight=1), + ProxyPoolEndpoint(pool_id=int(pool.id), endpoint_id=int(fail_ep.id), enabled=1, weight=1), + ] + ) + + session.add_all( + [ + RuntimeSetting(key="proxy.enabled", value_json="true", description=None, updated_by=None), + RuntimeSetting(key="proxy.fail_closed", value_json="true", description=None, updated_by=None), + RuntimeSetting( + key="proxy.route_mode", + value_json=json.dumps("all", separators=(",", ":"), ensure_ascii=False), + description=None, + updated_by=None, + ), + RuntimeSetting( + key="proxy.default_pool_id", + value_json=str(int(pool.id)), + description=None, + updated_by=None, + ), + ] + ) + await session.commit() + + asyncio.run(_seed()) + + async def _run() -> str: + runtime = await load_runtime_config(app.state.engine) + picked = await select_proxy_uri_for_url( + app.state.engine, + app.state.settings, + runtime, + url="https://i.pximg.net/img-original/img/2020/01/01/00/00/00/12345678_p0.jpg", + ) + assert picked is not None + return picked.uri + + uri = asyncio.run(_run()) + assert uri == "http://1.1.1.1:8080" + + +def test_select_proxy_uri_for_url_ignores_disabled_preferred_pool(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "proxy_pool_routing_disabled_preferred.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> tuple[int, int]: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + disabled_pool = ProxyPool(name="disabled", description=None, enabled=0) + enabled_pool = ProxyPool(name="enabled", description=None, enabled=1) + session.add_all([disabled_pool, enabled_pool]) + await session.flush() + + disabled_ep = ProxyEndpoint( + scheme="http", + host="1.1.1.1", + port=8000, + username="", + password_enc="", + enabled=1, + source="manual", + source_ref=None, + ) + enabled_ep = ProxyEndpoint( + scheme="http", + host="2.2.2.2", + port=8001, + username="", + password_enc="", + enabled=1, + source="manual", + source_ref=None, + ) + session.add_all([disabled_ep, enabled_ep]) + await session.flush() + + session.add_all( + [ + ProxyPoolEndpoint( + pool_id=int(disabled_pool.id), + endpoint_id=int(disabled_ep.id), + enabled=1, + weight=1, + ), + ProxyPoolEndpoint( + pool_id=int(enabled_pool.id), + endpoint_id=int(enabled_ep.id), + enabled=1, + weight=1, + ), + ] + ) + + session.add_all( + [ + RuntimeSetting(key="proxy.enabled", value_json="true", description=None, updated_by=None), + RuntimeSetting(key="proxy.fail_closed", value_json="true", description=None, updated_by=None), + RuntimeSetting( + key="proxy.route_mode", + value_json=json.dumps("all", separators=(",", ":"), ensure_ascii=False), + description=None, + updated_by=None, + ), + RuntimeSetting( + key="proxy.default_pool_id", + value_json=str(int(disabled_pool.id)), + description=None, + updated_by=None, + ), + RuntimeSetting( + key="proxy.route_pools", + value_json=json.dumps({"i.pximg.net": int(disabled_pool.id)}, separators=(",", ":"), ensure_ascii=False), + description=None, + updated_by=None, + ), + ] + ) + await session.commit() + return int(disabled_pool.id), int(enabled_pool.id) + + _disabled_id, enabled_id = asyncio.run(_seed()) + + async def _run() -> tuple[str, int]: + runtime = await load_runtime_config(app.state.engine) + picked = await select_proxy_uri_for_url( + app.state.engine, + app.state.settings, + runtime, + url="https://i.pximg.net/img-original/img/2020/01/01/00/00/00/12345678_p0.jpg", + ) + assert picked is not None + return picked.uri, int(picked.pool_id) + + uri, pool_id = asyncio.run(_run()) + assert uri == "http://2.2.2.2:8001" + assert pool_id == enabled_id diff --git a/backend/tests/test_proxy_uri.py b/backend/tests/test_proxy_uri.py new file mode 100644 index 0000000..01f35d6 --- /dev/null +++ b/backend/tests/test_proxy_uri.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import pytest + +from app.core.proxy_uri import parse_proxy_uri + + +def test_parse_proxy_uri_password_contains_at() -> None: + u = "http://inliver:inliverBAIPIAO@123@152.53.91.30:2323" + parts = parse_proxy_uri(u) + assert parts.scheme == "http" + assert parts.host == "152.53.91.30" + assert parts.port == 2323 + assert parts.username == "inliver" + assert parts.password == "inliverBAIPIAO@123" + + +def test_parse_proxy_uri_password_urlencoded() -> None: + u = "http://inliver:inliverBAIPIAO%40123@152.53.91.30:2323" + parts = parse_proxy_uri(u) + assert parts.scheme == "http" + assert parts.host == "152.53.91.30" + assert parts.port == 2323 + assert parts.username == "inliver" + assert parts.password == "inliverBAIPIAO@123" + + +def test_parse_proxy_uri_socks5_no_auth() -> None: + u = "socks5://127.0.0.1:1080" + parts = parse_proxy_uri(u) + assert parts.scheme == "socks5" + assert parts.host == "127.0.0.1" + assert parts.port == 1080 + assert parts.username is None + assert parts.password is None + + +def test_parse_proxy_uri_rejects_missing_port() -> None: + with pytest.raises(ValueError, match="missing port"): + parse_proxy_uri("http://127.0.0.1") + + +def test_parse_proxy_uri_rejects_unknown_scheme() -> None: + with pytest.raises(ValueError, match="unsupported scheme"): + parse_proxy_uri("ftp://127.0.0.1:21") diff --git a/backend/tests/test_public_api_key_required.py b/backend/tests/test_public_api_key_required.py new file mode 100644 index 0000000..574f0b4 --- /dev/null +++ b/backend/tests/test_public_api_key_required.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.api_keys import api_key_hint, hmac_sha256_hex +from app.db.models.api_keys import ApiKey +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_public_api_key_required_enforces_and_rate_limits(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "public_api_key_required.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + api_key = "k_" + ("x" * 40) + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("PUBLIC_API_KEY_REQUIRED", "true") + monkeypatch.setenv("PUBLIC_API_KEY_RPM", "1") + monkeypatch.setenv("PUBLIC_API_KEY_BURST", "1") + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + key_hash = hmac_sha256_hex(secret_key="secret_test", message=api_key) + hint = api_key_hint(api_key) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + ApiKey( + name="public1", + key_hash=key_hash, + hint=hint, + enabled=1, + ) + ) + session.add( + Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + x_restrict=0, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + healthz = client.get("/healthz") + assert healthz.status_code == 200 + + missing = client.get("/random?format=json&attempts=1") + assert missing.status_code == 401 + missing_body = missing.json() + assert missing_body["ok"] is False + assert missing_body["code"] == "UNAUTHORIZED" + + invalid = client.get("/random?format=json&attempts=1", headers={"X-API-Key": "bad"}) + assert invalid.status_code == 401 + invalid_body = invalid.json() + assert invalid_body["ok"] is False + assert invalid_body["code"] == "UNAUTHORIZED" + + ok1 = client.get("/random?format=json&attempts=1", headers={"X-API-Key": api_key}) + assert ok1.status_code == 200 + + limited = client.get("/random?format=json&attempts=1", headers={"X-API-Key": api_key}) + assert limited.status_code == 429 + limited_body = limited.json() + assert limited_body["ok"] is False + assert limited_body["code"] == "RATE_LIMITED" + diff --git a/backend/tests/test_random_attempts_retry.py b/backend/tests/test_random_attempts_retry.py new file mode 100644 index 0000000..ba20d88 --- /dev/null +++ b/backend/tests/test_random_attempts_retry.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import httpx +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_random_attempts_retries_on_upstream_failure(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_attempts_retry.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setattr("app.api.public.random.random.random", lambda: 0.0) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/bad.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/good.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + ), + ] + ) + await session.commit() + await app.state.engine.dispose() + + asyncio.run(_seed()) + + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path.endswith("/bad.jpg"): + return httpx.Response(404, request=req) + return httpx.Response(200, headers={"Content-Type": "image/jpeg"}, content=b"ok", request=req) + + app.state.httpx_transport = httpx.MockTransport(handler) + + with TestClient(app) as client: + resp = client.get("/random?attempts=2", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + assert resp.content == b"ok" + + +def test_random_attempts_exhausted_returns_502(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_attempts_exhausted.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setattr("app.api.public.random.random.random", lambda: 0.0) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/bad.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + ) + ) + await session.commit() + await app.state.engine.dispose() + + asyncio.run(_seed()) + + def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response(404, request=req) + + app.state.httpx_transport = httpx.MockTransport(handler) + + with TestClient(app) as client: + resp = client.get("/random?attempts=2", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 502 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "UPSTREAM_STREAM_ERROR" + assert body["request_id"] == "req_test" diff --git a/backend/tests/test_random_contract_seed_ai_type.py b/backend/tests/test_random_contract_seed_ai_type.py new file mode 100644 index 0000000..6efe8b2 --- /dev/null +++ b/backend/tests/test_random_contract_seed_ai_type.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_random_seed_is_deterministic_for_json(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_seed.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.2, + x_restrict=0, + ai_type=0, + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/2.jpg", + proxy_path="/i/2.jpg", + random_key=0.8, + x_restrict=0, + ai_type=0, + ), + ] + ) + await session.commit() + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + r1 = client.get("/random", params={"format": "json", "seed": "abc", "attempts": 1}) + r2 = client.get("/random", params={"format": "json", "seed": "abc", "attempts": 1}) + assert r1.status_code == 200 + assert r2.status_code == 200 + assert r1.json()["data"]["image"]["id"] == r2.json()["data"]["image"]["id"] + + +def test_random_ai_type_filter_and_simple_json_shape(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_ai_type.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=10, + page_index=0, + ext="jpg", + original_url="https://example.test/a0.jpg", + proxy_path="/i/10.jpg", + random_key=0.3, + x_restrict=0, + ai_type=0, + ), + Image( + illust_id=11, + page_index=0, + ext="jpg", + original_url="https://example.test/a1.jpg", + proxy_path="/i/11.jpg", + random_key=0.6, + x_restrict=0, + ai_type=1, + ), + ] + ) + await session.commit() + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp_ai1 = client.get("/random", params={"format": "json", "ai_type": "1", "attempts": 1}) + assert resp_ai1.status_code == 200 + assert resp_ai1.json()["data"]["image"]["ai_type"] == 1 + + resp_ai0 = client.get("/random", params={"format": "json", "ai_type": "0", "attempts": 1}) + assert resp_ai0.status_code == 200 + assert resp_ai0.json()["data"]["image"]["ai_type"] == 0 + + resp_simple = client.get("/random", params={"format": "simple_json", "attempts": 1}) + assert resp_simple.status_code == 200 + body = resp_simple.json() + assert body["ok"] is True + assert body["code"] == "OK" + assert "tags" not in body.get("data", {}) + + resp_bad = client.get("/random", params={"format": "json", "ai_type": "bad"}) + assert resp_bad.status_code == 400 + bad_body = resp_bad.json() + assert bad_body["ok"] is False + assert bad_body["code"] == "BAD_REQUEST" + diff --git a/backend/tests/test_random_created_filters.py b/backend/tests/test_random_created_filters.py new file mode 100644 index 0000000..4170d7d --- /dev/null +++ b/backend/tests/test_random_created_filters.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.random_pick import pick_random_image +from app.db.session import create_sessionmaker + + +def test_created_from_to_filters(tmp_path: Path) -> None: + db_path = tmp_path / "created_filters.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/old.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + created_at_pixiv="2024-01-01T00:00:00Z", + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/new.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + created_at_pixiv="2024-06-01T00:00:00Z", + ), + ] + ) + await session.commit() + + async with Session() as session: + picked_from = await pick_random_image(session, r=0.0, created_from="2024-05-01T00:00:00Z") + assert picked_from is not None + assert int(picked_from.illust_id) == 2 + + async with Session() as session: + picked_to = await pick_random_image(session, r=0.0, created_to="2024-02-01T00:00:00Z") + assert picked_to is not None + assert int(picked_to.illust_id) == 1 + + async with Session() as session: + picked_between = await pick_random_image( + session, + r=0.0, + created_from="2024-01-01T00:00:00Z", + created_to="2024-12-31T23:59:59Z", + ) + assert picked_between is not None + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_random_fail_cooldown.py b/backend/tests/test_random_fail_cooldown.py new file mode 100644 index 0000000..f3a8ae3 --- /dev/null +++ b/backend/tests/test_random_fail_cooldown.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import httpx +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_fail_cooldown_skips_recent_failures(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_fail_cooldown.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("RANDOM_FAIL_COOLDOWN_SECONDS", "3600") + monkeypatch.setattr("app.api.public.random.random.random", lambda: 0.0) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/bad.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/good.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + ), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + def handler(req: httpx.Request) -> httpx.Response: + if req.url.path.endswith("/bad.jpg"): + return httpx.Response(404, request=req) + return httpx.Response(200, headers={"Content-Type": "image/jpeg"}, content=b"ok", request=req) + + app.state.httpx_transport = httpx.MockTransport(handler) + + with TestClient(app) as client: + first = client.get("/random?attempts=1", headers={"X-Request-Id": "req_test"}) + assert first.status_code == 502 + + second = client.get("/random?attempts=1", headers={"X-Request-Id": "req_test"}) + assert second.status_code == 200 + assert second.content == b"ok" + + async def _assert_db() -> None: + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + bad = (await session.get(Image, 1)) + assert bad is not None + assert bad.last_fail_at is not None + assert int(bad.fail_count) >= 1 + + asyncio.run(_assert_db()) + diff --git a/backend/tests/test_random_filters.py b/backend/tests/test_random_filters.py new file mode 100644 index 0000000..e133824 --- /dev/null +++ b/backend/tests/test_random_filters.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.random_pick import pick_random_image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_pick_random_image_supports_popularity_and_illust_type_filters(tmp_path: Path) -> None: + db_path = tmp_path / "random_popularity_filters.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + illust_type=0, + bookmark_count=5, + view_count=10, + comment_count=0, + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/2.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + illust_type=1, + bookmark_count=100, + view_count=1000, + comment_count=50, + ), + ] + ) + await session.commit() + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, min_bookmarks=50) + assert picked is not None + assert int(picked.illust_id) == 2 + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, min_views=500) + assert picked is not None + assert int(picked.illust_id) == 2 + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, min_comments=10) + assert picked is not None + assert int(picked.illust_id) == 2 + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, illust_type=0) + assert picked is not None + assert int(picked.illust_id) == 1 + + await engine.dispose() + + asyncio.run(_run()) + + +def test_random_api_supports_layout_adaptive_and_new_filters(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_api_filters.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/p.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + orientation=1, + width=1000, + height=1000, + illust_type=0, + bookmark_count=10, + view_count=100, + comment_count=1, + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/l.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + orientation=2, + width=1000, + height=1000, + illust_type=1, + bookmark_count=100, + view_count=1000, + comment_count=50, + ), + ] + ) + await session.commit() + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + r_layout = client.get("/random", params={"format": "json", "layout": "portrait", "attempts": 1}) + assert r_layout.status_code == 200 + assert r_layout.json()["data"]["image"]["illust_id"] == "1" + + r_min_bookmarks = client.get("/random", params={"format": "json", "min_bookmarks": 50, "attempts": 1}) + assert r_min_bookmarks.status_code == 200 + assert r_min_bookmarks.json()["data"]["image"]["illust_id"] == "2" + + r_illust_type = client.get("/random", params={"format": "json", "illust_type": "manga", "attempts": 1}) + assert r_illust_type.status_code == 200 + assert r_illust_type.json()["data"]["image"]["illust_id"] == "2" + + headers_mobile = {"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X)"} + r_adaptive = client.get("/random", params={"format": "json", "adaptive": 1, "attempts": 1}, headers=headers_mobile) + assert r_adaptive.status_code == 200 + assert r_adaptive.json()["data"]["image"]["illust_id"] == "1" + + # adaptive 不覆盖用户显式传入的 orientation + r_adaptive_keep = client.get( + "/random", + params={"format": "json", "adaptive": 1, "orientation": "landscape", "attempts": 1}, + headers=headers_mobile, + ) + assert r_adaptive_keep.status_code == 200 + assert r_adaptive_keep.json()["data"]["image"]["illust_id"] == "2" + diff --git a/backend/tests/test_random_geometry_filters.py b/backend/tests/test_random_geometry_filters.py new file mode 100644 index 0000000..297cdfd --- /dev/null +++ b/backend/tests/test_random_geometry_filters.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.random_pick import pick_random_image +from app.db.session import create_sessionmaker + + +def test_orientation_filter(tmp_path: Path) -> None: + db_path = tmp_path / "orientation_filter.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.2, + x_restrict=0, + orientation=1, + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/2.jpg", + proxy_path="/i/2.jpg", + random_key=0.1, + x_restrict=0, + orientation=2, + ), + ] + ) + await session.commit() + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, orientation=1) + assert picked is not None + assert int(picked.illust_id) == 1 + + await engine.dispose() + + asyncio.run(_run()) + + +def test_min_width_height_pixels_filters(tmp_path: Path) -> None: + db_path = tmp_path / "min_filters.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=10, + page_index=0, + ext="jpg", + original_url="https://example.test/small.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + width=800, + height=600, + ), + Image( + illust_id=11, + page_index=0, + ext="jpg", + original_url="https://example.test/large.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + width=1920, + height=1080, + ), + ] + ) + await session.commit() + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, min_width=1000, min_height=700) + assert picked is not None + assert int(picked.illust_id) == 11 + + async with Session() as session: + picked_pixels = await pick_random_image(session, r=0.0, min_pixels=1920 * 1080) + assert picked_pixels is not None + assert int(picked_pixels.illust_id) == 11 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_random_image_mode.py b/backend/tests/test_random_image_mode.py new file mode 100644 index 0000000..5212cb8 --- /dev/null +++ b/backend/tests/test_random_image_mode.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +import httpx +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_random_image_streams_bytes(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_image.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + x_restrict=0, + ) + ) + await session.commit() + await app.state.engine.dispose() + + asyncio.run(_seed()) + + def handler(req: httpx.Request) -> httpx.Response: + assert req.headers.get("Referer") == "https://www.pixiv.net/" + return httpx.Response(200, headers={"Content-Type": "image/jpeg"}, content=b"img-bytes") + + app.state.httpx_transport = httpx.MockTransport(handler) + + with TestClient(app) as client: + resp = client.get("/random", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + assert resp.content == b"img-bytes" + assert resp.headers["Cache-Control"] == "no-store" + assert resp.headers["X-Request-Id"] == "req_test" + + +def test_random_image_redirects_to_proxy_path(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_redirect.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + image_id: int | None = None + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img = Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + x_restrict=0, + ) + session.add(img) + await session.commit() + await session.refresh(img) + nonlocal image_id + image_id = img.id + + await app.state.engine.dispose() + + asyncio.run(_seed()) + assert image_id is not None + + with TestClient(app) as client: + resp = client.get( + "/random?redirect=1", + headers={"X-Request-Id": "req_test"}, + follow_redirects=False, + ) + assert resp.status_code == 302 + assert resp.headers["Location"] == f"/i/{image_id}.jpg" + assert resp.headers["Cache-Control"] == "no-store" + assert resp.headers["X-Request-Id"] == "req_test" + + +def test_random_image_no_match_returns_404(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_image_empty.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + await app.state.engine.dispose() + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.get("/random", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 404 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "NO_MATCH" + assert body["request_id"] == "req_test" + hints = body["details"]["hints"] + assert hints["applied_filters"]["r18"] == 0 + assert isinstance(hints["suggestions"], list) + + +def test_random_json_returns_shape_and_hides_origin_by_default(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_json.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + x_restrict=0, + ) + ) + await session.commit() + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp = client.get("/random?format=json", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["code"] == "OK" + assert body["request_id"] == "req_test" + assert body["data"]["image"]["id"].isdigit() + image_id = body["data"]["image"]["id"] + assert body["data"]["urls"]["proxy"] == f"/i/{image_id}.jpg" + assert body["data"]["urls"]["origin"] is None diff --git a/backend/tests/test_random_json_tags.py b/backend/tests/test_random_json_tags.py new file mode 100644 index 0000000..092c552 --- /dev/null +++ b/backend/tests/test_random_json_tags.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_random_json_includes_tags(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_json_tags.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + tag = Tag(name="cat") + img = Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/origin.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + x_restrict=0, + ) + session.add_all([tag, img]) + await session.flush() + session.add(ImageTag(image_id=img.id, tag_id=tag.id)) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp = client.get("/random?format=json", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["data"]["tags"] == ["cat"] + diff --git a/backend/tests/test_random_key_index.py b/backend/tests/test_random_key_index.py new file mode 100644 index 0000000..369c0f3 --- /dev/null +++ b/backend/tests/test_random_key_index.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from app.db.engine import create_engine +from app.db.models.base import Base + + +def test_images_has_random_key_index(tmp_path: Path) -> None: + db_path = tmp_path / "random_key_index.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + async with engine.connect() as conn: + idx_list = (await conn.exec_driver_sql("PRAGMA index_list('images')")).mappings().all() + names = {str(r.get('name')) for r in idx_list} + assert "idx_images_filter" in names + + idx_info = (await conn.exec_driver_sql("PRAGMA index_info('idx_images_filter')")).mappings().all() + cols = [str(r.get("name")) for r in idx_info] + assert "random_key" in cols + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_random_no_match_hints.py b/backend/tests/test_random_no_match_hints.py new file mode 100644 index 0000000..14472df --- /dev/null +++ b/backend/tests/test_random_no_match_hints.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import asyncio +import re +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.main import create_app + + +_HAS_CHINESE = re.compile(r"[\u4e00-\u9fff]") + + +def test_random_no_match_hints_are_chinese(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_no_match_hints.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + await app.state.engine.dispose() + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.get("/random?format=json", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 404 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "NO_MATCH" + assert body["request_id"] == "req_test" + + msg = str(body.get("message") or "") + assert _HAS_CHINESE.search(msg), f"expected Chinese message, got: {msg!r}" + + hints = body.get("details", {}).get("hints", {}) + suggestions = hints.get("suggestions") + assert isinstance(suggestions, list) + assert suggestions + for s in suggestions: + assert isinstance(s, str) + assert _HAS_CHINESE.search(s), f"expected Chinese suggestion, got: {s!r}" + diff --git a/backend/tests/test_random_quality_strategy.py b/backend/tests/test_random_quality_strategy.py new file mode 100644 index 0000000..408ffba --- /dev/null +++ b/backend/tests/test_random_quality_strategy.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import asyncio +import random +import math +from pathlib import Path + +from fastapi.testclient import TestClient +from sqlalchemy import select + +from app.api.public.random import _quality_score # noqa: PLC2701 +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.random_pick import pick_random_image +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_random_quality_strategy_picks_weighted_by_score(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_quality_strategy.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.10, + x_restrict=0, + ai_type=0, + width=1200, + height=800, + bookmark_count=10, + view_count=100, + comment_count=1, + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/2.jpg", + proxy_path="/i/2.jpg", + random_key=0.30, + x_restrict=0, + ai_type=0, + width=800, + height=800, + bookmark_count=500, + view_count=20000, + comment_count=50, + ), + Image( + illust_id=3, + page_index=0, + ext="jpg", + original_url="https://example.test/3.jpg", + proxy_path="/i/3.jpg", + random_key=0.60, + x_restrict=0, + ai_type=0, + width=3000, + height=2000, + bookmark_count=200, + view_count=4000, + comment_count=10, + ), + Image( + illust_id=4, + page_index=0, + ext="jpg", + original_url="https://example.test/4.jpg", + proxy_path="/i/4.jpg", + random_key=0.85, + x_restrict=0, + ai_type=0, + width=900, + height=1600, + bookmark_count=30, + view_count=300, + comment_count=2, + ), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + seed = "seed_test_001" + samples = 3 + + async def _compute_expected_quality_pick() -> int: + rng = random.Random(seed) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + start = float(rng.random()) + ordered = (await session.execute(select(Image).order_by(Image.random_key.asc()))).scalars().all() + assert ordered + ordered2 = sorted(ordered, key=lambda x: float(x.random_key)) + start_idx = 0 + for i, img in enumerate(ordered2): + if float(img.random_key) >= start: + start_idx = i + break + take = min(int(samples), len(ordered2)) + picked_imgs = [ordered2[(start_idx + i) % len(ordered2)] for i in range(take)] + candidates: list[tuple[int, float]] = [(int(img.id), float(_quality_score(img))) for img in picked_imgs] + + assert candidates + max_logit = max(s for _id, s in candidates) + weights = [math.exp(float(s) - float(max_logit)) for _id, s in candidates] + total = float(sum(weights)) + assert total > 0 + + r = float(rng.random()) * total + for (img_id, _s), w in zip(candidates, weights, strict=True): + r -= float(w) + if r <= 0: + return int(img_id) + return int(candidates[-1][0]) + + expected_quality_id = asyncio.run(_compute_expected_quality_pick()) + + with TestClient(app) as client: + resp = client.get( + "/random", + params={ + "format": "json", + "attempts": 1, + "seed": seed, + "strategy": "quality", + "quality_samples": samples, + }, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert int(body["data"]["image"]["id"]) == expected_quality_id + assert body["data"]["debug"]["picked_by"] == "quality_weighted" + assert body["data"]["debug"]["quality_samples"] == samples + assert isinstance(body["data"]["debug"]["quality_score"], float) + + +def test_random_strategy_random_key_matches_pick_random_image(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_strategy_random_key.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=10, + page_index=0, + ext="jpg", + original_url="https://example.test/a.jpg", + proxy_path="/i/10.jpg", + random_key=0.20, + x_restrict=0, + ai_type=0, + ), + Image( + illust_id=11, + page_index=0, + ext="jpg", + original_url="https://example.test/b.jpg", + proxy_path="/i/11.jpg", + random_key=0.80, + x_restrict=0, + ai_type=0, + ), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + seed = "seed_test_002" + rng = random.Random(seed) + + async def _compute_expected() -> int: + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img = await pick_random_image(session, r=rng.random()) + assert img is not None + return int(img.id) + + expected_id = asyncio.run(_compute_expected()) + + with TestClient(app) as client: + resp = client.get("/random", params={"format": "simple_json", "attempts": 1, "seed": seed, "strategy": "random"}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert int(body["data"]["image"]["id"]) == expected_id + assert body["data"]["debug"]["picked_by"] == "random_key" + + +def test_random_quality_samples_allows_1000(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_quality_samples_1000.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=100, + page_index=0, + ext="jpg", + original_url="https://example.test/100.jpg", + proxy_path="/i/100.jpg", + random_key=0.20, + x_restrict=0, + ai_type=0, + width=1200, + height=800, + bookmark_count=10, + view_count=100, + comment_count=1, + ), + Image( + illust_id=101, + page_index=0, + ext="jpg", + original_url="https://example.test/101.jpg", + proxy_path="/i/101.jpg", + random_key=0.80, + x_restrict=0, + ai_type=0, + width=800, + height=800, + bookmark_count=500, + view_count=20000, + comment_count=50, + ), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp = client.get( + "/random", + params={ + "format": "json", + "attempts": 1, + "seed": "seed_quality_1000", + "strategy": "quality", + "quality_samples": 1000, + }, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["data"]["debug"]["quality_samples"] == 1000 diff --git a/backend/tests/test_random_r18_filter.py b/backend/tests/test_random_r18_filter.py new file mode 100644 index 0000000..d87f512 --- /dev/null +++ b/backend/tests/test_random_r18_filter.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.random_pick import pick_random_image +from app.db.session import create_sessionmaker + + +def test_r18_strict_excludes_null_x_restrict(tmp_path: Path) -> None: + db_path = tmp_path / "r18_strict.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add( + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.5, + x_restrict=None, + ) + ) + await session.commit() + + async with Session() as session: + strict = await pick_random_image(session, r=0.0, r18=0, r18_strict=True) + non_strict = await pick_random_image(session, r=0.0, r18=0, r18_strict=False) + assert strict is None + assert non_strict is not None + assert int(non_strict.illust_id) == 1 + + await engine.dispose() + + asyncio.run(_run()) + + +def test_r18_filter_selects_only_r18(tmp_path: Path) -> None: + db_path = tmp_path / "r18_only.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=10, + page_index=0, + ext="jpg", + original_url="https://example.test/safe.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + ), + Image( + illust_id=11, + page_index=0, + ext="jpg", + original_url="https://example.test/r18.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=1, + ), + ] + ) + await session.commit() + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, r18=1, r18_strict=True) + assert picked is not None + assert int(picked.illust_id) == 11 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_random_runtime_defaults.py b/backend/tests/test_random_runtime_defaults.py new file mode 100644 index 0000000..d36c55b --- /dev/null +++ b/backend/tests/test_random_runtime_defaults.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.models.runtime_settings import RuntimeSetting +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_random_uses_runtime_defaults_for_strategy_and_quality_samples(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_runtime_defaults.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + seed = "seed_runtime_defaults_001" + samples = 3 + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + RuntimeSetting( + key="random.defaults", + value_json=json.dumps( + {"strategy": "quality", "quality_samples": samples}, + separators=(",", ":"), + ensure_ascii=False, + ), + description=None, + updated_by=None, + ) + ) + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.10, + x_restrict=0, + ai_type=0, + width=1200, + height=800, + bookmark_count=10, + view_count=100, + comment_count=1, + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/2.jpg", + proxy_path="/i/2.jpg", + random_key=0.30, + x_restrict=0, + ai_type=0, + width=800, + height=800, + bookmark_count=500, + view_count=20000, + comment_count=50, + ), + Image( + illust_id=3, + page_index=0, + ext="jpg", + original_url="https://example.test/3.jpg", + proxy_path="/i/3.jpg", + random_key=0.60, + x_restrict=0, + ai_type=0, + width=3000, + height=2000, + bookmark_count=200, + view_count=4000, + comment_count=10, + ), + Image( + illust_id=4, + page_index=0, + ext="jpg", + original_url="https://example.test/4.jpg", + proxy_path="/i/4.jpg", + random_key=0.85, + x_restrict=0, + ai_type=0, + width=900, + height=1600, + bookmark_count=30, + view_count=300, + comment_count=2, + ), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp1 = client.get("/random", params={"format": "json", "attempts": 1, "seed": seed}) + assert resp1.status_code == 200 + body1 = resp1.json() + assert body1["ok"] is True + assert body1["data"]["debug"]["strategy"] == "quality" + assert body1["data"]["debug"]["strategy_source"] == "runtime" + assert body1["data"]["debug"]["quality_samples"] == samples + assert body1["data"]["debug"]["quality_samples_source"] == "runtime" + id1 = int(body1["data"]["image"]["id"]) + + resp2 = client.get( + "/random", + params={ + "format": "json", + "attempts": 1, + "seed": seed, + "strategy": "quality", + "quality_samples": samples, + }, + ) + assert resp2.status_code == 200 + body2 = resp2.json() + assert body2["ok"] is True + assert int(body2["data"]["image"]["id"]) == id1 + assert body2["data"]["debug"]["strategy_source"] == "query" + assert body2["data"]["debug"]["quality_samples_source"] == "query" + diff --git a/backend/tests/test_random_settings_defaults_alignment.py b/backend/tests/test_random_settings_defaults_alignment.py new file mode 100644 index 0000000..cd76627 --- /dev/null +++ b/backend/tests/test_random_settings_defaults_alignment.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.models.runtime_settings import RuntimeSetting +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_random_uses_runtime_defaults_for_attempts_r18_strict_and_fail_cooldown(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "random_runtime_defaults_alignment.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + session.add( + RuntimeSetting( + key="random.defaults", + value_json=json.dumps( + { + "default_attempts": 7, + "default_r18_strict": False, + "fail_cooldown_ms": 12345, + "strategy": "random", + }, + separators=(",", ":"), + ensure_ascii=False, + ), + description=None, + updated_by=None, + ) + ) + session.add( + Image( + illust_id=123, + page_index=0, + ext="jpg", + original_url="https://example.test/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.10, + x_restrict=None, + ) + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + resp = client.get("/random", params={"format": "json", "seed": "seed_alignment_001"}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + debug = body["data"]["debug"] + assert debug["attempts"] == 7 + assert debug["attempts_source"] == "runtime" + assert debug["r18_strict"] == 0 + assert debug["r18_strict_source"] == "runtime" + assert debug["fail_cooldown_ms"] == 12345 + assert debug["fail_cooldown_source"] == "runtime" + assert debug["strategy"] == "random" + assert debug["strategy_source"] == "runtime" + diff --git a/backend/tests/test_random_tag_filters.py b/backend/tests/test_random_tag_filters.py new file mode 100644 index 0000000..8065e0a --- /dev/null +++ b/backend/tests/test_random_tag_filters.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.random_pick import pick_random_image +from app.db.session import create_sessionmaker + + +def test_included_tags_filters_images(tmp_path: Path) -> None: + db_path = tmp_path / "included_tags.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + cat = Tag(name="cat") + dog = Tag(name="dog") + session.add_all([cat, dog]) + + img_cat = Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/cat.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + ) + img_dog = Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/dog.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + ) + session.add_all([img_cat, img_dog]) + await session.flush() + + session.add_all( + [ + ImageTag(image_id=img_cat.id, tag_id=cat.id), + ImageTag(image_id=img_dog.id, tag_id=dog.id), + ] + ) + await session.commit() + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, included_tags=["cat"]) + assert picked is not None + assert int(picked.illust_id) == 1 + + await engine.dispose() + + asyncio.run(_run()) + + +def test_included_tags_requires_all(tmp_path: Path) -> None: + db_path = tmp_path / "included_all.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + cat = Tag(name="cat") + dog = Tag(name="dog") + session.add_all([cat, dog]) + + img_cat_only = Image( + illust_id=10, + page_index=0, + ext="jpg", + original_url="https://example.test/cat_only.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + ) + img_both = Image( + illust_id=11, + page_index=0, + ext="jpg", + original_url="https://example.test/both.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + ) + session.add_all([img_cat_only, img_both]) + await session.flush() + + session.add_all( + [ + ImageTag(image_id=img_cat_only.id, tag_id=cat.id), + ImageTag(image_id=img_both.id, tag_id=cat.id), + ImageTag(image_id=img_both.id, tag_id=dog.id), + ] + ) + await session.commit() + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, included_tags=["cat", "dog"]) + assert picked is not None + assert int(picked.illust_id) == 11 + + await engine.dispose() + + asyncio.run(_run()) + + +def test_excluded_tags_filters_images(tmp_path: Path) -> None: + db_path = tmp_path / "excluded_tags.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + cat = Tag(name="cat") + dog = Tag(name="dog") + session.add_all([cat, dog]) + + img_ok = Image( + illust_id=20, + page_index=0, + ext="jpg", + original_url="https://example.test/ok.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + ) + img_bad = Image( + illust_id=21, + page_index=0, + ext="jpg", + original_url="https://example.test/bad.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + ) + session.add_all([img_ok, img_bad]) + await session.flush() + + session.add_all( + [ + ImageTag(image_id=img_ok.id, tag_id=cat.id), + ImageTag(image_id=img_bad.id, tag_id=dog.id), + ] + ) + await session.commit() + + async with Session() as session: + picked = await pick_random_image(session, r=0.0, excluded_tags=["dog"]) + assert picked is not None + assert int(picked.illust_id) == 20 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_random_user_illust_filters.py b/backend/tests/test_random_user_illust_filters.py new file mode 100644 index 0000000..b6844a2 --- /dev/null +++ b/backend/tests/test_random_user_illust_filters.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path + +from app.db.engine import create_engine +from app.db.models.base import Base +from app.db.models.images import Image +from app.db.random_pick import pick_random_image +from app.db.session import create_sessionmaker + + +def test_user_id_and_illust_id_filters(tmp_path: Path) -> None: + db_path = tmp_path / "user_illust_filters.db" + engine = create_engine("sqlite+aiosqlite:///" + db_path.as_posix()) + + async def _run() -> None: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(engine) + async with Session() as session: + session.add_all( + [ + Image( + illust_id=1, + page_index=0, + ext="jpg", + original_url="https://example.test/1.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + x_restrict=0, + user_id=100, + ), + Image( + illust_id=2, + page_index=0, + ext="jpg", + original_url="https://example.test/2.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + x_restrict=0, + user_id=101, + ), + ] + ) + await session.commit() + + async with Session() as session: + picked_user = await pick_random_image(session, r=0.0, user_id=100) + assert picked_user is not None + assert int(picked_user.illust_id) == 1 + + async with Session() as session: + picked_illust = await pick_random_image(session, r=0.0, illust_id=2) + assert picked_illust is not None + assert int(picked_illust.user_id or 0) == 101 + + await engine.dispose() + + asyncio.run(_run()) + diff --git a/backend/tests/test_refresh_backoff.py b/backend/tests/test_refresh_backoff.py new file mode 100644 index 0000000..441539e --- /dev/null +++ b/backend/tests/test_refresh_backoff.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from app.pixiv.refresh_backoff import refresh_backoff_seconds + + +def test_refresh_backoff_seconds_network_short() -> None: + assert refresh_backoff_seconds(attempt=0, status_code=None) == 0 + assert refresh_backoff_seconds(attempt=1, status_code=None) == 5 + assert refresh_backoff_seconds(attempt=2, status_code=None) == 30 + assert refresh_backoff_seconds(attempt=3, status_code=None) == 120 + + +def test_refresh_backoff_seconds_auth_long() -> None: + assert refresh_backoff_seconds(attempt=1, status_code=400) == 3600 + assert refresh_backoff_seconds(attempt=2, status_code=401) == 6 * 3600 + assert refresh_backoff_seconds(attempt=3, status_code=403) == 24 * 3600 + + +def test_refresh_backoff_seconds_auth_caps() -> None: + assert refresh_backoff_seconds(attempt=5, status_code=400) == 7 * 24 * 3600 + assert refresh_backoff_seconds(attempt=20, status_code=401) == 30 * 24 * 3600 + diff --git a/backend/tests/test_request_id.py b/backend/tests/test_request_id.py new file mode 100644 index 0000000..e096c2c --- /dev/null +++ b/backend/tests/test_request_id.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from app.core.request_id import ( + REQUEST_ID_HEADER, + get_or_create_request_id, + get_request_id_from_headers, + new_request_id, + set_request_id_header, + set_request_id_on_state, +) + + +def test_new_request_id_format() -> None: + rid = new_request_id() + assert rid.startswith("req_") + assert len(rid) > 8 + + +def test_get_request_id_from_headers() -> None: + assert get_request_id_from_headers({REQUEST_ID_HEADER: " req_1 "}) == "req_1" + assert get_request_id_from_headers({}) is None + + +def test_get_or_create_request_id_uses_header() -> None: + req = SimpleNamespace(headers={REQUEST_ID_HEADER: "req_x"}) + assert get_or_create_request_id(req) == "req_x" + + +def test_get_or_create_request_id_uses_state_when_present() -> None: + req = SimpleNamespace(headers={}, state=SimpleNamespace(request_id="req_state")) + assert get_or_create_request_id(req) == "req_state" + + +def test_set_request_id_on_state_and_header() -> None: + req = SimpleNamespace(headers={}) + res = SimpleNamespace(headers={}) + set_request_id_on_state(req, "req_abc") + set_request_id_header(res, "req_abc") + assert req.state.request_id == "req_abc" + assert res.headers[REQUEST_ID_HEADER] == "req_abc" diff --git a/backend/tests/test_sensitive_no_echo.py b/backend/tests/test_sensitive_no_echo.py new file mode 100644 index 0000000..44bf86e --- /dev/null +++ b/backend/tests/test_sensitive_no_echo.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import asyncio +import json +import logging +from pathlib import Path + +import httpx +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.db.models.base import Base +from app.main import create_app + + +def test_sensitive_refresh_token_not_in_logs_or_response(tmp_path: Path, monkeypatch, caplog) -> None: + db_path = tmp_path / "sensitive_token_create.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + refresh_token = "rt_sensitive_secret_123" + caplog.set_level(logging.INFO) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/tokens", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"label": "acc1", "refresh_token": refresh_token, "enabled": True, "weight": 1.0}, + ) + assert resp.status_code == 200 + + dumped = json.dumps(resp.json(), ensure_ascii=False) + assert refresh_token not in dumped + + assert refresh_token not in caplog.text + + +def test_sensitive_proxy_password_not_in_logs_or_response(tmp_path: Path, monkeypatch, caplog) -> None: + db_path = tmp_path / "sensitive_proxy_import.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + password = "proxy_pass_sensitive_123" + import_text = "\n".join([f"http://u:{password}@1.2.3.4:8080", "socks5://5.6.7.8:1080"]) + + caplog.set_level(logging.INFO) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxies/endpoints/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"text": import_text, "source": "manual", "conflict_policy": "skip"}, + ) + assert resp.status_code == 200 + + dumped = json.dumps(resp.json(), ensure_ascii=False) + assert password not in dumped + + assert password not in caplog.text + assert import_text not in caplog.text + + +def test_sensitive_easy_proxies_password_not_in_logs_or_response(tmp_path: Path, monkeypatch, caplog) -> None: + db_path = tmp_path / "sensitive_easy_import.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + + base_url = "http://easy-proxies.test:9090" + easy_password = "easy_admin_password_secret_sensitive" + easy_proxy_pass = "easy_proxy_pass_secret_sensitive" + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + def handler(req: httpx.Request) -> httpx.Response: + if str(req.url) == f"{base_url}/api/auth": + data = json.loads((req.content or b"{}").decode("utf-8")) + assert data["password"] == easy_password + return httpx.Response(200, json={"token": "t"}) + if str(req.url) == f"{base_url}/api/export": + return httpx.Response(200, text=f"http://u:{easy_proxy_pass}@1.2.3.4:8080\n") + return httpx.Response(404) + + app.state.httpx_transport = httpx.MockTransport(handler) + + caplog.set_level(logging.INFO) + + admin_token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app) as client: + resp = client.post( + "/admin/api/proxies/easy-proxies/import", + headers={"Authorization": f"Bearer {admin_token}", "X-Request-Id": "req_test"}, + json={"base_url": base_url, "password": easy_password, "conflict_policy": "skip_non_easy_proxies"}, + ) + assert resp.status_code == 200 + + dumped = json.dumps(resp.json(), ensure_ascii=False) + assert easy_password not in dumped + assert easy_proxy_pass not in dumped + + assert easy_password not in caplog.text + assert easy_proxy_pass not in caplog.text + diff --git a/backend/tests/test_smoke.py b/backend/tests/test_smoke.py new file mode 100644 index 0000000..984f5e5 --- /dev/null +++ b/backend/tests/test_smoke.py @@ -0,0 +1,3 @@ +def test_smoke() -> None: + assert True + diff --git a/backend/tests/test_sqlite_pragmas.py b/backend/tests/test_sqlite_pragmas.py new file mode 100644 index 0000000..9399698 --- /dev/null +++ b/backend/tests/test_sqlite_pragmas.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import asyncio +import sqlite3 +from pathlib import Path + +from app.db.engine import SQLITE_BUSY_TIMEOUT_MS, create_engine +from app.db.session import is_sqlite_busy_error, with_sqlite_busy_retry + + +def _sqlite_url(db_path: Path) -> str: + return "sqlite+aiosqlite:///" + db_path.as_posix() + + +def test_sqlite_pragmas_applied(tmp_path: Path) -> None: + db_path = tmp_path / "test.db" + engine = create_engine(_sqlite_url(db_path)) + + async def _check() -> None: + async with engine.connect() as conn: + fk = (await conn.exec_driver_sql("PRAGMA foreign_keys")).scalar_one() + assert int(fk) == 1 + + journal = (await conn.exec_driver_sql("PRAGMA journal_mode")).scalar_one() + assert str(journal).lower() == "wal" + + busy = (await conn.exec_driver_sql("PRAGMA busy_timeout")).scalar_one() + assert int(busy) == SQLITE_BUSY_TIMEOUT_MS + + sync = (await conn.exec_driver_sql("PRAGMA synchronous")).scalar_one() + assert int(sync) == 1 + + temp_store = (await conn.exec_driver_sql("PRAGMA temp_store")).scalar_one() + assert int(temp_store) == 2 + + await engine.dispose() + + asyncio.run(_check()) + + +def test_is_sqlite_busy_error() -> None: + assert is_sqlite_busy_error(sqlite3.OperationalError("database is locked")) + assert is_sqlite_busy_error(sqlite3.OperationalError("database table is locked")) + assert not is_sqlite_busy_error(ValueError("nope")) + + +def test_with_sqlite_busy_retry_retries_then_succeeds() -> None: + attempts = {"n": 0} + + async def _op() -> int: + attempts["n"] += 1 + if attempts["n"] < 3: + raise sqlite3.OperationalError("database is locked") + return 123 + + result = asyncio.run(with_sqlite_busy_retry(_op, retries=5, base_delay_s=0.0)) + assert result == 123 + assert attempts["n"] == 3 + diff --git a/backend/tests/test_tags_list.py b/backend/tests/test_tags_list.py new file mode 100644 index 0000000..bd4915a --- /dev/null +++ b/backend/tests/test_tags_list.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import asyncio +import sqlite3 +from pathlib import Path + +from fastapi.testclient import TestClient +import pytest + +from app.db.models.base import Base +from app.db.models.image_tags import ImageTag +from app.db.models.images import Image +from app.db.models.tags import Tag +from app.db.session import create_sessionmaker +from app.main import create_app + + +def test_tags_list_search_and_pagination(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "tags_list.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img1 = Image( + illust_id=100, + page_index=0, + ext="jpg", + original_url="https://example.test/100.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + ) + img2 = Image( + illust_id=101, + page_index=0, + ext="jpg", + original_url="https://example.test/101.jpg", + proxy_path="/i/2.jpg", + random_key=0.2, + ) + session.add_all([img1, img2]) + + t_cat = Tag(name="cat") + t_car = Tag(name="car") + t_dog = Tag(name="dog") + session.add_all([t_cat, t_car, t_dog]) + + await session.commit() + await session.refresh(img1) + await session.refresh(img2) + await session.refresh(t_cat) + await session.refresh(t_car) + await session.refresh(t_dog) + + session.add_all( + [ + ImageTag(image_id=int(img1.id), tag_id=int(t_car.id)), + ImageTag(image_id=int(img1.id), tag_id=int(t_cat.id)), + ImageTag(image_id=int(img2.id), tag_id=int(t_cat.id)), + ImageTag(image_id=int(img2.id), tag_id=int(t_dog.id)), + ] + ) + await session.commit() + + await app.state.engine.dispose() + + asyncio.run(_seed()) + + with TestClient(app) as client: + page1 = client.get("/tags", params={"limit": 2}, headers={"X-Request-Id": "req_test"}) + assert page1.status_code == 200 + body1 = page1.json() + assert body1["ok"] is True + assert body1["request_id"] == "req_test" + assert page1.headers["X-Request-Id"] == "req_test" + assert [i["name"] for i in body1["items"]] == ["car", "cat"] + assert body1["items"][1]["count_images"] == 2 + assert body1["next_cursor"] == "cat" + + page2 = client.get( + "/tags", + params={"limit": 2, "cursor": body1["next_cursor"]}, + headers={"X-Request-Id": "req_test"}, + ) + assert page2.status_code == 200 + body2 = page2.json() + assert [i["name"] for i in body2["items"]] == ["dog"] + assert body2["next_cursor"] == "" + + search = client.get("/tags", params={"q": "ca"}, headers={"X-Request-Id": "req_test"}) + assert search.status_code == 200 + names = [i["name"] for i in search.json()["items"]] + assert names == ["car", "cat"] + + +def test_tags_list_invalid_limit_returns_400(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "tags_list_invalid_limit.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + await app.state.engine.dispose() + + asyncio.run(_migrate()) + + with TestClient(app) as client: + resp = client.get("/tags", params={"limit": 0}, headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 400 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "BAD_REQUEST" + assert body["request_id"] == "req_test" + + +def test_tags_list_uses_fts_when_available(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "tags_list_fts.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + + app = create_app() + + async def _seed() -> bool: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + tag_id: int + Session = create_sessionmaker(app.state.engine) + async with Session() as session: + img = Image( + illust_id=100, + page_index=0, + ext="jpg", + original_url="https://example.test/100.jpg", + proxy_path="/i/1.jpg", + random_key=0.1, + ) + session.add(img) + + t_cat = Tag(name="cat", translated_name=None) + session.add(t_cat) + + await session.commit() + await session.refresh(img) + await session.refresh(t_cat) + + tag_id = int(t_cat.id) + session.add(ImageTag(image_id=int(img.id), tag_id=int(t_cat.id))) + await session.commit() + + await app.state.engine.dispose() + + try: + con = sqlite3.connect(db_path) + try: + con.execute("CREATE VIRTUAL TABLE tags_fts USING fts5(name, translated_name, tokenize='trigram');") + except sqlite3.OperationalError: + con.execute("CREATE VIRTUAL TABLE tags_fts USING fts5(name, translated_name);") + con.execute( + "INSERT INTO tags_fts(rowid, name, translated_name) VALUES (?,?,?);", + (tag_id, "neko", ""), + ) + con.commit() + return True + except sqlite3.OperationalError: + return False + finally: + try: + con.close() + except Exception: + pass + + fts_ok = asyncio.run(_seed()) + if not fts_ok: + pytest.skip("fts5 not available in sqlite3 build") + + with TestClient(app) as client: + resp = client.get("/tags", params={"q": "neko"}, headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + names = [i["name"] for i in resp.json()["items"]] + assert names == ["cat"] diff --git a/backend/tests/test_token_strategy.py b/backend/tests/test_token_strategy.py new file mode 100644 index 0000000..c2e4699 --- /dev/null +++ b/backend/tests/test_token_strategy.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import pytest + +from app.pixiv.token_strategy import NoTokenAvailable, TokenCandidate, choose_token + + +def test_round_robin_basic() -> None: + tokens = [ + TokenCandidate(id=1, enabled=True, weight=1.0, error_count=0), + TokenCandidate(id=2, enabled=True, weight=1.0, error_count=0), + TokenCandidate(id=3, enabled=True, weight=1.0, error_count=0), + ] + + t1, last = choose_token(tokens, strategy="round_robin", now=100.0, last_id=None) + assert t1.id == 1 + + t2, last = choose_token(tokens, strategy="round_robin", now=100.0, last_id=last) + assert t2.id == 2 + + t3, last = choose_token(tokens, strategy="round_robin", now=100.0, last_id=last) + assert t3.id == 3 + + t4, last = choose_token(tokens, strategy="round_robin", now=100.0, last_id=last) + assert t4.id == 1 + + +def test_round_robin_skips_backoff() -> None: + tokens = [ + TokenCandidate(id=1, enabled=True, weight=1.0, error_count=0), + TokenCandidate(id=2, enabled=True, weight=1.0, error_count=0, backoff_until=200.0), + TokenCandidate(id=3, enabled=True, weight=1.0, error_count=0), + ] + + t1, last = choose_token(tokens, strategy="round_robin", now=100.0, last_id=1) + assert t1.id == 3 + + t2, last = choose_token(tokens, strategy="round_robin", now=100.0, last_id=last) + assert t2.id == 1 + + +def test_least_error_with_round_robin_tiebreak() -> None: + tokens = [ + TokenCandidate(id=1, enabled=True, weight=1.0, error_count=2), + TokenCandidate(id=2, enabled=True, weight=1.0, error_count=0), + TokenCandidate(id=3, enabled=True, weight=1.0, error_count=0), + ] + + t1, last = choose_token(tokens, strategy="least_error", now=100.0, last_id=None) + assert t1.id == 2 + + t2, last = choose_token(tokens, strategy="least_error", now=100.0, last_id=last) + assert t2.id == 3 + + t3, last = choose_token(tokens, strategy="least_error", now=100.0, last_id=last) + assert t3.id == 2 + + +def test_weighted_choice_deterministic_r() -> None: + tokens = [ + TokenCandidate(id=1, enabled=True, weight=1.0, error_count=0), + TokenCandidate(id=2, enabled=True, weight=3.0, error_count=0), + TokenCandidate(id=3, enabled=True, weight=0.0, error_count=0), + ] + + t1, _ = choose_token(tokens, strategy="weighted", now=100.0, last_id=None, r=0.0) + assert t1.id == 1 + + t2, _ = choose_token(tokens, strategy="weighted", now=100.0, last_id=None, r=0.25) + assert t2.id == 2 + + t3, _ = choose_token(tokens, strategy="weighted", now=100.0, last_id=None, r=0.999) + assert t3.id == 2 + + +def test_weighted_falls_back_to_round_robin_when_all_zero() -> None: + tokens = [ + TokenCandidate(id=1, enabled=True, weight=0.0, error_count=0), + TokenCandidate(id=2, enabled=True, weight=0.0, error_count=0), + ] + + t1, last = choose_token(tokens, strategy="weighted", now=100.0, last_id=None, r=0.99) + assert t1.id == 1 + + t2, _ = choose_token(tokens, strategy="weighted", now=100.0, last_id=last, r=0.99) + assert t2.id == 2 + + +def test_no_token_available_reports_next_retry() -> None: + tokens = [ + TokenCandidate(id=1, enabled=True, weight=1.0, error_count=0, backoff_until=200.0), + TokenCandidate(id=2, enabled=True, weight=1.0, error_count=0, backoff_until=150.0), + ] + + with pytest.raises(NoTokenAvailable) as excinfo: + choose_token(tokens, strategy="round_robin", now=100.0, last_id=None) + assert excinfo.value.next_retry_at == 150.0 + diff --git a/backend/tests/test_unhandled_exception_json.py b/backend/tests/test_unhandled_exception_json.py new file mode 100644 index 0000000..478700d --- /dev/null +++ b/backend/tests/test_unhandled_exception_json.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi.testclient import TestClient + +from app.core.security import create_jwt +from app.main import create_app + + +def test_unhandled_exception_returns_json_with_request_id(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "unhandled_exception.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + + app = create_app() + + token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600) + with TestClient(app, raise_server_exceptions=False) as client: + resp = client.get( + "/admin/api/audit?limit=50", + headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"}, + ) + assert resp.status_code == 500 + body = resp.json() + assert body["ok"] is False + assert body["code"] == "INTERNAL_ERROR" + assert body["request_id"] == "req_test" + assert isinstance(body.get("details"), dict) + assert str(body.get("message") or "").strip() diff --git a/backend/tests/test_version.py b/backend/tests/test_version.py new file mode 100644 index 0000000..00bf3e5 --- /dev/null +++ b/backend/tests/test_version.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.api.public.version import router as version_router + + +def test_version_returns_fields(monkeypatch) -> None: + monkeypatch.setenv("APP_BUILD_TIME", "2026-02-10T00:00:00Z") + monkeypatch.setenv("APP_VERSION", "1.2.3") + monkeypatch.setenv("APP_COMMIT", "abc123") + + app = FastAPI() + app.include_router(version_router) + client = TestClient(app) + + resp = client.get("/version", headers={"X-Request-Id": "req_test"}) + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["build_time"] == "2026-02-10T00:00:00Z" + assert body["version"] == "1.2.3" + assert body["git_commit"] == "abc123" + assert body["request_id"] == "req_test" + assert resp.headers["X-Request-Id"] == "req_test" + diff --git a/backend/tests/test_worker_jobs_loop.py b/backend/tests/test_worker_jobs_loop.py new file mode 100644 index 0000000..125d5c8 --- /dev/null +++ b/backend/tests/test_worker_jobs_loop.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from cryptography.fernet import Fernet + +from app.db.models.base import Base +from app.db.models.imports import Import +from app.db.models.jobs import JobRow +from app.db.session import create_sessionmaker +from app.main import create_app +from app.worker import build_default_dispatcher, poll_and_execute_jobs + + +def test_worker_claims_and_executes_pending_job(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "worker_jobs_loop.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + field_key = Fernet.generate_key().decode("ascii") + + monkeypatch.setenv("APP_ENV", "dev") + monkeypatch.setenv("DATABASE_URL", db_url) + monkeypatch.setenv("SECRET_KEY", "secret_test") + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("FIELD_ENCRYPTION_KEY", field_key) + + app = create_app() + + async def _migrate() -> None: + async with app.state.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + asyncio.run(_migrate()) + + async def _seed_and_run() -> tuple[str, int, int]: + Session = create_sessionmaker(app.state.engine) + + async with Session() as session: + imp = Import(created_by="test", source="manual") + session.add(imp) + await session.flush() + import_id = int(imp.id) + + text = "https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg\n" + job = JobRow( + type="import_images", + status="pending", + payload_json=json.dumps( + {"import_id": import_id, "text": text}, + ensure_ascii=False, + separators=(",", ":"), + ), + ref_type="import", + ref_id=str(import_id), + ) + session.add(job) + await session.flush() + job_id = int(job.id) + await session.commit() + + dispatcher = build_default_dispatcher(app.state.engine) + ran = await poll_and_execute_jobs(app.state.engine, dispatcher, worker_id="test-worker", max_jobs=5) + + async with Session() as session: + row = await session.get(JobRow, job_id) + assert row is not None + imp2 = await session.get(Import, import_id) + assert imp2 is not None + return (str(row.status), int(imp2.success or 0), int(ran)) + + status, success, ran = asyncio.run(_seed_and_run()) + assert ran >= 1 + assert status == "completed" + assert success == 1 + diff --git a/backend/tests/test_worker_smoke.py b/backend/tests/test_worker_smoke.py new file mode 100644 index 0000000..8eb077e --- /dev/null +++ b/backend/tests/test_worker_smoke.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import asyncio + +from app.worker import main_async + + +def test_worker_smoke_exits_cleanly(monkeypatch) -> None: + monkeypatch.setenv("DATABASE_URL", "sqlite+aiosqlite:///:memory:") + asyncio.run(main_async(max_iterations=1, poll_interval_s=0.0)) + diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..d32f5ea --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,50 @@ +# This file is loaded by `deploy/docker-compose.yml` via `env_file`. +# (Recommended) For production, copy to `deploy/.env` and point compose to it. +APP_ENV=dev + +# SQLite file inside the container. Must live on the shared /app/data volume. +DATABASE_URL=sqlite+aiosqlite:///./data/app.db + +# Worker（任务执行器） +# - docker compose 默认只启动 1 个 worker 容器；worker 进程内部会并发执行多个 job。 +# - (Recommended) 默认开启“自动并发”：并发度会按启用 token 数自动增减，但会被上限限制。 +WORKER_AUTO_CONCURRENCY=true +# 自动并发刷新频率（秒）：token 数变化后多久生效 +WORKER_AUTO_CONCURRENCY_REFRESH_SECONDS=15 +# 并发上限（无论 token 有多少都不会超过这个值） +WORKER_MAX_CONCURRENCY=20 + +# Secrets (do NOT commit real values). +SECRET_KEY=dev-secret-key-change-me +# FIELD_ENCRYPTION_KEY is required in prod. In dev, if left blank, the app auto-generates +# a Fernet key and persists it to `./data/field_encryption_key`. +FIELD_ENCRYPTION_KEY= +# Alternatively, load it from a file (recommended for container/secret mounts): +# FIELD_ENCRYPTION_KEY_FILE=./data/field_encryption_key + +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin-change-me + +# Pixiv OAuth (required for token refresh + metadata hydrate jobs). +# If left blank, token refresh / hydration jobs will be disabled. +PIXIV_OAUTH_CLIENT_ID= +PIXIV_OAUTH_CLIENT_SECRET= +PIXIV_OAUTH_HASH_SECRET= + +# Optional build metadata +APP_VERSION=dev +APP_BUILD_TIME= +APP_COMMIT= + +# Optional: imgproxy (signed URL) +# IMGPROXY_BASE_URL=http://imgproxy:8080 +# IMGPROXY_KEY= +# IMGPROXY_SALT= +# IMGPROXY_MAX_DIM=2048 +# IMGPROXY_DEFAULT_OPTIONS= +# IMGPROXY_URL_CHUNK_SIZE=16 + +# Optional: public API key protection (external access) +# PUBLIC_API_KEY_REQUIRED=false +# PUBLIC_API_KEY_RPM=0 +# PUBLIC_API_KEY_BURST=0 diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..61617b6 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,44 @@ +services: + api: + build: + context: .. + dockerfile: backend/Dockerfile + env_file: + - .env + ports: + - "23222:8000" + command: + [ + "sh", + "-c", + "alembic upgrade head && exec uvicorn app.main:app --host 0.0.0.0 --port 8000", + ] + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz',timeout=2).read()", + ] + interval: 10s + timeout: 3s + retries: 10 + start_period: 10s + volumes: + - ../data:/app/data + restart: unless-stopped + + worker: + build: + context: .. + dockerfile: backend/Dockerfile + env_file: + - .env + command: ["python", "-m", "app.worker"] + depends_on: + api: + condition: service_healthy + volumes: + - ../data:/app/data + restart: unless-stopped diff --git a/frontend/.eslintrc.cjs b/frontend/.eslintrc.cjs new file mode 100644 index 0000000..f2c3eda --- /dev/null +++ b/frontend/.eslintrc.cjs @@ -0,0 +1,25 @@ +module.exports = { + root: true, + env: { + browser: true, + es2022: true, + node: true, + }, + parser: "@typescript-eslint/parser", + parserOptions: { + ecmaVersion: "latest", + sourceType: "module", + }, + plugins: ["@typescript-eslint", "react-hooks", "react-refresh"], + extends: [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:react-hooks/recommended", + ], + ignorePatterns: ["dist", "node_modules"], + rules: { + "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], + "react-refresh/only-export-components": ["warn", { "allowConstantExport": true }] + }, +}; + diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..f879f92 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,12 @@ + + + + + + 随机图片管理后台 + + +
+ + + diff --git a/frontend/package-lock.json b/frontend/package-lock.json new file mode 100644 index 0000000..519e1a1 --- /dev/null +++ b/frontend/package-lock.json @@ -0,0 +1,5973 @@ +{ + "name": "random-mage-frontend", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "random-mage-frontend", + "version": "0.1.0", + "dependencies": { + "@tanstack/react-query": "^5.59.0", + "antd": "^5.21.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^6.26.2" + }, + "devDependencies": { + "@testing-library/jest-dom": "^6.6.3", + "@testing-library/react": "^16.0.1", + "@types/node": "^22.9.3", + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@typescript-eslint/eslint-plugin": "^7.18.0", + "@typescript-eslint/parser": "^7.18.0", + "@vitejs/plugin-react": "^4.3.1", + "eslint": "^8.57.1", + "eslint-plugin-react-hooks": "^4.6.2", + "eslint-plugin-react-refresh": "^0.4.12", + "jsdom": "^25.0.1", + "typescript": "^5.6.3", + "vite": "^5.4.10", + "vitest": "^2.1.3" + } + }, + "node_modules/@adobe/css-tools": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.4.4.tgz", + "integrity": "sha512-Elp+iwUx5rN5+Y8xLt5/GRoG20WGoDCQ/1Fb+1LiGtvwbDavuSk0jhD/eZdckHAuzcDzccnkv+rEjyWfRx18gg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@ant-design/colors": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/@ant-design/colors/-/colors-7.2.1.tgz", + "integrity": "sha512-lCHDcEzieu4GA3n8ELeZ5VQ8pKQAWcGGLRTQ50aQM2iqPpq2evTxER84jfdPvsPAtEcZ7m44NI45edFMo8oOYQ==", + "license": "MIT", + "dependencies": { + "@ant-design/fast-color": "^2.0.6" + } + }, + "node_modules/@ant-design/cssinjs": { + "version": "1.24.0", + "resolved": "https://registry.npmjs.org/@ant-design/cssinjs/-/cssinjs-1.24.0.tgz", + "integrity": "sha512-K4cYrJBsgvL+IoozUXYjbT6LHHNt+19a9zkvpBPxLjFHas1UpPM2A5MlhROb0BT8N8WoavM5VsP9MeSeNK/3mg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.11.1", + "@emotion/hash": "^0.8.0", + "@emotion/unitless": "^0.7.5", + "classnames": "^2.3.1", + "csstype": "^3.1.3", + "rc-util": "^5.35.0", + "stylis": "^4.3.4" + }, + "peerDependencies": { + "react": ">=16.0.0", + "react-dom": ">=16.0.0" + } + }, + "node_modules/@ant-design/cssinjs-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@ant-design/cssinjs-utils/-/cssinjs-utils-1.1.3.tgz", + "integrity": "sha512-nOoQMLW1l+xR1Co8NFVYiP8pZp3VjIIzqV6D6ShYF2ljtdwWJn5WSsH+7kvCktXL/yhEtWURKOfH5Xz/gzlwsg==", + "license": "MIT", + "dependencies": { + "@ant-design/cssinjs": "^1.21.0", + "@babel/runtime": "^7.23.2", + "rc-util": "^5.38.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/@ant-design/fast-color": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@ant-design/fast-color/-/fast-color-2.0.6.tgz", + "integrity": "sha512-y2217gk4NqL35giHl72o6Zzqji9O7vHh9YmhUVkPtAOpoTCH4uWxo/pr4VE8t0+ChEPs0qo4eJRC5Q1eXWo3vA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.24.7" + }, + "engines": { + "node": ">=8.x" + } + }, + "node_modules/@ant-design/icons": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/@ant-design/icons/-/icons-5.6.1.tgz", + "integrity": "sha512-0/xS39c91WjPAZOWsvi1//zjx6kAp4kxWwctR6kuU6p133w8RU0D2dSCvZC19uQyharg/sAvYxGYWl01BbZZfg==", + "license": "MIT", + "dependencies": { + "@ant-design/colors": "^7.0.0", + "@ant-design/icons-svg": "^4.4.0", + "@babel/runtime": "^7.24.8", + "classnames": "^2.2.6", + "rc-util": "^5.31.1" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "react": ">=16.0.0", + "react-dom": ">=16.0.0" + } + }, + "node_modules/@ant-design/icons-svg": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@ant-design/icons-svg/-/icons-svg-4.4.2.tgz", + "integrity": "sha512-vHbT+zJEVzllwP+CM+ul7reTEfBR0vgxFe7+lREAsAA7YGsYpboiq2sQNeQeRvh09GfQgs/GyFEvZpJ9cLXpXA==", + "license": "MIT" + }, + "node_modules/@ant-design/react-slick": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@ant-design/react-slick/-/react-slick-1.1.2.tgz", + "integrity": "sha512-EzlvzE6xQUBrZuuhSAFTdsr4P2bBBHGZwKFemEfq8gIGyIQCxalYfZW/T2ORbtQx5rU69o+WycP3exY/7T1hGA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.4", + "classnames": "^2.2.5", + "json2mq": "^0.2.0", + "resize-observer-polyfill": "^1.5.1", + "throttle-debounce": "^5.0.0" + }, + "peerDependencies": { + "react": ">=16.9.0" + } + }, + "node_modules/@asamuzakjp/css-color": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", + "integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^2.1.3", + "@csstools/css-color-parser": "^3.0.9", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "lru-cache": "^10.4.3" + } + }, + "node_modules/@asamuzakjp/css-color/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/@babel/code-frame": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.0.tgz", + "integrity": "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", + "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.6", + "@babel/helpers": "^7.28.6", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/traverse": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", + "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", + "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.28.6", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", + "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", + "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.28.6", + "@babel/helper-validator-identifier": "^7.28.5", + "@babel/traverse": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", + "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.6.tgz", + "integrity": "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.0.tgz", + "integrity": "sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.27.1.tgz", + "integrity": "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.27.1.tgz", + "integrity": "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz", + "integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", + "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.28.6", + "@babel/parser": "^7.28.6", + "@babel/types": "^7.28.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", + "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.0", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.29.0", + "@babel/template": "^7.28.6", + "@babel/types": "^7.29.0", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@csstools/color-helpers": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz", + "integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz", + "integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz", + "integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^5.1.0", + "@csstools/css-calc": "^2.1.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz", + "integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "peer": true, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", + "integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@emotion/hash": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.8.0.tgz", + "integrity": "sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow==", + "license": "MIT" + }, + "node_modules/@emotion/unitless": { + "version": "0.7.5", + "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.7.5.tgz", + "integrity": "sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg==", + "license": "MIT" + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@eslint/eslintrc/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/@eslint/js": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", + "deprecated": "Use @eslint/config-array instead", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanwhocodes/object-schema": "^2.0.3", + "debug": "^4.3.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/config-array/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@humanwhocodes/config-array/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", + "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", + "deprecated": "Use @eslint/object-schema instead", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@rc-component/async-validator": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@rc-component/async-validator/-/async-validator-5.1.0.tgz", + "integrity": "sha512-n4HcR5siNUXRX23nDizbZBQPO0ZM/5oTtmKZ6/eqL0L2bo747cklFdZGRN2f+c9qWGICwDzrhW0H7tE9PptdcA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.24.4" + }, + "engines": { + "node": ">=14.x" + } + }, + "node_modules/@rc-component/color-picker": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@rc-component/color-picker/-/color-picker-2.0.1.tgz", + "integrity": "sha512-WcZYwAThV/b2GISQ8F+7650r5ZZJ043E57aVBFkQ+kSY4C6wdofXgB0hBx+GPGpIU0Z81eETNoDUJMr7oy/P8Q==", + "license": "MIT", + "dependencies": { + "@ant-design/fast-color": "^2.0.6", + "@babel/runtime": "^7.23.6", + "classnames": "^2.2.6", + "rc-util": "^5.38.1" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/@rc-component/context": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@rc-component/context/-/context-1.4.0.tgz", + "integrity": "sha512-kFcNxg9oLRMoL3qki0OMxK+7g5mypjgaaJp/pkOis/6rVxma9nJBF/8kCIuTYHUQNr0ii7MxqE33wirPZLJQ2w==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "rc-util": "^5.27.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/@rc-component/mini-decimal": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rc-component/mini-decimal/-/mini-decimal-1.1.0.tgz", + "integrity": "sha512-jS4E7T9Li2GuYwI6PyiVXmxTiM6b07rlD9Ge8uGZSCz3WlzcG5ZK7g5bbuKNeZ9pgUuPK/5guV781ujdVpm4HQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.0" + }, + "engines": { + "node": ">=8.x" + } + }, + "node_modules/@rc-component/mutate-observer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rc-component/mutate-observer/-/mutate-observer-1.1.0.tgz", + "integrity": "sha512-QjrOsDXQusNwGZPf4/qRQasg7UFEj06XiCJ8iuiq/Io7CrHrgVi6Uuetw60WAMG1799v+aM8kyc+1L/GBbHSlw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.0", + "classnames": "^2.3.2", + "rc-util": "^5.24.4" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/@rc-component/portal": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@rc-component/portal/-/portal-1.1.2.tgz", + "integrity": "sha512-6f813C0IsasTZms08kfA8kPAGxbbkYToa8ALaiDIGGECU4i9hj8Plgbx0sNJDrey3EtHO30hmdaxtT0138xZcg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.0", + "classnames": "^2.3.2", + "rc-util": "^5.24.4" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/@rc-component/qrcode": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@rc-component/qrcode/-/qrcode-1.1.1.tgz", + "integrity": "sha512-LfLGNymzKdUPjXUbRP+xOhIWY4jQ+YMj5MmWAcgcAq1Ij8XP7tRmAXqyuv96XvLUBE/5cA8hLFl9eO1JQMujrA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.24.7" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/@rc-component/tour": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@rc-component/tour/-/tour-1.15.1.tgz", + "integrity": "sha512-Tr2t7J1DKZUpfJuDZWHxyxWpfmj8EZrqSgyMZ+BCdvKZ6r1UDsfU46M/iWAAFBy961Ssfom2kv5f3UcjIL2CmQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.0", + "@rc-component/portal": "^1.0.0-9", + "@rc-component/trigger": "^2.0.0", + "classnames": "^2.3.2", + "rc-util": "^5.24.4" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/@rc-component/trigger": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rc-component/trigger/-/trigger-2.3.1.tgz", + "integrity": "sha512-ORENF39PeXTzM+gQEshuk460Z8N4+6DkjpxlpE7Q3gYy1iBpLrx0FOJz3h62ryrJZ/3zCAUIkT1Pb/8hHWpb3A==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.23.2", + "@rc-component/portal": "^1.1.0", + "classnames": "^2.3.2", + "rc-motion": "^2.0.0", + "rc-resize-observer": "^1.3.1", + "rc-util": "^5.44.0" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/@remix-run/router": { + "version": "1.23.2", + "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.2.tgz", + "integrity": "sha512-Ic6m2U/rMjTkhERIa/0ZtXJP17QUi2CbWE7cqx4J58M8aA3QTfW+2UlQ4psvTX9IO1RfNVhK3pcpdjej7L+t2w==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", + "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz", + "integrity": "sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.57.1.tgz", + "integrity": "sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.57.1.tgz", + "integrity": "sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.57.1.tgz", + "integrity": "sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.57.1.tgz", + "integrity": "sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.57.1.tgz", + "integrity": "sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.57.1.tgz", + "integrity": "sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.57.1.tgz", + "integrity": "sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.57.1.tgz", + "integrity": "sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.57.1.tgz", + "integrity": "sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.57.1.tgz", + "integrity": "sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.57.1.tgz", + "integrity": "sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.57.1.tgz", + "integrity": "sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.57.1.tgz", + "integrity": "sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.57.1.tgz", + "integrity": "sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.57.1.tgz", + "integrity": "sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.57.1.tgz", + "integrity": "sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.57.1.tgz", + "integrity": "sha512-ABca4ceT4N+Tv/GtotnWAeXZUZuM/9AQyCyKYyKnpk4yoA7QIAuBt6Hkgpw8kActYlew2mvckXkvx0FfoInnLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.57.1.tgz", + "integrity": "sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.57.1.tgz", + "integrity": "sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.57.1.tgz", + "integrity": "sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.57.1.tgz", + "integrity": "sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.57.1.tgz", + "integrity": "sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.57.1.tgz", + "integrity": "sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.57.1.tgz", + "integrity": "sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@tanstack/query-core": { + "version": "5.90.20", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.90.20.tgz", + "integrity": "sha512-OMD2HLpNouXEfZJWcKeVKUgQ5n+n3A2JFmBaScpNDUqSrQSjiveC7dKMe53uJUg1nDG16ttFPz2xfilz6i2uVg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@tanstack/react-query": { + "version": "5.90.21", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.90.21.tgz", + "integrity": "sha512-0Lu6y5t+tvlTJMTO7oh5NSpJfpg/5D41LlThfepTixPYkJ0sE2Jj0m0f6yYqujBwIXlId87e234+MxG3D3g7kg==", + "license": "MIT", + "dependencies": { + "@tanstack/query-core": "5.90.20" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^18 || ^19" + } + }, + "node_modules/@testing-library/dom": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", + "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/runtime": "^7.12.5", + "@types/aria-query": "^5.0.1", + "aria-query": "5.3.0", + "dom-accessibility-api": "^0.5.9", + "lz-string": "^1.5.0", + "picocolors": "1.1.1", + "pretty-format": "^27.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@testing-library/jest-dom": { + "version": "6.9.1", + "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz", + "integrity": "sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@adobe/css-tools": "^4.4.0", + "aria-query": "^5.0.0", + "css.escape": "^1.5.1", + "dom-accessibility-api": "^0.6.3", + "picocolors": "^1.1.1", + "redent": "^3.0.0" + }, + "engines": { + "node": ">=14", + "npm": ">=6", + "yarn": ">=1" + } + }, + "node_modules/@testing-library/jest-dom/node_modules/dom-accessibility-api": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz", + "integrity": "sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@testing-library/react": { + "version": "16.3.2", + "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.3.2.tgz", + "integrity": "sha512-XU5/SytQM+ykqMnAnvB2umaJNIOsLF3PVv//1Ew4CTcpz0/BRyy/af40qqrt7SjKpDdT1saBMc42CUok5gaw+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@testing-library/dom": "^10.0.0", + "@types/react": "^18.0.0 || ^19.0.0", + "@types/react-dom": "^18.0.0 || ^19.0.0", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@types/aria-query": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", + "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.19.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.11.tgz", + "integrity": "sha512-BH7YwL6rA93ReqeQS1c4bsPpcfOmJasG+Fkr6Y59q83f9M1WcBRHR2vM+P9eOisYRcN3ujQoiZY8uk5W+1WL8w==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.28", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.28.tgz", + "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.18.0.tgz", + "integrity": "sha512-94EQTWZ40mzBc42ATNIBimBEDltSJ9RQHCC8vc/PDbxi4k8dVwUAv4o98dk50M1zB+JGFxp43FP7f8+FP8R6Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.10.0", + "@typescript-eslint/scope-manager": "7.18.0", + "@typescript-eslint/type-utils": "7.18.0", + "@typescript-eslint/utils": "7.18.0", + "@typescript-eslint/visitor-keys": "7.18.0", + "graphemer": "^1.4.0", + "ignore": "^5.3.1", + "natural-compare": "^1.4.0", + "ts-api-utils": "^1.3.0" + }, + "engines": { + "node": "^18.18.0 || >=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^7.0.0", + "eslint": "^8.56.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.18.0.tgz", + "integrity": "sha512-4Z+L8I2OqhZV8qA132M4wNL30ypZGYOQVBfMgxDH/K5UX0PNqTu1c6za9ST5r9+tavvHiTWmBnKzpCJ/GlVFtg==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "@typescript-eslint/scope-manager": "7.18.0", + "@typescript-eslint/types": "7.18.0", + "@typescript-eslint/typescript-estree": "7.18.0", + "@typescript-eslint/visitor-keys": "7.18.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || >=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.56.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.18.0.tgz", + "integrity": "sha512-jjhdIE/FPF2B7Z1uzc6i3oWKbGcHb87Qw7AWj6jmEqNOfDFbJWtjt/XfwCpvNkpGWlcJaog5vTR+VV8+w9JflA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "7.18.0", + "@typescript-eslint/visitor-keys": "7.18.0" + }, + "engines": { + "node": "^18.18.0 || >=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.18.0.tgz", + "integrity": "sha512-XL0FJXuCLaDuX2sYqZUUSOJ2sG5/i1AAze+axqmLnSkNEVMVYLF+cbwlB2w8D1tinFuSikHmFta+P+HOofrLeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/typescript-estree": "7.18.0", + "@typescript-eslint/utils": "7.18.0", + "debug": "^4.3.4", + "ts-api-utils": "^1.3.0" + }, + "engines": { + "node": "^18.18.0 || >=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.56.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/types": { + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.18.0.tgz", + "integrity": "sha512-iZqi+Ds1y4EDYUtlOOC+aUmxnE9xS/yCigkjA7XpTKV6nCBd3Hp/PRGGmdwnfkV2ThMyYldP1wRpm/id99spTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || >=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.18.0.tgz", + "integrity": "sha512-aP1v/BSPnnyhMHts8cf1qQ6Q1IFwwRvAQGRvBFkWlo3/lH29OXA3Pts+c10nxRxIBrDnoMqzhgdwVe5f2D6OzA==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@typescript-eslint/types": "7.18.0", + "@typescript-eslint/visitor-keys": "7.18.0", + "debug": "^4.3.4", + "globby": "^11.1.0", + "is-glob": "^4.0.3", + "minimatch": "^9.0.4", + "semver": "^7.6.0", + "ts-api-utils": "^1.3.0" + }, + "engines": { + "node": "^18.18.0 || >=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.18.0.tgz", + "integrity": "sha512-kK0/rNa2j74XuHVcoCZxdFBMF+aq/vH83CXAOHieC+2Gis4mF8jJXT5eAfyD3K0sAxtPuwxaIOIOvhwzVDt/kw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.4.0", + "@typescript-eslint/scope-manager": "7.18.0", + "@typescript-eslint/types": "7.18.0", + "@typescript-eslint/typescript-estree": "7.18.0" + }, + "engines": { + "node": "^18.18.0 || >=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.56.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "7.18.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.18.0.tgz", + "integrity": "sha512-cDF0/Gf81QpY3xYyJKDV14Zwdmid5+uuENhjH2EqFaF0ni+yAyq/LzMaIJdhNJXZI7uLzwIlA+V7oWoyn6Curg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "7.18.0", + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^18.18.0 || >=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "dev": true, + "license": "ISC" + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", + "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.27", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/@vitest/expect": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz", + "integrity": "sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", + "chai": "^5.1.2", + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.9.tgz", + "integrity": "sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "2.1.9", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.12" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.9.tgz", + "integrity": "sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.9.tgz", + "integrity": "sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "2.1.9", + "pathe": "^1.1.2" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.9.tgz", + "integrity": "sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "2.1.9", + "magic-string": "^0.30.12", + "pathe": "^1.1.2" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.9.tgz", + "integrity": "sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^3.0.2" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.9.tgz", + "integrity": "sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "2.1.9", + "loupe": "^3.1.2", + "tinyrainbow": "^1.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/antd": { + "version": "5.29.3", + "resolved": "https://registry.npmjs.org/antd/-/antd-5.29.3.tgz", + "integrity": "sha512-3DdbGCa9tWAJGcCJ6rzR8EJFsv2CtyEbkVabZE14pfgUHfCicWCj0/QzQVLDYg8CPfQk9BH7fHCoTXHTy7MP/A==", + "license": "MIT", + "dependencies": { + "@ant-design/colors": "^7.2.1", + "@ant-design/cssinjs": "^1.23.0", + "@ant-design/cssinjs-utils": "^1.1.3", + "@ant-design/fast-color": "^2.0.6", + "@ant-design/icons": "^5.6.1", + "@ant-design/react-slick": "~1.1.2", + "@babel/runtime": "^7.26.0", + "@rc-component/color-picker": "~2.0.1", + "@rc-component/mutate-observer": "^1.1.0", + "@rc-component/qrcode": "~1.1.0", + "@rc-component/tour": "~1.15.1", + "@rc-component/trigger": "^2.3.0", + "classnames": "^2.5.1", + "copy-to-clipboard": "^3.3.3", + "dayjs": "^1.11.11", + "rc-cascader": "~3.34.0", + "rc-checkbox": "~3.5.0", + "rc-collapse": "~3.9.0", + "rc-dialog": "~9.6.0", + "rc-drawer": "~7.3.0", + "rc-dropdown": "~4.2.1", + "rc-field-form": "~2.7.1", + "rc-image": "~7.12.0", + "rc-input": "~1.8.0", + "rc-input-number": "~9.5.0", + "rc-mentions": "~2.20.0", + "rc-menu": "~9.16.1", + "rc-motion": "^2.9.5", + "rc-notification": "~5.6.4", + "rc-pagination": "~5.1.0", + "rc-picker": "~4.11.3", + "rc-progress": "~4.0.0", + "rc-rate": "~2.13.1", + "rc-resize-observer": "^1.4.3", + "rc-segmented": "~2.7.0", + "rc-select": "~14.16.8", + "rc-slider": "~11.1.9", + "rc-steps": "~6.0.1", + "rc-switch": "~4.1.0", + "rc-table": "~7.54.0", + "rc-tabs": "~15.7.0", + "rc-textarea": "~1.10.2", + "rc-tooltip": "~6.4.0", + "rc-tree": "~5.13.1", + "rc-tree-select": "~5.27.0", + "rc-upload": "~4.11.0", + "rc-util": "^5.44.4", + "scroll-into-view-if-needed": "^3.1.0", + "throttle-debounce": "^5.0.2" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ant-design" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/aria-query": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", + "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "dequal": "^2.0.3" + } + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.9.19", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz", + "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001769", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001769.tgz", + "integrity": "sha512-BCfFL1sHijQlBGWBMuJyhZUhzo7wer5sVj9hqekB/7xn0Ypy+pER/edCYQm4exbXj4WiySGp40P8UuTh6w1srg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/classnames": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz", + "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==", + "license": "MIT" + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/compute-scroll-into-view": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/compute-scroll-into-view/-/compute-scroll-into-view-3.1.1.tgz", + "integrity": "sha512-VRhuHOLoKYOy4UbilLbUzbYg93XLjv2PncJC50EuTWPA3gaja1UjBsUP/D/9/juV3vQFr6XBEzn9KCAHdUvOHw==", + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/copy-to-clipboard": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz", + "integrity": "sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==", + "license": "MIT", + "dependencies": { + "toggle-selection": "^1.0.6" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/css.escape": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz", + "integrity": "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cssstyle": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz", + "integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@asamuzakjp/css-color": "^3.2.0", + "rrweb-cssom": "^0.8.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/cssstyle/node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "dev": true, + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "license": "MIT" + }, + "node_modules/data-urls": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", + "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/dayjs": { + "version": "1.11.19", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.19.tgz", + "integrity": "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw==", + "license": "MIT", + "peer": true + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "dev": true, + "license": "MIT" + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/dom-accessibility-api": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", + "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", + "dev": true, + "license": "MIT" + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.286", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.286.tgz", + "integrity": "sha512-9tfDXhJ4RKFNerfjdCcZfufu49vg620741MNs26a9+bhLThdB+plgMeou98CAaHu/WATj2iHOOHTp1hWtABj2A==", + "dev": true, + "license": "ISC" + }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-plugin-react-hooks": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.2.tgz", + "integrity": "sha512-QzliNJq4GinDBcD8gPB5v0wh6g8q3SUi6EFF0x8N/BL9PoVs0atuGc47ozMRyOWAKdwaZ5OnbOEa3WR+dSGKuQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0" + } + }, + "node_modules/eslint-plugin-react-refresh": { + "version": "0.4.26", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.4.26.tgz", + "integrity": "sha512-1RETEylht2O6FM/MvgnyvT+8K21wLqDNg4qD51Zj3guhjt433XbnnkVttHMyaVyAFD03QSV4LPS5iE3VQmO7XQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "eslint": ">=8.40" + } + }, + "node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/eslint/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "dev": true, + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", + "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "dev": true, + "license": "ISC" + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "dev": true, + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true, + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", + "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-encoding": "^3.1.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsdom": { + "version": "25.0.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-25.0.1.tgz", + "integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "cssstyle": "^4.1.0", + "data-urls": "^5.0.0", + "decimal.js": "^10.4.3", + "form-data": "^4.0.0", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.5", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.12", + "parse5": "^7.1.2", + "rrweb-cssom": "^0.7.1", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^5.0.0", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^2.11.2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/json2mq": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/json2mq/-/json2mq-0.2.0.tgz", + "integrity": "sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==", + "license": "MIT", + "dependencies": { + "string-convert": "^0.2.0" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/lz-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", + "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", + "dev": true, + "license": "MIT", + "bin": { + "lz-string": "bin/bin.js" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/min-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", + "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nwsapi": { + "version": "2.2.23", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.23.tgz", + "integrity": "sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", + "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/pretty-format": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", + "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1", + "ansi-styles": "^5.0.0", + "react-is": "^17.0.1" + }, + "engines": { + "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/rc-cascader": { + "version": "3.34.0", + "resolved": "https://registry.npmjs.org/rc-cascader/-/rc-cascader-3.34.0.tgz", + "integrity": "sha512-KpXypcvju9ptjW9FaN2NFcA2QH9E9LHKq169Y0eWtH4e/wHQ5Wh5qZakAgvb8EKZ736WZ3B0zLLOBsrsja5Dag==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.25.7", + "classnames": "^2.3.1", + "rc-select": "~14.16.2", + "rc-tree": "~5.13.0", + "rc-util": "^5.43.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-checkbox": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/rc-checkbox/-/rc-checkbox-3.5.0.tgz", + "integrity": "sha512-aOAQc3E98HteIIsSqm6Xk2FPKIER6+5vyEFMZfo73TqM+VVAIqOkHoPjgKLqSNtVLWScoaM7vY2ZrGEheI79yg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "classnames": "^2.3.2", + "rc-util": "^5.25.2" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-collapse": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/rc-collapse/-/rc-collapse-3.9.0.tgz", + "integrity": "sha512-swDdz4QZ4dFTo4RAUMLL50qP0EY62N2kvmk2We5xYdRwcRn8WcYtuetCJpwpaCbUfUt5+huLpVxhvmnK+PHrkA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "classnames": "2.x", + "rc-motion": "^2.3.4", + "rc-util": "^5.27.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-dialog": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/rc-dialog/-/rc-dialog-9.6.0.tgz", + "integrity": "sha512-ApoVi9Z8PaCQg6FsUzS8yvBEQy0ZL2PkuvAgrmohPkN3okps5WZ5WQWPc1RNuiOKaAYv8B97ACdsFU5LizzCqg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "@rc-component/portal": "^1.0.0-8", + "classnames": "^2.2.6", + "rc-motion": "^2.3.0", + "rc-util": "^5.21.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-drawer": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/rc-drawer/-/rc-drawer-7.3.0.tgz", + "integrity": "sha512-DX6CIgiBWNpJIMGFO8BAISFkxiuKitoizooj4BDyee8/SnBn0zwO2FHrNDpqqepj0E/TFTDpmEBCyFuTgC7MOg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.23.9", + "@rc-component/portal": "^1.1.1", + "classnames": "^2.2.6", + "rc-motion": "^2.6.1", + "rc-util": "^5.38.1" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-dropdown": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/rc-dropdown/-/rc-dropdown-4.2.1.tgz", + "integrity": "sha512-YDAlXsPv3I1n42dv1JpdM7wJ+gSUBfeyPK59ZpBD9jQhK9jVuxpjj3NmWQHOBceA1zEPVX84T2wbdb2SD0UjmA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@rc-component/trigger": "^2.0.0", + "classnames": "^2.2.6", + "rc-util": "^5.44.1" + }, + "peerDependencies": { + "react": ">=16.11.0", + "react-dom": ">=16.11.0" + } + }, + "node_modules/rc-field-form": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rc-field-form/-/rc-field-form-2.7.1.tgz", + "integrity": "sha512-vKeSifSJ6HoLaAB+B8aq/Qgm8a3dyxROzCtKNCsBQgiverpc4kWDQihoUwzUj+zNWJOykwSY4dNX3QrGwtVb9A==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.0", + "@rc-component/async-validator": "^5.0.3", + "rc-util": "^5.32.2" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-image": { + "version": "7.12.0", + "resolved": "https://registry.npmjs.org/rc-image/-/rc-image-7.12.0.tgz", + "integrity": "sha512-cZ3HTyyckPnNnUb9/DRqduqzLfrQRyi+CdHjdqgsyDpI3Ln5UX1kXnAhPBSJj9pVRzwRFgqkN7p9b6HBDjmu/Q==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.11.2", + "@rc-component/portal": "^1.0.2", + "classnames": "^2.2.6", + "rc-dialog": "~9.6.0", + "rc-motion": "^2.6.2", + "rc-util": "^5.34.1" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-input": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/rc-input/-/rc-input-1.8.0.tgz", + "integrity": "sha512-KXvaTbX+7ha8a/k+eg6SYRVERK0NddX8QX7a7AnRvUa/rEH0CNMlpcBzBkhI0wp2C8C4HlMoYl8TImSN+fuHKA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.11.1", + "classnames": "^2.2.1", + "rc-util": "^5.18.1" + }, + "peerDependencies": { + "react": ">=16.0.0", + "react-dom": ">=16.0.0" + } + }, + "node_modules/rc-input-number": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/rc-input-number/-/rc-input-number-9.5.0.tgz", + "integrity": "sha512-bKaEvB5tHebUURAEXw35LDcnRZLq3x1k7GxfAqBMzmpHkDGzjAtnUL8y4y5N15rIFIg5IJgwr211jInl3cipag==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "@rc-component/mini-decimal": "^1.0.1", + "classnames": "^2.2.5", + "rc-input": "~1.8.0", + "rc-util": "^5.40.1" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-mentions": { + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/rc-mentions/-/rc-mentions-2.20.0.tgz", + "integrity": "sha512-w8HCMZEh3f0nR8ZEd466ATqmXFCMGMN5UFCzEUL0bM/nGw/wOS2GgRzKBcm19K++jDyuWCOJOdgcKGXU3fXfbQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.22.5", + "@rc-component/trigger": "^2.0.0", + "classnames": "^2.2.6", + "rc-input": "~1.8.0", + "rc-menu": "~9.16.0", + "rc-textarea": "~1.10.0", + "rc-util": "^5.34.1" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-menu": { + "version": "9.16.1", + "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-9.16.1.tgz", + "integrity": "sha512-ghHx6/6Dvp+fw8CJhDUHFHDJ84hJE3BXNCzSgLdmNiFErWSOaZNsihDAsKq9ByTALo/xkNIwtDFGIl6r+RPXBg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "@rc-component/trigger": "^2.0.0", + "classnames": "2.x", + "rc-motion": "^2.4.3", + "rc-overflow": "^1.3.1", + "rc-util": "^5.27.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-motion": { + "version": "2.9.5", + "resolved": "https://registry.npmjs.org/rc-motion/-/rc-motion-2.9.5.tgz", + "integrity": "sha512-w+XTUrfh7ArbYEd2582uDrEhmBHwK1ZENJiSJVb7uRxdE7qJSYjbO2eksRXmndqyKqKoYPc9ClpPh5242mV1vA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.11.1", + "classnames": "^2.2.1", + "rc-util": "^5.44.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-notification": { + "version": "5.6.4", + "resolved": "https://registry.npmjs.org/rc-notification/-/rc-notification-5.6.4.tgz", + "integrity": "sha512-KcS4O6B4qzM3KH7lkwOB7ooLPZ4b6J+VMmQgT51VZCeEcmghdeR4IrMcFq0LG+RPdnbe/ArT086tGM8Snimgiw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "classnames": "2.x", + "rc-motion": "^2.9.0", + "rc-util": "^5.20.1" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-overflow": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/rc-overflow/-/rc-overflow-1.5.0.tgz", + "integrity": "sha512-Lm/v9h0LymeUYJf0x39OveU52InkdRXqnn2aYXfWmo8WdOonIKB2kfau+GF0fWq6jPgtdO9yMqveGcK6aIhJmg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.11.1", + "classnames": "^2.2.1", + "rc-resize-observer": "^1.0.0", + "rc-util": "^5.37.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-pagination": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/rc-pagination/-/rc-pagination-5.1.0.tgz", + "integrity": "sha512-8416Yip/+eclTFdHXLKTxZvn70duYVGTvUUWbckCCZoIl3jagqke3GLsFrMs0bsQBikiYpZLD9206Ej4SOdOXQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "classnames": "^2.3.2", + "rc-util": "^5.38.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-picker": { + "version": "4.11.3", + "resolved": "https://registry.npmjs.org/rc-picker/-/rc-picker-4.11.3.tgz", + "integrity": "sha512-MJ5teb7FlNE0NFHTncxXQ62Y5lytq6sh5nUw0iH8OkHL/TjARSEvSHpr940pWgjGANpjCwyMdvsEV55l5tYNSg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.24.7", + "@rc-component/trigger": "^2.0.0", + "classnames": "^2.2.1", + "rc-overflow": "^1.3.2", + "rc-resize-observer": "^1.4.0", + "rc-util": "^5.43.0" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "date-fns": ">= 2.x", + "dayjs": ">= 1.x", + "luxon": ">= 3.x", + "moment": ">= 2.x", + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + }, + "peerDependenciesMeta": { + "date-fns": { + "optional": true + }, + "dayjs": { + "optional": true + }, + "luxon": { + "optional": true + }, + "moment": { + "optional": true + } + } + }, + "node_modules/rc-progress": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/rc-progress/-/rc-progress-4.0.0.tgz", + "integrity": "sha512-oofVMMafOCokIUIBnZLNcOZFsABaUw8PPrf1/y0ZBvKZNpOiu5h4AO9vv11Sw0p4Hb3D0yGWuEattcQGtNJ/aw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "classnames": "^2.2.6", + "rc-util": "^5.16.1" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-rate": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/rc-rate/-/rc-rate-2.13.1.tgz", + "integrity": "sha512-QUhQ9ivQ8Gy7mtMZPAjLbxBt5y9GRp65VcUyGUMF3N3fhiftivPHdpuDIaWIMOTEprAjZPC08bls1dQB+I1F2Q==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "classnames": "^2.2.5", + "rc-util": "^5.0.1" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-resize-observer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/rc-resize-observer/-/rc-resize-observer-1.4.3.tgz", + "integrity": "sha512-YZLjUbyIWox8E9i9C3Tm7ia+W7euPItNWSPX5sCcQTYbnwDb5uNpnLHQCG1f22oZWUhLw4Mv2tFmeWe68CDQRQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.20.7", + "classnames": "^2.2.1", + "rc-util": "^5.44.1", + "resize-observer-polyfill": "^1.5.1" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-segmented": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rc-segmented/-/rc-segmented-2.7.1.tgz", + "integrity": "sha512-izj1Nw/Dw2Vb7EVr+D/E9lUTkBe+kKC+SAFSU9zqr7WV2W5Ktaa9Gc7cB2jTqgk8GROJayltaec+DBlYKc6d+g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.11.1", + "classnames": "^2.2.1", + "rc-motion": "^2.4.4", + "rc-util": "^5.17.0" + }, + "peerDependencies": { + "react": ">=16.0.0", + "react-dom": ">=16.0.0" + } + }, + "node_modules/rc-select": { + "version": "14.16.8", + "resolved": "https://registry.npmjs.org/rc-select/-/rc-select-14.16.8.tgz", + "integrity": "sha512-NOV5BZa1wZrsdkKaiK7LHRuo5ZjZYMDxPP6/1+09+FB4KoNi8jcG1ZqLE3AVCxEsYMBe65OBx71wFoHRTP3LRg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "@rc-component/trigger": "^2.1.1", + "classnames": "2.x", + "rc-motion": "^2.0.1", + "rc-overflow": "^1.3.1", + "rc-util": "^5.16.1", + "rc-virtual-list": "^3.5.2" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": "*", + "react-dom": "*" + } + }, + "node_modules/rc-slider": { + "version": "11.1.9", + "resolved": "https://registry.npmjs.org/rc-slider/-/rc-slider-11.1.9.tgz", + "integrity": "sha512-h8IknhzSh3FEM9u8ivkskh+Ef4Yo4JRIY2nj7MrH6GQmrwV6mcpJf5/4KgH5JaVI1H3E52yCdpOlVyGZIeph5A==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "classnames": "^2.2.5", + "rc-util": "^5.36.0" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-steps": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/rc-steps/-/rc-steps-6.0.1.tgz", + "integrity": "sha512-lKHL+Sny0SeHkQKKDJlAjV5oZ8DwCdS2hFhAkIjuQt1/pB81M0cA0ErVFdHq9+jmPmFw1vJB2F5NBzFXLJxV+g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.16.7", + "classnames": "^2.2.3", + "rc-util": "^5.16.1" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-switch": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/rc-switch/-/rc-switch-4.1.0.tgz", + "integrity": "sha512-TI8ufP2Az9oEbvyCeVE4+90PDSljGyuwix3fV58p7HV2o4wBnVToEyomJRVyTaZeqNPAp+vqeo4Wnj5u0ZZQBg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.21.0", + "classnames": "^2.2.1", + "rc-util": "^5.30.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-table": { + "version": "7.54.0", + "resolved": "https://registry.npmjs.org/rc-table/-/rc-table-7.54.0.tgz", + "integrity": "sha512-/wDTkki6wBTjwylwAGjpLKYklKo9YgjZwAU77+7ME5mBoS32Q4nAwoqhA2lSge6fobLW3Tap6uc5xfwaL2p0Sw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "@rc-component/context": "^1.4.0", + "classnames": "^2.2.5", + "rc-resize-observer": "^1.1.0", + "rc-util": "^5.44.3", + "rc-virtual-list": "^3.14.2" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-tabs": { + "version": "15.7.0", + "resolved": "https://registry.npmjs.org/rc-tabs/-/rc-tabs-15.7.0.tgz", + "integrity": "sha512-ZepiE+6fmozYdWf/9gVp7k56PKHB1YYoDsKeQA1CBlJ/POIhjkcYiv0AGP0w2Jhzftd3AVvZP/K+V+Lpi2ankA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.11.2", + "classnames": "2.x", + "rc-dropdown": "~4.2.0", + "rc-menu": "~9.16.0", + "rc-motion": "^2.6.2", + "rc-resize-observer": "^1.0.0", + "rc-util": "^5.34.1" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-textarea": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/rc-textarea/-/rc-textarea-1.10.2.tgz", + "integrity": "sha512-HfaeXiaSlpiSp0I/pvWpecFEHpVysZ9tpDLNkxQbMvMz6gsr7aVZ7FpWP9kt4t7DB+jJXesYS0us1uPZnlRnwQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "classnames": "^2.2.1", + "rc-input": "~1.8.0", + "rc-resize-observer": "^1.0.0", + "rc-util": "^5.27.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-tooltip": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/rc-tooltip/-/rc-tooltip-6.4.0.tgz", + "integrity": "sha512-kqyivim5cp8I5RkHmpsp1Nn/Wk+1oeloMv9c7LXNgDxUpGm+RbXJGL+OPvDlcRnx9DBeOe4wyOIl4OKUERyH1g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.11.2", + "@rc-component/trigger": "^2.0.0", + "classnames": "^2.3.1", + "rc-util": "^5.44.3" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-tree": { + "version": "5.13.1", + "resolved": "https://registry.npmjs.org/rc-tree/-/rc-tree-5.13.1.tgz", + "integrity": "sha512-FNhIefhftobCdUJshO7M8uZTA9F4OPGVXqGfZkkD/5soDeOhwO06T/aKTrg0WD8gRg/pyfq+ql3aMymLHCTC4A==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.1", + "classnames": "2.x", + "rc-motion": "^2.0.1", + "rc-util": "^5.16.1", + "rc-virtual-list": "^3.5.1" + }, + "engines": { + "node": ">=10.x" + }, + "peerDependencies": { + "react": "*", + "react-dom": "*" + } + }, + "node_modules/rc-tree-select": { + "version": "5.27.0", + "resolved": "https://registry.npmjs.org/rc-tree-select/-/rc-tree-select-5.27.0.tgz", + "integrity": "sha512-2qTBTzwIT7LRI1o7zLyrCzmo5tQanmyGbSaGTIf7sYimCklAToVVfpMC6OAldSKolcnjorBYPNSKQqJmN3TCww==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.25.7", + "classnames": "2.x", + "rc-select": "~14.16.2", + "rc-tree": "~5.13.0", + "rc-util": "^5.43.0" + }, + "peerDependencies": { + "react": "*", + "react-dom": "*" + } + }, + "node_modules/rc-upload": { + "version": "4.11.0", + "resolved": "https://registry.npmjs.org/rc-upload/-/rc-upload-4.11.0.tgz", + "integrity": "sha512-ZUyT//2JAehfHzjWowqROcwYJKnZkIUGWaTE/VogVrepSl7AFNbQf4+zGfX4zl9Vrj/Jm8scLO0R6UlPDKK4wA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "classnames": "^2.2.5", + "rc-util": "^5.2.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-util": { + "version": "5.44.4", + "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.44.4.tgz", + "integrity": "sha512-resueRJzmHG9Q6rI/DfK6Kdv9/Lfls05vzMs1Sk3M2P+3cJa+MakaZyWY8IPfehVuhPJFKrIY1IK4GqbiaiY5w==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "react-is": "^18.2.0" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/rc-util/node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, + "node_modules/rc-virtual-list": { + "version": "3.19.2", + "resolved": "https://registry.npmjs.org/rc-virtual-list/-/rc-virtual-list-3.19.2.tgz", + "integrity": "sha512-Ys6NcjwGkuwkeaWBDqfI3xWuZ7rDiQXlH1o2zLfFzATfEgXcqpk8CkgMfbJD81McqjcJVez25a3kPxCR807evA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.20.0", + "classnames": "^2.2.6", + "rc-resize-observer": "^1.0.0", + "rc-util": "^5.36.0" + }, + "engines": { + "node": ">=8.x" + }, + "peerDependencies": { + "react": ">=16.9.0", + "react-dom": ">=16.9.0" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "peer": true, + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", + "dev": true, + "license": "MIT" + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-router": { + "version": "6.30.3", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.3.tgz", + "integrity": "sha512-XRnlbKMTmktBkjCLE8/XcZFlnHvr2Ltdr1eJX4idL55/9BbORzyZEaIkBFDhFGCEWBBItsVrDxwx3gnisMitdw==", + "license": "MIT", + "dependencies": { + "@remix-run/router": "1.23.2" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "react": ">=16.8" + } + }, + "node_modules/react-router-dom": { + "version": "6.30.3", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.3.tgz", + "integrity": "sha512-pxPcv1AczD4vso7G4Z3TKcvlxK7g7TNt3/FNGMhfqyntocvYKj+GCatfigGDjbLozC4baguJ0ReCigoDJXb0ag==", + "license": "MIT", + "dependencies": { + "@remix-run/router": "1.23.2", + "react-router": "6.30.3" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "react": ">=16.8", + "react-dom": ">=16.8" + } + }, + "node_modules/redent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", + "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "indent-string": "^4.0.0", + "strip-indent": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resize-observer-polyfill": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz", + "integrity": "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==", + "license": "MIT" + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rollup": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.57.1.tgz", + "integrity": "sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.57.1", + "@rollup/rollup-android-arm64": "4.57.1", + "@rollup/rollup-darwin-arm64": "4.57.1", + "@rollup/rollup-darwin-x64": "4.57.1", + "@rollup/rollup-freebsd-arm64": "4.57.1", + "@rollup/rollup-freebsd-x64": "4.57.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.57.1", + "@rollup/rollup-linux-arm-musleabihf": "4.57.1", + "@rollup/rollup-linux-arm64-gnu": "4.57.1", + "@rollup/rollup-linux-arm64-musl": "4.57.1", + "@rollup/rollup-linux-loong64-gnu": "4.57.1", + "@rollup/rollup-linux-loong64-musl": "4.57.1", + "@rollup/rollup-linux-ppc64-gnu": "4.57.1", + "@rollup/rollup-linux-ppc64-musl": "4.57.1", + "@rollup/rollup-linux-riscv64-gnu": "4.57.1", + "@rollup/rollup-linux-riscv64-musl": "4.57.1", + "@rollup/rollup-linux-s390x-gnu": "4.57.1", + "@rollup/rollup-linux-x64-gnu": "4.57.1", + "@rollup/rollup-linux-x64-musl": "4.57.1", + "@rollup/rollup-openbsd-x64": "4.57.1", + "@rollup/rollup-openharmony-arm64": "4.57.1", + "@rollup/rollup-win32-arm64-msvc": "4.57.1", + "@rollup/rollup-win32-ia32-msvc": "4.57.1", + "@rollup/rollup-win32-x64-gnu": "4.57.1", + "@rollup/rollup-win32-x64-msvc": "4.57.1", + "fsevents": "~2.3.2" + } + }, + "node_modules/rrweb-cssom": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.1.tgz", + "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==", + "dev": true, + "license": "MIT" + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "dev": true, + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/scroll-into-view-if-needed": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/scroll-into-view-if-needed/-/scroll-into-view-if-needed-3.1.0.tgz", + "integrity": "sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ==", + "license": "MIT", + "dependencies": { + "compute-scroll-into-view": "^3.0.2" + } + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-convert": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", + "integrity": "sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==", + "license": "MIT" + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-indent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", + "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "min-indent": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/stylis": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.3.6.tgz", + "integrity": "sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ==", + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true, + "license": "MIT" + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true, + "license": "MIT" + }, + "node_modules/throttle-debounce": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-5.0.2.tgz", + "integrity": "sha512-B71/4oyj61iNH0KeCamLuE2rmKuTO5byTOSVwECM5FA7TiAiAW+UqTKZ9ERueC4qvgSttUhdmq1mXC3kJqGX7A==", + "license": "MIT", + "engines": { + "node": ">=12.22" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-1.2.0.tgz", + "integrity": "sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-3.0.2.tgz", + "integrity": "sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tldts": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", + "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tldts-core": "^6.1.86" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", + "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toggle-selection": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz", + "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==", + "license": "MIT" + }, + "node_modules/tough-cookie": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", + "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^6.1.32" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/ts-api-utils": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz", + "integrity": "sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "typescript": ">=4.2.0" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.9.tgz", + "integrity": "sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.3.7", + "es-module-lexer": "^1.5.4", + "pathe": "^1.1.2", + "vite": "^5.0.0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vitest": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.9.tgz", + "integrity": "sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "2.1.9", + "@vitest/mocker": "2.1.9", + "@vitest/pretty-format": "^2.1.9", + "@vitest/runner": "2.1.9", + "@vitest/snapshot": "2.1.9", + "@vitest/spy": "2.1.9", + "@vitest/utils": "2.1.9", + "chai": "^5.1.2", + "debug": "^4.3.7", + "expect-type": "^1.1.0", + "magic-string": "^0.30.12", + "pathe": "^1.1.2", + "std-env": "^3.8.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.1", + "tinypool": "^1.0.1", + "tinyrainbow": "^1.2.0", + "vite": "^5.0.0", + "vite-node": "2.1.9", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/node": "^18.0.0 || >=20.0.0", + "@vitest/browser": "2.1.9", + "@vitest/ui": "2.1.9", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", + "dev": true, + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", + "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "dev": true, + "license": "MIT" + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..865c9f7 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,38 @@ +{ + "name": "random-mage-frontend", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "lint": "eslint .", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@tanstack/react-query": "^5.59.0", + "antd": "^5.21.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^6.26.2" + }, + "devDependencies": { + "@testing-library/jest-dom": "^6.6.3", + "@testing-library/react": "^16.0.1", + "@types/node": "^22.9.3", + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@typescript-eslint/eslint-plugin": "^7.18.0", + "@typescript-eslint/parser": "^7.18.0", + "@vitejs/plugin-react": "^4.3.1", + "eslint": "^8.57.1", + "eslint-plugin-react-hooks": "^4.6.2", + "eslint-plugin-react-refresh": "^0.4.12", + "jsdom": "^25.0.1", + "typescript": "^5.6.3", + "vite": "^5.4.10", + "vitest": "^2.1.3" + } +} diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts new file mode 100644 index 0000000..d082dc7 --- /dev/null +++ b/frontend/src/api/client.ts @@ -0,0 +1,152 @@ +import { clearAdminToken, getAdminToken } from "../auth/tokenStorage"; + +export type ApiErrorBody = { + ok: false; + code: string; + message: string; + request_id: string; + details: Record; +}; + +export class ApiError extends Error { + status: number; + body: ApiErrorBody | null; + + constructor(message: string, options: { status: number; body?: ApiErrorBody | null }) { + super(message); + this.name = "ApiError"; + this.status = options.status; + this.body = options.body ?? null; + } +} + +function containsChinese(text: string): boolean { + return /[\u4e00-\u9fff]/.test(text); +} + +function translateErrorCode(code: string): string | null { + switch (code) { + case "BAD_REQUEST": + return "请求参数错误"; + case "UNAUTHORIZED": + return "未登录或登录已失效"; + case "FORBIDDEN": + return "无权限"; + case "NOT_FOUND": + return "资源不存在"; + case "RATE_LIMITED": + return "请求过于频繁，请稍后再试"; + case "INTERNAL_ERROR": + return "服务器内部错误"; + case "NO_MATCH": + return "没有匹配的图片"; + case "UPSTREAM_STREAM_ERROR": + return "上游图片流错误"; + case "UPSTREAM_403": + return "上游拒绝访问（403）"; + case "UPSTREAM_404": + return "上游资源不存在（404）"; + case "UPSTREAM_RATE_LIMIT": + return "上游限流，请稍后再试"; + case "INVALID_UPLOAD_TYPE": + return "不支持的上传类型"; + case "PAYLOAD_TOO_LARGE": + return "上传内容过大"; + case "UNSUPPORTED_URL": + return "不支持的 URL"; + case "TOKEN_REFRESH_FAILED": + return "令牌刷新失败"; + case "TOKEN_BACKOFF": + return "令牌退避中，请稍后再试"; + case "NO_TOKEN_AVAILABLE": + return "没有可用令牌"; + case "PROXY_REQUIRED": + return "必须启用代理"; + case "PROXY_AUTH_FAILED": + return "代理认证失败"; + case "PROXY_CONNECT_FAILED": + return "代理连接失败"; + default: + return null; + } +} + +export function formatApiErrorMessage(status: number, body: ApiErrorBody | null): string { + const bodyCode = body && typeof body.code === "string" ? String(body.code).trim() : ""; + const rawMessage = body && typeof body.message === "string" ? body.message.trim() : ""; + const translated = bodyCode ? translateErrorCode(bodyCode) : null; + + if (translated) { + return rawMessage && containsChinese(rawMessage) ? rawMessage : translated; + } + return rawMessage || `HTTP ${status}`; +} + +function getApiBaseUrl(): string { + const raw = (import.meta.env.VITE_API_BASE_URL || "").trim(); + return raw.endsWith("/") ? raw.slice(0, -1) : raw; +} + +export async function apiFetch(path: string, init?: RequestInit): Promise { + const base = getApiBaseUrl(); + const url = base ? base + path : path; + + const headers = new Headers(init?.headers || {}); + const token = getAdminToken(); + if (token) headers.set("Authorization", `Bearer ${token}`); + const body = init?.body; + const isFormData = + typeof FormData !== "undefined" && body && body instanceof FormData; + if (!headers.has("Content-Type") && body && !isFormData) { + headers.set("Content-Type", "application/json"); + } + + return fetch(url, { ...init, headers }); +} + +export async function apiJson(path: string, init?: RequestInit): Promise { + const resp = await apiFetch(path, init); + const text = await resp.text(); + const contentType = resp.headers.get("content-type") || ""; + const isJson = contentType.toLowerCase().includes("application/json"); + + let data: unknown = null; + if (text && isJson) { + try { + data = JSON.parse(text); + } catch { + data = null; + } + } + + if (resp.ok) { + if (!isJson) throw new ApiError("Response is not JSON", { status: resp.status }); + return data as T; + } + + const body = data && typeof data === "object" ? (data as ApiErrorBody) : null; + + if ( + resp.status === 401 && + path.startsWith("/admin/api/") && + !path.startsWith("/admin/api/login") + ) { + try { + clearAdminToken(); + } catch { + // ignore + } + try { + window.dispatchEvent( + new CustomEvent("admin:unauthorized", { + detail: { path, request_id: body?.request_id ? String(body.request_id) : null }, + }), + ); + } catch { + // ignore + } + } + + const msg = formatApiErrorMessage(resp.status, body); + throw new ApiError(msg, { status: resp.status, body }); +} diff --git a/frontend/src/app/AdminLayout.tsx b/frontend/src/app/AdminLayout.tsx new file mode 100644 index 0000000..b20acd3 --- /dev/null +++ b/frontend/src/app/AdminLayout.tsx @@ -0,0 +1,135 @@ +import { useQuery } from "@tanstack/react-query"; +import { Button, Layout, Menu, Popover, Space, Typography } from "antd"; +import React from "react"; +import { Outlet, useLocation, useNavigate } from "react-router-dom"; + +import { apiJson } from "../api/client"; + +type NavItem = { key: string; label: string; external?: boolean; href?: string }; + +const NAV_ITEMS: NavItem[] = [ + { key: "/admin", label: "主页" }, + { key: "docs", label: "使用文档", external: true, href: "/docs" }, + { key: "/admin/import", label: "导入链接" }, + { key: "/admin/images", label: "图片管理" }, + { key: "/admin/hydration", label: "补全管理" }, + { key: "/admin/jobs", label: "任务队列" }, + { key: "/admin/tokens", label: "令牌管理" }, + { key: "/admin/proxies", label: "代理节点" }, + { key: "/admin/proxy-pools", label: "代理池" }, + { key: "/admin/bindings", label: "绑定关系" }, + { key: "/admin/settings", label: "系统设置" }, + { key: "/admin/audit", label: "审计日志" }, + { key: "/admin/recommendation", label: "推荐策略" }, + { key: "/admin/random", label: "随机调试" }, +]; + +function pickSelectedKey(pathname: string): string { + const path = String(pathname || "").trim() || "/admin"; + const candidates = [...NAV_ITEMS].sort((a, b) => b.key.length - a.key.length); + for (const item of candidates) { + if (path === item.key) return item.key; + if (path.startsWith(item.key + "/")) return item.key; + } + return "/admin"; +} + +type VersionResponse = { + ok: true; + version: string; + build_time: string; + git_commit: string; + request_id: string; +}; + +export function AdminLayout() { + const navigate = useNavigate(); + const location = useLocation(); + const isRoot = location.pathname === "/admin" || location.pathname === "/admin/"; + const selectedKey = pickSelectedKey(location.pathname); + + const version = useQuery({ + queryKey: ["public", "version"], + queryFn: () => apiJson("/version"), + staleTime: 60_000, + refetchOnWindowFocus: false, + }); + + const versionText = (() => { + if (version.isLoading) return "版本: 加载中…"; + if (version.isError || !version.data) return "版本: （获取失败）"; + const commitRaw = String(version.data.git_commit || "").trim(); + const commitShort = commitRaw ? commitRaw.slice(0, 7) : ""; + const ver = String(version.data.version || "").trim() || "（未知）"; + return commitShort ? `版本: ${ver} (${commitShort})` : `版本: ${ver}`; + })(); + + return ( + + +
+ navigate("/admin")}> + 随机图片管理后台 + + 一站式管理面板 +
+ ({ key: it.key, label: it.label }))} + onClick={(e) => { + const key = String(e.key || ""); + const item = NAV_ITEMS.find((x) => x.key === key); + if (item?.external && item.href) { + window.location.assign(String(item.href)); + return; + } + navigate(key); + }} + /> + + + + +
+ + {!isRoot ? ( + + ) : null} + + + + + {versionText} + +
如果你更新了代码但页面还是旧的，通常是因为没有重建镜像。
+
+ 请在部署目录执行：docker compose up -d --build +
+
+ {version.data?.request_id ? ( + 请求ID: {version.data.request_id} + ) : null} +
+
+ } + > + + + + + + + + + + + ); +} diff --git a/frontend/src/app/App.tsx b/frontend/src/app/App.tsx new file mode 100644 index 0000000..12d5e62 --- /dev/null +++ b/frontend/src/app/App.tsx @@ -0,0 +1,130 @@ +import { ConfigProvider } from "antd"; +import zhCN from "antd/locale/zh_CN"; +import React, { useEffect } from "react"; +import { BrowserRouter, Navigate, Route, Routes, useLocation, useNavigate } from "react-router-dom"; + +import { AdminLayout } from "./AdminLayout"; +import { getAdminToken } from "../auth/tokenStorage"; +import { DashboardPage } from "../pages/DashboardPage"; +import { ImportPage } from "../pages/ImportPage"; +import { ImportDetailPage } from "../pages/ImportDetailPage"; +import { ImagesPage } from "../pages/ImagesPage"; +import { LoginPage } from "../pages/LoginPage"; +import { PlaygroundPage } from "../pages/PlaygroundPage"; +import { TagsPage } from "../pages/TagsPage"; +import { AuthorsPage } from "../pages/AuthorsPage"; +import { TokensPage } from "../pages/TokensPage"; +import { ProxiesPage } from "../pages/ProxiesPage"; +import { ProxyPoolsPage } from "../pages/ProxyPoolsPage"; +import { BindingsPage } from "../pages/BindingsPage"; +import { JobsPage } from "../pages/JobsPage"; +import { HydrationPage } from "../pages/HydrationPage"; +import { SettingsPage } from "../pages/SettingsPage"; +import { AuditPage } from "../pages/AuditPage"; +import { RecommendationPage } from "../pages/RecommendationPage"; +import { PlaceholderPage } from "../pages/PlaceholderPage"; + +function buildNextParam(location: { pathname: string; search: string }): string { + const next = `${location.pathname}${location.search || ""}`; + return encodeURIComponent(next); +} + +function AriaI18nPatches() { + useEffect(() => { + if (typeof document === "undefined") return; + + const translations: Record = { + "Increase Value": "增加数值", + "Decrease Value": "减少数值", + }; + + const applyOnce = () => { + for (const [from, to] of Object.entries(translations)) { + document.querySelectorAll(`[aria-label="${from}"]`).forEach((node) => { + node.setAttribute("aria-label", to); + }); + } + }; + + applyOnce(); + + if (typeof MutationObserver === "undefined") return; + const observer = new MutationObserver(() => applyOnce()); + observer.observe(document.body, { subtree: true, childList: true }); + return () => observer.disconnect(); + }, []); + + return null; +} + +function RequireAdminToken({ children }: { children: React.ReactElement }) { + const location = useLocation(); + const token = getAdminToken(); + if (!token) { + const next = buildNextParam(location); + return ; + } + return children; +} + +function UnauthorizedListener() { + const navigate = useNavigate(); + const location = useLocation(); + + useEffect(() => { + const handler = () => { + if (location.pathname.startsWith("/admin/login")) return; + const next = buildNextParam(location); + navigate(`/admin/login?reason=unauthorized&next=${next}`, { replace: true }); + }; + + window.addEventListener("admin:unauthorized", handler); + return () => { + window.removeEventListener("admin:unauthorized", handler); + }; + }, [location, navigate]); + + return null; +} + +export function App() { + return ( + + + + + + } /> + } /> + + + + } + > + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + } /> + + + + ); +} diff --git a/frontend/src/auth/tokenStorage.test.ts b/frontend/src/auth/tokenStorage.test.ts new file mode 100644 index 0000000..ab5f136 --- /dev/null +++ b/frontend/src/auth/tokenStorage.test.ts @@ -0,0 +1,18 @@ +import { beforeEach, describe, expect, it } from "vitest"; + +import { clearAdminToken, getAdminToken, setAdminToken } from "./tokenStorage"; + +describe("tokenStorage", () => { + beforeEach(() => { + localStorage.clear(); + }); + + it("stores and clears admin token", () => { + expect(getAdminToken()).toBeNull(); + setAdminToken("t1"); + expect(getAdminToken()).toBe("t1"); + clearAdminToken(); + expect(getAdminToken()).toBeNull(); + }); +}); + diff --git a/frontend/src/auth/tokenStorage.ts b/frontend/src/auth/tokenStorage.ts new file mode 100644 index 0000000..ffc51d7 --- /dev/null +++ b/frontend/src/auth/tokenStorage.ts @@ -0,0 +1,16 @@ +const STORAGE_KEY = "admin_token"; + +export function getAdminToken(): string | null { + const value = localStorage.getItem(STORAGE_KEY); + if (!value) return null; + return value; +} + +export function setAdminToken(token: string): void { + localStorage.setItem(STORAGE_KEY, token); +} + +export function clearAdminToken(): void { + localStorage.removeItem(STORAGE_KEY); +} + diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx new file mode 100644 index 0000000..7a4fb10 --- /dev/null +++ b/frontend/src/main.tsx @@ -0,0 +1,18 @@ +import "antd/dist/reset.css"; + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import React from "react"; +import ReactDOM from "react-dom/client"; + +import { App } from "./app/App"; + +const queryClient = new QueryClient(); + +ReactDOM.createRoot(document.getElementById("root")!).render( + + + + + , +); + diff --git a/frontend/src/pages/AuditPage.test.tsx b/frontend/src/pages/AuditPage.test.tsx new file mode 100644 index 0000000..fa0e75d --- /dev/null +++ b/frontend/src/pages/AuditPage.test.tsx @@ -0,0 +1,60 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { AuditPage } from "./AuditPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("AuditPage", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/admin/api/audit?limit=50")) { + return new Response( + JSON.stringify({ + ok: true, + items: [ + { + id: "a1", + created_at: "2026-02-11T00:00:00Z", + actor: "admin", + action: "CREATE", + resource: "token", + record_id: "1", + request_id: "req_1", + detail_json: { k: "v" }, + }, + ], + next_cursor: "", + request_id: "req_audit", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders list", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("审计日志")).toBeInTheDocument(); + expect(await screen.findByText("token")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_audit/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/AuditPage.tsx b/frontend/src/pages/AuditPage.tsx new file mode 100644 index 0000000..165b272 --- /dev/null +++ b/frontend/src/pages/AuditPage.tsx @@ -0,0 +1,90 @@ +import { useQuery } from "@tanstack/react-query"; +import { Alert, Card, Skeleton, Space, Table, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React from "react"; + +import { ApiError, apiJson } from "../api/client"; + +type AuditItem = { + id: string; + created_at: string; + actor: string | null; + action: string; + resource: string; + record_id: string | null; + request_id: string | null; + detail_json: Record | null; +}; + +type AuditListResponse = { + ok: true; + items: AuditItem[]; + next_cursor: string; + request_id: string; +}; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +const columns: ColumnsType = [ + { title: "时间", dataIndex: "created_at", key: "created_at" }, + { title: "操作者", dataIndex: "actor", key: "actor" }, + { title: "动作", dataIndex: "action", key: "action" }, + { title: "资源", dataIndex: "resource", key: "resource" }, + { title: "记录ID", dataIndex: "record_id", key: "record_id" }, + { title: "请求ID", dataIndex: "request_id", key: "request_id" }, + { + title: "详情", + key: "detail_json", + render: (_, row) => ( + 
+        {row.detail_json ? JSON.stringify(row.detail_json) : ""}
+
+ ), + }, +]; + +export function AuditPage() { + const query = useQuery({ + queryKey: ["admin", "audit", { limit: 50 }], + queryFn: () => apiJson("/admin/api/audit?limit=50"), + }); + + return ( + + + 审计日志 + + + {query.isLoading ? ( + + ) : query.isError ? ( + + ) : !query.data ? ( + + ) : query.data.items.length === 0 ? ( + + ) : ( + + 请求ID: {query.data.request_id} + + rowKey={(row) => row.id} + columns={columns} + dataSource={query.data.items} + pagination={false} + size="small" + style={{ marginTop: 12 }} + /> + + )} + + ); +} + diff --git a/frontend/src/pages/AuthorsPage.test.tsx b/frontend/src/pages/AuthorsPage.test.tsx new file mode 100644 index 0000000..44c2db4 --- /dev/null +++ b/frontend/src/pages/AuthorsPage.test.tsx @@ -0,0 +1,49 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { AuthorsPage } from "./AuthorsPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("AuthorsPage", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/authors?limit=50")) { + return new Response( + JSON.stringify({ + ok: true, + items: [{ user_id: "9", user_name: "u", count_images: 12 }], + next_cursor: "", + request_id: "req_authors", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders list", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("作者列表")).toBeInTheDocument(); + expect(await screen.findByText("9")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_authors/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/AuthorsPage.tsx b/frontend/src/pages/AuthorsPage.tsx new file mode 100644 index 0000000..52aadea --- /dev/null +++ b/frontend/src/pages/AuthorsPage.tsx @@ -0,0 +1,73 @@ +import { useQuery } from "@tanstack/react-query"; +import { Alert, Card, Skeleton, Space, Table, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React from "react"; + +import { ApiError, apiJson } from "../api/client"; + +type AuthorItem = { + user_id: string; + user_name: string | null; + count_images: number | null; +}; + +type AuthorsListResponse = { + ok: true; + items: AuthorItem[]; + next_cursor: string; + request_id: string; +}; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +const columns: ColumnsType = [ + { title: "作者ID", dataIndex: "user_id", key: "user_id" }, + { title: "作者名", dataIndex: "user_name", key: "user_name" }, + { title: "图片数量", dataIndex: "count_images", key: "count_images" }, +]; + +export function AuthorsPage() { + const query = useQuery({ + queryKey: ["public", "authors", { limit: 50 }], + queryFn: () => apiJson("/authors?limit=50"), + }); + + return ( + + + 作者列表 + + + {query.isLoading ? ( + + ) : query.isError ? ( + + ) : !query.data ? ( + + ) : query.data.items.length === 0 ? ( + + ) : ( + + 请求ID: {query.data.request_id} + + rowKey={(row) => row.user_id} + columns={columns} + dataSource={query.data.items} + pagination={false} + size="small" + style={{ marginTop: 12 }} + /> + + )} + + ); +} + diff --git a/frontend/src/pages/BindingsPage.test.tsx b/frontend/src/pages/BindingsPage.test.tsx new file mode 100644 index 0000000..6bbed36 --- /dev/null +++ b/frontend/src/pages/BindingsPage.test.tsx @@ -0,0 +1,202 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import React from "react"; +import { MemoryRouter } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { BindingsPage } from "./BindingsPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("BindingsPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + let listCalls = 0; + let overrideProxyId: string | null = null; + let overrideExpiresAt: string | null = null; + let recomputeCalls = 0; + + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/admin/api/bindings/recompute")) { + recomputeCalls += 1; + expect(init?.method).toBe("POST"); + const body = init?.body ? JSON.parse(String(init.body)) : {}; + expect(body.pool_id).toBe(1); + expect(body.max_tokens_per_proxy).toBe(2); + + const strict = body.strict !== undefined ? Boolean(body.strict) : true; + if (strict) { + return new Response( + JSON.stringify({ + ok: false, + code: "BAD_REQUEST", + message: "代理容量不足（请增加节点或调高单代理最多绑定令牌数）", + request_id: `req_capacity_${recomputeCalls}`, + details: { token_count: 5, proxy_count: 1, max_tokens_per_proxy: 2, weight_sum: 1, capacity: 2 }, + }), + { status: 400, headers: { "Content-Type": "application/json" } }, + ); + } + + return new Response( + JSON.stringify({ + ok: true, + pool_id: "1", + recomputed: 5, + strict: false, + over_capacity_assigned: 3, + capacity: 2, + token_count: 5, + proxy_count: 1, + max_tokens_per_proxy: 2, + request_id: `req_recompute_${recomputeCalls}`, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/bindings?pool_id=1")) { + listCalls += 1; + return new Response( + JSON.stringify({ + ok: true, + items: [ + { + id: "1", + token: { id: "1", label: "acc1" }, + pool: { id: "1", name: "pixiv" }, + primary_proxy: { id: "10", scheme: "http", host: "1.2.3.4", port: 8080, username: "u1" }, + override_proxy: overrideProxyId + ? { id: overrideProxyId, scheme: "http", host: "9.9.9.9", port: 8080, username: "" } + : null, + override_expires_at: overrideExpiresAt, + effective_proxy_id: overrideProxyId ? overrideProxyId : "10", + effective_mode: overrideProxyId ? "override" : "primary", + }, + ], + request_id: `req_bindings_${listCalls}`, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/bindings/1/override")) { + expect(init?.method).toBe("POST"); + const body = init?.body ? JSON.parse(String(init.body)) : {}; + expect(body.override_proxy_id).toBe(11); + expect(body.ttl_ms).toBe(30 * 60 * 1000); + expect(body.reason).toBe("test"); + overrideProxyId = "11"; + overrideExpiresAt = "2026-02-15T00:00:00Z"; + return new Response( + JSON.stringify({ + ok: true, + binding_id: "1", + override_proxy_id: "11", + override_expires_at: overrideExpiresAt, + request_id: "req_override", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/bindings/1/clear-override")) { + expect(init?.method).toBe("POST"); + overrideProxyId = null; + overrideExpiresAt = null; + return new Response(JSON.stringify({ ok: true, binding_id: "1", request_id: "req_clear" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders list", async () => { + const qc = makeClient(); + render( + + + + + , + ); + + expect(await screen.findByText("令牌与代理绑定")).toBeInTheDocument(); + expect(await screen.findByText("acc1（#1）")).toBeInTheDocument(); + expect(await screen.findByText("pixiv（#1）")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_bindings_1/)).toBeInTheDocument(); + }); + + it("sets and clears override", async () => { + const qc = makeClient(); + render( + + + + + , + ); + + expect(await screen.findByText("令牌与代理绑定")).toBeInTheDocument(); + expect(await screen.findByText("acc1（#1）")).toBeInTheDocument(); + + fireEvent.click(await screen.findByRole("button", { name: "设置覆盖" })); + const dialog = await screen.findByRole("dialog", { name: /设置覆盖代理/ }); + fireEvent.change(within(dialog).getByPlaceholderText("例如：10"), { target: { value: "11" } }); + fireEvent.change(within(dialog).getByPlaceholderText("例如：60"), { target: { value: "30" } }); + fireEvent.change(within(dialog).getByPlaceholderText("例如：临时切换节点排查问题"), { target: { value: "test" } }); + + const form = dialog.querySelector("form"); + expect(form).not.toBeNull(); + fireEvent.submit(form as HTMLFormElement); + + expect(await screen.findByText(/已设置覆盖代理：绑定 #1 → 节点 #11/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_override/)).toBeInTheDocument(); + expect((await screen.findAllByText(/9\.9\.9\.9:8080/)).length).toBeGreaterThan(0); + + fireEvent.click(await screen.findByRole("button", { name: "清除覆盖" })); + expect(await screen.findByText(/已清除覆盖代理：绑定 #1/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_clear/)).toBeInTheDocument(); + + await waitFor(() => { + expect(screen.queryAllByText(/9\.9\.9\.9:8080/)).toHaveLength(0); + }); + }); + + it("continues recompute when proxy capacity is insufficient", async () => { + const qc = makeClient(); + render( + + + + + , + ); + + expect(await screen.findByText("令牌与代理绑定")).toBeInTheDocument(); + + fireEvent.click(await screen.findByRole("button", { name: "重新计算绑定" })); + expect(await screen.findByText("重新计算绑定失败")).toBeInTheDocument(); + expect(await screen.findByText(/令牌数=5，代理数=1，单代理上限=2/)).toBeInTheDocument(); + expect(await screen.findByText(/权重和=1/)).toBeInTheDocument(); + expect(await screen.findByText(/总容量=2/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_capacity_1/)).toBeInTheDocument(); + + fireEvent.click(await screen.findByRole("button", { name: "继续计算（允许超出容量）" })); + expect(await screen.findByText("重新计算绑定完成")).toBeInTheDocument(); + expect(await screen.findByText(/重算数量:\s*5/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_recompute_2/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/BindingsPage.tsx b/frontend/src/pages/BindingsPage.tsx new file mode 100644 index 0000000..663c580 --- /dev/null +++ b/frontend/src/pages/BindingsPage.tsx @@ -0,0 +1,467 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Alert, Button, Card, Form, Input, InputNumber, Modal, Skeleton, Space, Table, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React, { useMemo, useState } from "react"; +import { useSearchParams } from "react-router-dom"; + +import { ApiError, apiJson } from "../api/client"; + +type ProxyRef = { + id: string; + scheme: string; + host: string; + port: number; + username: string; +}; + +type BindingItem = { + id: string; + created_at: string; + updated_at: string; + token: { id: string; label: string | null }; + pool: { id: string; name: string }; + primary_proxy: ProxyRef; + override_proxy: ProxyRef | null; + override_expires_at: string | null; + effective_proxy_id: string; + effective_mode: "primary" | "override"; +}; + +type BindingsListResponse = { + ok: true; + items: BindingItem[]; + summary?: { pool_id: string; pool_endpoints_total: number; pool_endpoints_enabled: number }; + request_id: string; +}; + +type RecomputeResponse = { + ok: true; + pool_id: string; + recomputed: number; + strict?: boolean; + over_capacity_assigned?: number; + capacity?: number; + token_count?: number; + proxy_count?: number; + max_tokens_per_proxy?: number; + request_id: string; +}; + +type OverrideResponse = { + ok: true; + binding_id: string; + override_proxy_id: string; + override_expires_at: string; + request_id: string; +}; + +type ClearOverrideResponse = { + ok: true; + binding_id: string; + request_id: string; +}; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +function messageFromError(err: unknown): string { + if (err instanceof ApiError) return err.message; + if (err instanceof Error) return err.message; + return "未知错误"; +} + +function formatProxy(proxy: ProxyRef | null): string { + if (!proxy) return ""; + const host = String(proxy.host || "").includes(":") && !String(proxy.host || "").startsWith("[") ? `[${proxy.host}]` : proxy.host; + const user = String(proxy.username || "").trim(); + const auth = user ? `${user}@` : ""; + return `节点#${proxy.id} ${proxy.scheme}://${auth}${host}:${proxy.port}`; +} + +function modeLabel(mode: "primary" | "override"): string { + return mode === "override" ? "覆盖代理" : "主代理"; +} + +function stabilityLabel(row: BindingItem): string { + if (row.effective_mode === "override") return "固定（覆盖到期/清除会变）"; + return "固定（重算会变）"; +} + +const columns = (actions: { + onOpenOverride: (row: BindingItem) => void; + onClearOverride: (row: BindingItem) => void; + overridePendingId: string | null; + clearPendingId: string | null; +}): ColumnsType => [ + { + title: "令牌", + key: "token", + render: (_, row) => (row.token.label ? `${row.token.label}（#${row.token.id}）` : `#${row.token.id}`), + }, + { + title: "代理池", + key: "pool", + render: (_, row) => `${row.pool.name}（#${row.pool.id}）`, + }, + { + title: "当前生效代理", + key: "effective", + render: (_, row) => (row.effective_mode === "override" ? formatProxy(row.override_proxy) : formatProxy(row.primary_proxy)), + }, + { title: "生效模式", dataIndex: "effective_mode", key: "effective_mode", render: (value) => modeLabel(value) }, + { title: "固定性", key: "stability", render: (_, row) => stabilityLabel(row) }, + { title: "绑定更新时间", dataIndex: "updated_at", key: "updated_at", render: (value) => value || "-" }, + { title: "主代理", key: "primary_proxy", render: (_, row) => formatProxy(row.primary_proxy) }, + { title: "覆盖代理", key: "override_proxy", render: (_, row) => formatProxy(row.override_proxy) || "-" }, + { title: "覆盖过期时间", dataIndex: "override_expires_at", key: "override_expires_at", render: (value) => value || "-" }, + { + title: "操作", + key: "actions", + render: (_, row) => ( + + + + + ), + }, +]; + +type OverrideFormValues = { + override_proxy_id: number; + ttl_minutes: number; + reason: string; +}; + +export function BindingsPage() { + const queryClient = useQueryClient(); + const [searchParams, setSearchParams] = useSearchParams(); + + const initialPoolId = useMemo(() => { + const raw = searchParams.get("pool_id"); + const value = raw ? Number.parseInt(raw, 10) : 1; + return Number.isFinite(value) && value > 0 ? value : 1; + }, [searchParams]); + + const [poolId, setPoolId] = useState(initialPoolId); + const [maxTokensPerProxy, setMaxTokensPerProxy] = useState(2); + + const [actionMessage, setActionMessage] = useState(null); + const [actionRequestId, setActionRequestId] = useState(null); + const [actionErrorMessage, setActionErrorMessage] = useState(null); + const [actionErrorRequestId, setActionErrorRequestId] = useState(null); + + const [overrideOpen, setOverrideOpen] = useState(false); + const [overrideBinding, setOverrideBinding] = useState(null); + const [overrideForm] = Form.useForm(); + + const query = useQuery({ + queryKey: ["admin", "bindings", { poolId }], + queryFn: () => apiJson(`/admin/api/bindings?pool_id=${poolId}`), + }); + + const recompute = useMutation({ + mutationFn: (vars: { strict?: boolean } | undefined) => + apiJson("/admin/api/bindings/recompute", { + method: "POST", + body: JSON.stringify({ + pool_id: poolId, + max_tokens_per_proxy: maxTokensPerProxy, + strict: vars?.strict !== undefined ? Boolean(vars.strict) : true, + }), + }), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["admin", "bindings", { poolId }] }); + }, + }); + + const setOverride = useMutation({ + mutationFn: (payload: { bindingId: string; override_proxy_id: number; ttl_ms: number; reason: string }) => + apiJson(`/admin/api/bindings/${encodeURIComponent(payload.bindingId)}/override`, { + method: "POST", + body: JSON.stringify({ + override_proxy_id: payload.override_proxy_id, + ttl_ms: payload.ttl_ms, + reason: payload.reason, + }), + }), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + setActionMessage(`已设置覆盖代理：绑定 #${data.binding_id} → 节点 #${data.override_proxy_id}`); + setActionRequestId(data.request_id); + setOverrideOpen(false); + setOverrideBinding(null); + overrideForm.resetFields(); + queryClient.invalidateQueries({ queryKey: ["admin", "bindings", { poolId }] }); + }, + onError: (err) => { + setActionErrorMessage(messageFromError(err)); + setActionErrorRequestId(requestIdFromError(err)); + }, + }); + + const clearOverride = useMutation({ + mutationFn: (bindingId: string) => + apiJson(`/admin/api/bindings/${encodeURIComponent(bindingId)}/clear-override`, { method: "POST" }), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + setActionMessage(`已清除覆盖代理：绑定 #${data.binding_id}`); + setActionRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "bindings", { poolId }] }); + }, + onError: (err) => { + setActionErrorMessage(messageFromError(err)); + setActionErrorRequestId(requestIdFromError(err)); + }, + }); + + const setPoolIdAndSyncUrl = (next: number) => { + const value = Number.isFinite(next) && next > 0 ? next : 1; + setPoolId(value); + const params = new URLSearchParams(searchParams); + params.set("pool_id", String(value)); + setSearchParams(params, { replace: true }); + }; + + return ( + + + 令牌与代理绑定 + + + { + const summary = query.data?.summary; + const parts: string[] = [ + "绑定默认是固定的：仅会在你点击“重新计算绑定”、设置覆盖、清除覆盖或覆盖到期后发生变化。", + "这里显示的是“本项目侧的代理入口（host:port）”。如果你导入的是 easy-proxies 单入口 pool，你只会看到 pool 入口端口（例如 2323）；pool 内部实际命中的节点端口需要在 easy-proxies 面板查看。", + ]; + if (summary && typeof summary.pool_endpoints_total === "number" && typeof summary.pool_endpoints_enabled === "number") { + parts.push(`当前代理池入口：可用 ${summary.pool_endpoints_enabled}/${summary.pool_endpoints_total}。`); + if (summary.pool_endpoints_enabled === 1) { + parts.push("可用入口=1 时，所有令牌绑定到同一入口属于正常现象。"); + } + } + return parts.join(" "); + })() + } + /> + + {actionMessage ? : null} + {actionRequestId ? 请求ID: {actionRequestId} : null} + {actionErrorMessage ? : null} + {actionErrorRequestId ? 请求ID: {actionErrorRequestId} : null} + + + + 代理池ID: + setPoolIdAndSyncUrl(Number(value || 1))} /> + 单代理最多绑定令牌数: + setMaxTokensPerProxy(Number(value || 2))} + /> + + + + {recompute.isError ? ( + { + const rid = requestIdFromError(recompute.error); + const msg = messageFromError(recompute.error); + const details = + recompute.error instanceof ApiError && recompute.error.body && typeof recompute.error.body.details === "object" + ? recompute.error.body.details + : null; + const tokenCount = details && typeof details.token_count === "number" ? details.token_count : null; + const proxyCount = details && typeof details.proxy_count === "number" ? details.proxy_count : null; + const maxPer = details && typeof details.max_tokens_per_proxy === "number" ? details.max_tokens_per_proxy : null; + const weightSum = details && typeof details.weight_sum === "number" ? details.weight_sum : null; + const capacity = details && typeof details.capacity === "number" ? details.capacity : null; + const parts = [msg]; + if (tokenCount !== null && proxyCount !== null && maxPer !== null) { + const extra = [`令牌数=${tokenCount}，代理数=${proxyCount}，单代理上限=${maxPer}`]; + if (weightSum !== null) extra.push(`权重和=${weightSum}`); + if (capacity !== null) extra.push(`总容量=${capacity}`); + parts.push(extra.join("，")); + } + if (rid) parts.push(`请求ID: ${rid}`); + return parts.filter((p) => String(p || "").trim()).join("；"); + })() + } + style={{ marginTop: 12 }} + /> + ) : null} + + {recompute.isError && + recompute.error instanceof ApiError && + recompute.error.body && + recompute.error.body.code === "BAD_REQUEST" && + typeof recompute.error.body.details?.token_count === "number" && + typeof recompute.error.body.details?.proxy_count === "number" && + typeof recompute.error.body.details?.max_tokens_per_proxy === "number" ? ( + + ) : null} + + {recompute.isSuccess ? ( + + ) : null} + + + { + setOverrideOpen(false); + setOverrideBinding(null); + overrideForm.resetFields(); + }} + footer={null} + destroyOnClose + > + + form={overrideForm} + layout="vertical" + initialValues={{ override_proxy_id: 0, ttl_minutes: 60, reason: "" }} + onFinish={(values) => { + const bindingId = overrideBinding?.id; + if (!bindingId) return; + const ttlMinutes = Number(values.ttl_minutes || 0); + const ttlMs = Math.trunc(ttlMinutes * 60 * 1000); + setOverride.mutate({ + bindingId, + override_proxy_id: Number(values.override_proxy_id), + ttl_ms: ttlMs, + reason: String(values.reason || "").trim(), + }); + }} + > + + + + + + + + + + + + + + + + + + + + {query.isLoading ? ( + + ) : query.isError ? ( + + ) : !query.data ? ( + + ) : query.data.items.length === 0 ? ( + + ) : ( + + 请求ID: {query.data.request_id} + + rowKey={(row) => row.id} + columns={columns({ + onOpenOverride: (row) => { + setOverrideBinding(row); + setOverrideOpen(true); + const currentId = row.override_proxy?.id || row.primary_proxy.id; + const parsed = Number.parseInt(String(currentId || "0"), 10); + overrideForm.setFieldsValue({ + override_proxy_id: Number.isFinite(parsed) && parsed > 0 ? parsed : 1, + ttl_minutes: 60, + reason: "", + }); + }, + onClearOverride: (row) => clearOverride.mutate(row.id), + overridePendingId: setOverride.isPending ? setOverride.variables?.bindingId ?? null : null, + clearPendingId: clearOverride.isPending ? clearOverride.variables ?? null : null, + })} + dataSource={query.data.items} + pagination={false} + size="small" + style={{ marginTop: 12 }} + /> + + )} + + ); +} diff --git a/frontend/src/pages/DashboardPage.test.tsx b/frontend/src/pages/DashboardPage.test.tsx new file mode 100644 index 0000000..ebfeab0 --- /dev/null +++ b/frontend/src/pages/DashboardPage.test.tsx @@ -0,0 +1,191 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import React from "react"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { DashboardPage } from "./DashboardPage"; +import { ImagesPage } from "./ImagesPage"; +import { ImportPage } from "./ImportPage"; +import { JobsPage } from "./JobsPage"; +import { PlaygroundPage } from "./PlaygroundPage"; +import { ProxiesPage } from "./ProxiesPage"; +import { TokensPage } from "./TokensPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("DashboardPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, _init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/version")) { + return new Response( + JSON.stringify({ + ok: true, + version: "dev", + build_time: "2026-02-15T00:00:00Z", + git_commit: "abcdef1", + request_id: "req_version", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/settings")) { + return new Response( + JSON.stringify({ + ok: true, + settings: { + proxy: { enabled: false, fail_closed: true, route_mode: "pixiv_only", allowlist_domains: [] }, + random: {}, + security: { hide_origin_url_in_public_json: true }, + rate_limit: {}, + }, + request_id: "req_settings", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/summary")) { + return new Response( + JSON.stringify({ + ok: true, + counts: { + images: { total: 14, enabled: 14 }, + tokens: { total: 2, enabled: 1 }, + proxies: { endpoints_total: 1, endpoints_enabled: 1 }, + proxy_pools: { total: 0, enabled: 0 }, + bindings: { total: 0 }, + jobs: { counts: { pending: 2, running: 0, failed: 3 } }, + worker: { last_seen_at: "2026-02-13T00:00:00Z" }, + }, + request_id: "req_summary", + }), + { + status: 200, + headers: { "Content-Type": "application/json" }, + }, + ); + } + if (url.endsWith("/admin/api/stats/random")) { + return new Response( + JSON.stringify({ + ok: true, + stats: { + total_requests: 12, + total_ok: 10, + total_error: 2, + in_flight: 0, + window_seconds: 60, + last_window_requests: 3, + last_window_ok: 3, + last_window_error: 0, + last_window_success_rate: 1.0, + }, + request_id: "req_stats", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.includes("/admin/api/jobs?status=failed")) { + return new Response(JSON.stringify({ ok: true, items: [{}, {}, {}], next_cursor: "", request_id: "req_jobs" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.endsWith("/admin/api/hydration-runs")) { + return new Response(JSON.stringify({ ok: true, hydration_run_id: "10", job_id: "99", request_id: "req_hyd" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + function renderDashboard() { + const qc = makeClient(); + render( + + + + } /> + } /> + } /> + } /> + } /> + } /> + } /> + + + , + ); + } + + it("renders fetched counts", async () => { + renderDashboard(); + + expect(await screen.findByText("工作线程心跳: 2026-02-13T00:00:00Z")).toBeInTheDocument(); + expect(await screen.findByText("总数: 14")).toBeInTheDocument(); + expect(await screen.findByText("总数: 2")).toBeInTheDocument(); + expect(await screen.findByText("总请求: 12")).toBeInTheDocument(); + expect(await screen.findByText("近 1 分钟成功率: 100.0%")).toBeInTheDocument(); + expect(await screen.findByText("代理节点: 1/1 启用")).toBeInTheDocument(); + expect(await screen.findByText(/提交:\s*abcdef1/)).toBeInTheDocument(); + }); + + it("navigates to import", async () => { + renderDashboard(); + + expect(await screen.findByText("工作线程 / 队列")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /去导入链接/ })); + expect(await screen.findByText("导入图片链接")).toBeInTheDocument(); + }); + + it("creates hydration run", async () => { + renderDashboard(); + + expect(await screen.findByText("工作线程 / 队列")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /创建补全任务/ })); + expect(await screen.findByText("补全任务已创建")).toBeInTheDocument(); + expect(await screen.findByText(/补全运行ID:\s*10/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_hyd/)).toBeInTheDocument(); + }); + + it("navigates to tokens", async () => { + renderDashboard(); + + expect(await screen.findByText("工作线程 / 队列")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /去添加令牌/ })); + expect(await screen.findByRole("button", { name: /新增令牌/ })).toBeInTheDocument(); + }); + + it("navigates to proxies", async () => { + renderDashboard(); + + expect(await screen.findByText("工作线程 / 队列")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /去添加代理/ })); + expect(await screen.findByText("代理管理")).toBeInTheDocument(); + }); + + it("navigates to playground", async () => { + renderDashboard(); + + expect(await screen.findByText("工作线程 / 队列")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /打开随机测试/ })); + expect(await screen.findByText("随机接口调试")).toBeInTheDocument(); + }); +}); + diff --git a/frontend/src/pages/DashboardPage.tsx b/frontend/src/pages/DashboardPage.tsx new file mode 100644 index 0000000..35b7d34 --- /dev/null +++ b/frontend/src/pages/DashboardPage.tsx @@ -0,0 +1,372 @@ +import { useMutation, useQuery } from "@tanstack/react-query"; +import { Alert, Button, Card, Col, Row, Skeleton, Space, Typography } from "antd"; +import React from "react"; +import { useNavigate } from "react-router-dom"; + +import { ApiError, apiJson } from "../api/client"; + +type SummaryResponse = { + ok: true; + counts: { + images: { total: number; enabled: number }; + tokens: { total: number; enabled: number }; + proxies: { endpoints_total: number; endpoints_enabled: number }; + proxy_pools: { total: number; enabled: number }; + bindings: { total: number }; + jobs: { counts: Record }; + worker: { last_seen_at: string | null }; + }; + request_id: string; +}; + +type SettingsResponse = { + ok: true; + settings: { + proxy: { + enabled: boolean; + fail_closed: boolean; + route_mode: string; + allowlist_domains: string[]; + default_pool_id?: string; + }; + random: Record; + security: { hide_origin_url_in_public_json: boolean }; + rate_limit: Record; + }; + request_id: string; +}; + +type VersionResponse = { + ok: true; + version: string; + build_time: string; + git_commit: string; + request_id: string; +}; + +type RandomStatsResponse = { + ok: true; + stats: { + total_requests: number; + total_ok: number; + total_error: number; + in_flight: number; + window_seconds: number; + last_window_requests: number; + last_window_ok: number; + last_window_error: number; + last_window_success_rate: number; + }; + request_id: string; +}; + +type JobsResponse = { ok: true; items: unknown[]; next_cursor: string; request_id: string }; +type CreateHydrationRunResponse = { ok: true; hydration_run_id: string; job_id: string; request_id: string }; + +function asApiError(err: unknown): ApiError | null { + return err instanceof ApiError ? err : null; +} + +function requestIdFromError(err: unknown): string | null { + const apiErr = asApiError(err); + if (!apiErr?.body?.request_id) return null; + return String(apiErr.body.request_id); +} + +function messageFromError(err: unknown): string { + if (err instanceof ApiError) return err.message; + if (err instanceof Error) return err.message; + return "未知错误"; +} + +export function DashboardPage() { + const navigate = useNavigate(); + + const createHydration = useMutation({ + mutationFn: () => + apiJson("/admin/api/hydration-runs", { + method: "POST", + body: JSON.stringify({}), + }), + }); + + const settings = useQuery({ + queryKey: ["admin", "settings"], + queryFn: () => apiJson("/admin/api/settings"), + }); + + const summary = useQuery({ + queryKey: ["admin", "summary"], + queryFn: () => apiJson("/admin/api/summary"), + }); + + const failedJobs = useQuery({ + queryKey: ["admin", "jobs", "failed"], + queryFn: () => apiJson("/admin/api/jobs?status=failed&limit=10"), + }); + + const randomStats = useQuery({ + queryKey: ["admin", "stats", "random"], + queryFn: () => apiJson("/admin/api/stats/random"), + refetchInterval: 5000, + }); + + const version = useQuery({ + queryKey: ["public", "version"], + queryFn: () => apiJson("/version"), + }); + + const proxyEnabled = settings.data?.settings.proxy.enabled ?? false; + const defaultPoolId = settings.data?.settings.proxy.default_pool_id ?? ""; + + const counts = summary.data?.counts; + const imageCount = counts?.images.total ?? 0; + const imageEnabledCount = counts?.images.enabled ?? 0; + const tokenCount = counts?.tokens.total ?? 0; + const tokenEnabledCount = counts?.tokens.enabled ?? 0; + const proxyCount = counts?.proxies.endpoints_total ?? 0; + const proxyEnabledCount = counts?.proxies.endpoints_enabled ?? 0; + const proxyPoolCount = counts?.proxy_pools.total ?? 0; + const proxyPoolEnabledCount = counts?.proxy_pools.enabled ?? 0; + const bindingCount = counts?.bindings.total ?? 0; + + const jobsCounts = counts?.jobs.counts ?? {}; + const pendingJobs = jobsCounts.pending ?? 0; + const runningJobs = jobsCounts.running ?? 0; + const failedJobsTotal = jobsCounts.failed ?? 0; + const workerLastSeenAt = counts?.worker.last_seen_at ?? null; + + const failedJobCount = failedJobs.data?.items.length ?? 0; + + const lastWindowRequests = randomStats.data?.stats.last_window_requests ?? 0; + const lastWindowOk = randomStats.data?.stats.last_window_ok ?? 0; + const lastWindowError = randomStats.data?.stats.last_window_error ?? 0; + const lastWindowSuccessRate = randomStats.data?.stats.last_window_success_rate ?? 0; + const inFlight = randomStats.data?.stats.in_flight ?? 0; + const totalRequests = randomStats.data?.stats.total_requests ?? 0; + const totalOk = randomStats.data?.stats.total_ok ?? 0; + const totalError = randomStats.data?.stats.total_error ?? 0; + + return ( + <> + + + + + + + + + + {createHydration.isSuccess ? ( + + ) : null} + {createHydration.isError ? ( + + ) : null} + + + + + {summary.isLoading ? ( + + ) : summary.isError ? ( + + ) : ( + + 工作线程心跳: {workerLastSeenAt || "（暂无）"} + 等待任务: {pendingJobs} + 运行任务: {runningJobs} + 失败任务: {failedJobsTotal} + + )} + + + + + + {summary.isLoading ? ( + + ) : summary.isError ? ( + + ) : ( + + 总数: {imageCount} + 启用: {imageEnabledCount} + + + )} + + + + + + {summary.isLoading ? ( + + ) : summary.isError ? ( + + ) : ( + + 总数: {tokenCount} + 启用: {tokenEnabledCount} + + + )} + + + + + + {settings.isLoading || summary.isLoading ? ( + + ) : settings.isError ? ( + + ) : summary.isError ? ( + + ) : ( + + 代理总开关: {proxyEnabled ? "开启" : "关闭"} + 默认代理池ID: {defaultPoolId || "（未设置）"} + 代理节点: {proxyEnabledCount}/{proxyCount} 启用 + 代理池: {proxyPoolEnabledCount}/{proxyPoolCount} 启用 + 绑定关系: {bindingCount} + + + )} + + + + + + + + {randomStats.isLoading ? ( + + ) : randomStats.isError ? ( + + ) : !randomStats.data ? ( + + ) : ( + + 总请求: {totalRequests} + + 总成功/失败: {totalOk}/{totalError} + + + 近 1 分钟请求: {lastWindowRequests}（成功/失败: {lastWindowOk}/{lastWindowError}） + + 近 1 分钟成功率: {(lastWindowSuccessRate * 100).toFixed(1)}% + 当前并发（in-flight）: {inFlight} + 请求ID: {randomStats.data.request_id} + + )} + + + + + + {version.isLoading ? ( + + ) : version.isError ? ( + + ) : !version.data ? ( + + ) : ( + + 版本: {version.data.version || "（未知）"} + 构建时间: {version.data.build_time || "（未设置）"} + 提交: {version.data.git_commit || "（未设置）"} + 请求ID: {version.data.request_id} + + )} + + + + + + {failedJobs.isLoading ? ( + + ) : failedJobs.isError ? ( + + ) : ( + + 数量: {failedJobCount} + + + )} + + + + + ); +} + + diff --git a/frontend/src/pages/HydrationPage.test.tsx b/frontend/src/pages/HydrationPage.test.tsx new file mode 100644 index 0000000..8bbf96a --- /dev/null +++ b/frontend/src/pages/HydrationPage.test.tsx @@ -0,0 +1,169 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import React from "react"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { HydrationPage } from "./HydrationPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("HydrationPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + + if (url.endsWith("/admin/api/summary")) { + return new Response( + JSON.stringify({ + ok: true, + counts: { + hydration: { + enabled_images_total: 10, + missing: { tags: 7, geometry: 5, r18: 10, ai: 3, user: 2, title: 1, created_at: 4, popularity: 6 }, + }, + jobs: { counts: { pending: 2, running: 1, failed: 0 } }, + worker: { last_seen_at: "2026-02-14T00:00:00Z" }, + }, + request_id: "req_summary", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + if (url.includes("/admin/api/hydration-runs?") && (!init?.method || init.method === "GET")) { + return new Response( + JSON.stringify({ + ok: true, + items: [ + { + id: "1", + type: "backfill", + status: "pending", + criteria: { missing: ["tags", "geometry"] }, + cursor: { image_id: 100 }, + total: null, + processed: 5, + success: 5, + failed: 0, + started_at: null, + finished_at: null, + last_error: null, + created_at: "2026-02-14T00:00:00Z", + updated_at: "2026-02-14T00:01:00Z", + latest_job: { + id: "88", + status: "pending", + attempt: 0, + max_attempts: 3, + run_after: null, + last_error: null, + locked_by: null, + locked_at: null, + updated_at: "2026-02-14T00:01:00Z", + }, + }, + ], + next_cursor: "", + request_id: "req_runs", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + if (url.endsWith("/admin/api/hydration-runs") && init?.method === "POST") { + return new Response( + JSON.stringify({ ok: true, hydration_run_id: "10", job_id: "99", request_id: "req_create" }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + if (url.endsWith("/admin/api/hydration-runs/manual") && init?.method === "POST") { + return new Response( + JSON.stringify({ ok: true, created: true, job_id: "123", illust_id: "112233", request_id: "req_manual" }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + if (url.endsWith("/admin/api/hydration-runs/1/pause") && init?.method === "POST") { + return new Response( + JSON.stringify({ ok: true, hydration_run_id: "1", status: "paused", job_status: "paused", request_id: "req_pause" }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + return new Response( + JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), + { status: 404, headers: { "Content-Type": "application/json" } }, + ); + }); + + vi.stubGlobal("fetch", fetchMock); + }); + + it("renders hydration runs and worker queue summary", async () => { + const qc = makeClient(); + render( + + + + } /> + + + , + ); + + expect(await screen.findByText("元数据补全管理")).toBeInTheDocument(); + expect(await screen.findByText("工作线程心跳: 2026-02-14T00:00:00Z")).toBeInTheDocument(); + expect(await screen.findByText("元数据覆盖率统计")).toBeInTheDocument(); + expect(await screen.findByText("标签（缺 7）")).toBeInTheDocument(); + expect(await screen.findByText("#88 等待中 0/3")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_runs/)).toBeInTheDocument(); + }); + + it("creates backfill run and manual hydrate job", async () => { + const qc = makeClient(); + const fetchMock = global.fetch as unknown as ReturnType; + + render( + + + + } /> + + + , + ); + + expect(await screen.findByText("元数据补全管理")).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "创建全量补全任务" })); + expect(await screen.findByText("全量补全任务已创建")).toBeInTheDocument(); + expect(await screen.findByText(/补全运行ID:\s*10/)).toBeInTheDocument(); + + fireEvent.change(screen.getByPlaceholderText("例如：12345678"), { target: { value: "112233" } }); + fireEvent.click(screen.getByRole("button", { name: "创建单图补全任务" })); + + expect(await screen.findByText("单图补全任务已创建")).toBeInTheDocument(); + expect(await screen.findByText(/任务ID:\s*123/)).toBeInTheDocument(); + + expect( + fetchMock.mock.calls.some((call: unknown[]) => { + const url = String(call[0] ?? ""); + const init = (call[1] ?? {}) as RequestInit; + if (!url.endsWith("/admin/api/hydration-runs/manual") || init.method !== "POST" || !init.body) { + return false; + } + const body = JSON.parse(String(init.body)); + return Number(body.illust_id) === 112233; + }), + ).toBe(true); + }); +}); diff --git a/frontend/src/pages/HydrationPage.tsx b/frontend/src/pages/HydrationPage.tsx new file mode 100644 index 0000000..73149d3 --- /dev/null +++ b/frontend/src/pages/HydrationPage.tsx @@ -0,0 +1,628 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + Alert, + Button, + Card, + Checkbox, + Col, + Form, + InputNumber, + Row, + Select, + Skeleton, + Space, + Table, + Tag, + Typography, +} from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React, { useMemo, useState } from "react"; +import { useNavigate } from "react-router-dom"; + +import { ApiError, apiJson } from "../api/client"; + +type SummaryResponse = { + ok: true; + counts: { + hydration?: { + enabled_images_total: number; + missing: Record; + }; + jobs: { counts: Record }; + worker: { last_seen_at: string | null }; + }; + request_id: string; +}; + +type HydrationRunJob = { + id: string; + status: string; + attempt: number; + max_attempts: number; + run_after: string | null; + last_error: string | null; + locked_by: string | null; + locked_at: string | null; + updated_at: string | null; +}; + +type HydrationRunItem = { + id: string; + type: string; + status: string; + criteria: Record; + cursor: Record; + total: number | null; + processed: number; + success: number; + failed: number; + started_at: string | null; + finished_at: string | null; + last_error: string | null; + created_at: string | null; + updated_at: string | null; + latest_job: HydrationRunJob | null; +}; + +type HydrationRunsResponse = { + ok: true; + items: HydrationRunItem[]; + next_cursor: string; + request_id: string; +}; + +type CreateHydrationRunResponse = { + ok: true; + hydration_run_id: string; + job_id: string; + request_id: string; +}; + +type ManualHydrateResponse = { + ok: true; + created: boolean; + job_id: string; + illust_id: string; + request_id: string; +}; + +type RunActionResponse = { + ok: true; + hydration_run_id: string; + status: string; + job_status: string; + request_id: string; +}; + +type RunAction = "pause" | "resume" | "cancel"; + +type BackfillFormValues = { + missing: string[]; +}; + +type ManualFormValues = { + target_type: "illust_id" | "image_id"; + target_id: number; +}; + +const MISSING_OPTIONS = [ + { label: "标签", value: "tags" }, + { label: "尺寸与方向", value: "geometry" }, + { label: "R18 信息", value: "r18" }, + { label: "AI 信息", value: "ai" }, + { label: "作品类型（插画/漫画/动图）", value: "illust_type" }, + { label: "作者信息", value: "user" }, + { label: "标题", value: "title" }, + { label: "发布时间", value: "created_at" }, + { label: "热度（收藏/浏览/评论）", value: "popularity" }, +]; + +const MISSING_LABELS: Record = { + tags: "标签", + geometry: "尺寸与方向", + r18: "R18 信息", + ai: "AI 信息", + illust_type: "作品类型", + user: "作者信息", + title: "标题", + created_at: "发布时间", + popularity: "热度（收藏/浏览/评论）", +}; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +function messageFromError(err: unknown): string { + if (err instanceof ApiError) return err.message; + if (err instanceof Error) return err.message; + return "未知错误"; +} + +function statusColor(status: string): string { + switch (status) { + case "running": + return "processing"; + case "pending": + return "blue"; + case "completed": + return "success"; + case "failed": + return "error"; + case "paused": + return "warning"; + case "canceled": + return "default"; + default: + return "default"; + } +} + +function statusLabel(status: string): string { + switch (status) { + case "pending": + return "等待中"; + case "running": + return "运行中"; + case "paused": + return "已暂停"; + case "completed": + return "已完成"; + case "failed": + return "失败"; + case "canceled": + return "已取消"; + default: + return status || "未知"; + } +} + +function typeLabel(type: string): string { + if (type === "backfill") return "全量补全"; + if (type === "manual") return "手动补全"; + return type || "未知"; +} + +function missingSummary(criteria: Record): string { + const missing = criteria.missing; + if (Array.isArray(missing)) { + const values = missing.map((v) => String(v || "").trim()).filter((v) => v.length > 0); + if (values.length > 0) { + return values.map((v) => MISSING_LABELS[v] || v).join("、"); + } + } + return "默认（全部缺失字段）"; +} + +export function HydrationPage() { + const navigate = useNavigate(); + const qc = useQueryClient(); + const [statusFilter, setStatusFilter] = useState("all"); + const [actionState, setActionState] = useState<{ runId: string; action: RunAction } | null>(null); + + const [backfillForm] = Form.useForm(); + const [manualForm] = Form.useForm(); + + const summary = useQuery({ + queryKey: ["admin", "summary"], + queryFn: () => apiJson("/admin/api/summary"), + }); + + const hydrationStats = summary.data?.counts.hydration; + const enabledImagesTotal = hydrationStats?.enabled_images_total ?? 0; + const missingCounts = useMemo(() => hydrationStats?.missing ?? {}, [hydrationStats?.missing]); + + const missingOptionsWithCounts = useMemo(() => { + return MISSING_OPTIONS.map((opt) => { + const missing = Number(missingCounts[opt.value] ?? 0); + const label = enabledImagesTotal > 0 ? `${opt.label}（缺 ${missing}）` : opt.label; + return { label, value: opt.value }; + }); + }, [enabledImagesTotal, missingCounts]); + + const runs = useQuery({ + queryKey: ["admin", "hydration-runs", { statusFilter }], + queryFn: () => { + const query = new URLSearchParams({ limit: "30" }); + if (statusFilter !== "all") { + query.set("status", statusFilter); + } + return apiJson(`/admin/api/hydration-runs?${query.toString()}`); + }, + }); + + const createBackfill = useMutation({ + mutationFn: (values: BackfillFormValues) => { + const body = + Array.isArray(values.missing) && values.missing.length > 0 + ? { type: "backfill", criteria: { missing: values.missing } } + : { type: "backfill", criteria: {} }; + return apiJson("/admin/api/hydration-runs", { + method: "POST", + body: JSON.stringify(body), + }); + }, + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["admin", "hydration-runs"] }); + qc.invalidateQueries({ queryKey: ["admin", "summary"] }); + }, + }); + + const createManual = useMutation({ + mutationFn: (values: ManualFormValues) => { + const body = values.target_type === "image_id" ? { image_id: values.target_id } : { illust_id: values.target_id }; + return apiJson("/admin/api/hydration-runs/manual", { + method: "POST", + body: JSON.stringify(body), + }); + }, + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["admin", "hydration-runs"] }); + qc.invalidateQueries({ queryKey: ["admin", "summary"] }); + }, + }); + + const runAction = useMutation({ + mutationFn: ({ runId, action }: { runId: string; action: RunAction }) => + apiJson(`/admin/api/hydration-runs/${encodeURIComponent(runId)}/${action}`, { + method: "POST", + }), + onMutate: (variables) => { + setActionState({ runId: variables.runId, action: variables.action }); + }, + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["admin", "hydration-runs"] }); + qc.invalidateQueries({ queryKey: ["admin", "summary"] }); + }, + onSettled: () => { + setActionState(null); + }, + }); + + const jobsCounts = summary.data?.counts.jobs.counts ?? {}; + const pendingJobs = jobsCounts.pending ?? 0; + const runningJobs = jobsCounts.running ?? 0; + const failedJobs = jobsCounts.failed ?? 0; + const workerLastSeen = summary.data?.counts.worker.last_seen_at ?? null; + + const columns: ColumnsType = useMemo( + () => [ + { title: "运行ID", dataIndex: "id", key: "id", width: 100 }, + { + title: "类型", + dataIndex: "type", + key: "type", + width: 120, + render: (value: string) => typeLabel(value), + }, + { + title: "状态", + key: "status", + width: 120, + render: (_, row) => {statusLabel(row.status)}, + }, + { + title: "进度", + key: "progress", + width: 220, + render: (_, row) => { + const total = row.total; + const done = row.processed; + const overall = total !== null && total > 0 ? `${done}/${total}` : `${done}`; + return `${overall}（成功:${row.success}，失败:${row.failed}）`; + }, + }, + { + title: "补全范围", + key: "criteria", + width: 260, + render: (_, row) => missingSummary(row.criteria || {}), + }, + { + title: "关联任务", + key: "latest_job", + width: 220, + render: (_, row) => { + if (!row.latest_job) return "-"; + return `#${row.latest_job.id} ${statusLabel(row.latest_job.status)} ${row.latest_job.attempt}/${row.latest_job.max_attempts}`; + }, + }, + { + title: "最近更新", + dataIndex: "updated_at", + key: "updated_at", + width: 180, + render: (value: string | null) => value || "-", + }, + { + title: "错误信息", + key: "last_error", + width: 280, + render: (_, row) => row.last_error || row.latest_job?.last_error || "-", + }, + { + title: "操作", + key: "actions", + width: 260, + render: (_, row) => { + const status = String(row.status || ""); + const pauseDisabled = status !== "pending" && status !== "running"; + const resumeDisabled = status !== "paused"; + const cancelDisabled = status !== "pending" && status !== "running" && status !== "paused"; + + const isPauseLoading = runAction.isPending && actionState?.runId === row.id && actionState?.action === "pause"; + const isResumeLoading = runAction.isPending && actionState?.runId === row.id && actionState?.action === "resume"; + const isCancelLoading = runAction.isPending && actionState?.runId === row.id && actionState?.action === "cancel"; + + return ( + + + + + + ); + }, + }, + ], + [actionState, runAction], + ); + + return ( + + + 元数据补全管理 + + + + + + {summary.isLoading ? ( + + ) : summary.isError ? ( + + ) : ( + + 工作线程心跳: {workerLastSeen || "（暂无）"} + 等待任务: {pendingJobs} + 运行任务: {runningJobs} + 失败任务: {failedJobs} + + + + + + )} + + + + + + + form={backfillForm} + layout="vertical" + initialValues={{ missing: MISSING_OPTIONS.map((item) => item.value) }} + onFinish={(values) => createBackfill.mutate(values)} + > + + + + + + + + + + {createBackfill.isSuccess ? ( + + ) : null} + {createBackfill.isError ? ( + + ) : null} + + + + + + + form={manualForm} + layout="vertical" + initialValues={{ target_type: "illust_id", target_id: 0 }} + onFinish={(values) => createManual.mutate(values)} + > + + setStatusFilter(value)} + style={{ width: 200 }} + options={[ + { value: "all", label: "全部" }, + { value: "pending", label: "等待中" }, + { value: "running", label: "运行中" }, + { value: "paused", label: "已暂停" }, + { value: "completed", label: "已完成" }, + { value: "failed", label: "失败" }, + { value: "canceled", label: "已取消" }, + ]} + /> + + + + {runs.isLoading ? ( + + ) : runs.isError ? ( + + ) : !runs.data ? ( + + ) : runs.data.items.length === 0 ? ( + + ) : ( + <> + 请求ID: {runs.data.request_id} + + rowKey={(row) => row.id} + columns={columns} + dataSource={runs.data.items} + pagination={false} + size="small" + scroll={{ x: 1900 }} + style={{ marginTop: 12 }} + /> + + )} + + + ); +} diff --git a/frontend/src/pages/ImagesPage.test.tsx b/frontend/src/pages/ImagesPage.test.tsx new file mode 100644 index 0000000..0314549 --- /dev/null +++ b/frontend/src/pages/ImagesPage.test.tsx @@ -0,0 +1,76 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { ImagesPage } from "./ImagesPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("ImagesPage", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/admin/api/images?limit=50")) { + return new Response( + JSON.stringify({ + ok: true, + items: [ + { + id: "1", + illust_id: "111", + page_index: 0, + ext: "jpg", + status: 1, + width: 100, + height: 200, + orientation: 1, + x_restrict: 0, + ai_type: 1, + illust_type: 0, + bookmark_count: 10, + view_count: 20, + comment_count: 3, + user: { id: "9", name: "u" }, + title: "t", + created_at_pixiv: "2020-01-01T00:00:00Z", + original_url: "https://example.com/1.jpg", + proxy_path: "/i/1.jpg", + tag_count: 1, + missing: [], + }, + ], + next_cursor: "", + request_id: "req_images", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders list", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("图片管理")).toBeInTheDocument(); + expect(await screen.findByText("111")).toBeInTheDocument(); + expect(await screen.findByText("N")).toBeInTheDocument(); + expect(await screen.findByText("Y")).toBeInTheDocument(); + expect(await screen.findByText("插画")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_images/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/ImagesPage.tsx b/frontend/src/pages/ImagesPage.tsx new file mode 100644 index 0000000..db2dc56 --- /dev/null +++ b/frontend/src/pages/ImagesPage.tsx @@ -0,0 +1,377 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Alert, Button, Card, Popconfirm, Select, Skeleton, Space, Table, Tag, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React from "react"; +import { useMemo, useState } from "react"; + +import { ApiError, apiJson } from "../api/client"; + +type ImageItem = { + id: string; + illust_id: string; + page_index: number; + ext: string; + status: number; + width: number | null; + height: number | null; + orientation: number | null; + x_restrict: number | null; + ai_type: number | null; + illust_type: number | null; + bookmark_count: number | null; + view_count: number | null; + comment_count: number | null; + user: { id: string | null; name: string | null }; + title: string | null; + created_at_pixiv: string | null; + original_url: string; + proxy_path: string; + tag_count: number; + missing: string[]; +}; + +type ImagesListResponse = { + ok: true; + items: ImageItem[]; + next_cursor: string; + request_id: string; +}; + +type ManualHydrateResponse = { + ok: true; + created: boolean; + job_id: string; + illust_id: string; + request_id: string; +}; + +type DeleteImageResponse = { + ok: true; + image_id: string; + request_id: string; +}; + +type BulkDeleteImagesResponse = { + ok: true; + requested: number; + deleted: number; + missing: number; + request_id: string; +}; + +type ClearImagesResponse = { + ok: true; + deleted_image_tags: number; + deleted_images: number; + deleted_tags: number; + request_id: string; +}; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +function messageFromError(err: unknown): string { + if (err instanceof ApiError) return err.message; + if (err instanceof Error) return err.message; + return "未知错误"; +} + +const MISSING_LABELS: Record = { + tags: "标签", + geometry: "尺寸", + r18: "R18", + ai: "AI", + illust_type: "类型", + user: "作者", + title: "标题", + created_at: "时间", + popularity: "热度", +}; + +export function ImagesPage() { + const qc = useQueryClient(); + const [missing, setMissing] = useState([]); + const [selectedRowKeys, setSelectedRowKeys] = useState([]); + const [actionAlert, setActionAlert] = useState<{ type: "success" | "error"; message: string; requestId: string | null } | null>(null); + + const query = useQuery({ + queryKey: ["admin", "images", { limit: 50, missing }], + queryFn: () => { + const sp = new URLSearchParams({ limit: "50" }); + for (const key of missing) sp.append("missing", key); + return apiJson(`/admin/api/images?${sp.toString()}`); + }, + }); + + const deleteImage = useMutation({ + mutationFn: (imageId: string) => + apiJson(`/admin/api/images/${encodeURIComponent(imageId)}`, { method: "DELETE" }), + onMutate: () => setActionAlert(null), + onSuccess: (data) => { + setActionAlert({ type: "success", message: `图片已删除：#${data.image_id}`, requestId: data.request_id }); + setSelectedRowKeys((prev) => prev.filter((k) => String(k) !== String(data.image_id))); + qc.invalidateQueries({ queryKey: ["admin", "images"] }); + qc.invalidateQueries({ queryKey: ["admin", "summary"] }); + qc.invalidateQueries({ queryKey: ["public", "tags"] }); + }, + onError: (err) => { + setActionAlert({ type: "error", message: messageFromError(err), requestId: requestIdFromError(err) }); + }, + }); + + const bulkDelete = useMutation({ + mutationFn: (imageIds: string[]) => + apiJson("/admin/api/images/bulk-delete", { + method: "POST", + body: JSON.stringify({ image_ids: imageIds.map((v) => Number(v)) }), + }), + onMutate: () => setActionAlert(null), + onSuccess: (data) => { + setActionAlert({ + type: "success", + message: `批量删除完成：请求 ${data.requested}，成功删除 ${data.deleted}，未找到 ${data.missing}`, + requestId: data.request_id, + }); + setSelectedRowKeys([]); + qc.invalidateQueries({ queryKey: ["admin", "images"] }); + qc.invalidateQueries({ queryKey: ["admin", "summary"] }); + qc.invalidateQueries({ queryKey: ["public", "tags"] }); + }, + onError: (err) => { + setActionAlert({ type: "error", message: messageFromError(err), requestId: requestIdFromError(err) }); + }, + }); + + const clearImages = useMutation({ + mutationFn: () => + apiJson("/admin/api/images/clear", { + method: "POST", + body: JSON.stringify({ confirm: true, delete_tags: true }), + }), + onMutate: () => setActionAlert(null), + onSuccess: (data) => { + setActionAlert({ + type: "success", + message: `图片数据已清空：图片 ${data.deleted_images}，关联 ${data.deleted_image_tags}，标签 ${data.deleted_tags}`, + requestId: data.request_id, + }); + setSelectedRowKeys([]); + qc.invalidateQueries({ queryKey: ["admin", "images"] }); + qc.invalidateQueries({ queryKey: ["admin", "summary"] }); + qc.invalidateQueries({ queryKey: ["public", "tags"] }); + }, + onError: (err) => { + setActionAlert({ type: "error", message: messageFromError(err), requestId: requestIdFromError(err) }); + }, + }); + + const manualHydrate = useMutation({ + mutationFn: (imageId: string) => + apiJson("/admin/api/hydration-runs/manual", { + method: "POST", + body: JSON.stringify({ image_id: Number(imageId) }), + }), + onMutate: () => setActionAlert(null), + onSuccess: (data) => { + setActionAlert({ + type: "success", + message: data.created ? `单图补全任务已创建：任务 #${data.job_id}（作品 ${data.illust_id}）` : `任务已存在：任务 #${data.job_id}（作品 ${data.illust_id}）`, + requestId: data.request_id, + }); + qc.invalidateQueries({ queryKey: ["admin", "jobs"] }); + qc.invalidateQueries({ queryKey: ["admin", "hydration-runs"] }); + }, + onError: (err) => { + setActionAlert({ type: "error", message: err instanceof Error ? err.message : "创建补全任务失败", requestId: requestIdFromError(err) }); + }, + }); + + const columns: ColumnsType = useMemo( + () => [ + { + title: "图片ID", + dataIndex: "id", + key: "id", + width: 100, + render: (value: string, row) => ( + + ), + }, + { title: "作品ID", dataIndex: "illust_id", key: "illust_id", width: 110 }, + { title: "页码", dataIndex: "page_index", key: "page_index", width: 70 }, + { title: "格式", dataIndex: "ext", key: "ext", width: 70 }, + { + title: "尺寸", + key: "geometry", + width: 120, + render: (_, row) => (row.width && row.height ? `${row.width}×${row.height}` : "-"), + }, + { title: "R18", dataIndex: "x_restrict", key: "x_restrict", width: 70, render: (v) => (v == null ? "-" : Number(v) > 0 ? "Y" : "N") }, + { title: "AI", dataIndex: "ai_type", key: "ai_type", width: 60, render: (v) => (v == null ? "-" : Number(v) > 0 ? "Y" : "N") }, + { + title: "类型", + dataIndex: "illust_type", + key: "illust_type", + width: 90, + render: (v) => (v == null ? "-" : Number(v) === 0 ? "插画" : Number(v) === 1 ? "漫画" : Number(v) === 2 ? "动图" : String(v)), + }, + { title: "收藏", dataIndex: "bookmark_count", key: "bookmark_count", width: 90, render: (v) => (v == null ? "-" : String(v)) }, + { title: "浏览", dataIndex: "view_count", key: "view_count", width: 90, render: (v) => (v == null ? "-" : String(v)) }, + { title: "评论", dataIndex: "comment_count", key: "comment_count", width: 90, render: (v) => (v == null ? "-" : String(v)) }, + { title: "标签数", dataIndex: "tag_count", key: "tag_count", width: 80 }, + { + title: "缺失", + dataIndex: "missing", + key: "missing", + width: 220, + render: (values: string[]) => + (values || []).length ? ( + + {values.map((v) => ( + {MISSING_LABELS[v] || v} + ))} + + ) : ( + "-" + ), + }, + { title: "作者", key: "user", width: 140, render: (_, row) => (row.user?.name ? row.user.name : "-") }, + { title: "标题", dataIndex: "title", key: "title", width: 220, render: (v) => v || "-" }, + { title: "Pixiv 发布时间", dataIndex: "created_at_pixiv", key: "created_at_pixiv", width: 180, render: (v) => v || "-" }, + { + title: "操作", + key: "actions", + width: 160, + render: (_, row) => ( + + + deleteImage.mutate(row.id)} + > + + + + ), + }, + ], + [deleteImage, manualHydrate], + ); + + const selectedImageIds = useMemo(() => selectedRowKeys.map((k) => String(k)), [selectedRowKeys]); + + return ( + + + 图片管理 + + + + 缺失筛选: + setFile(e.target.files && e.target.files[0] ? e.target.files[0] : null)} + /> +
+ {file ? `已选择文件：${file.name}` : "未选择文件（可在下方粘贴链接）。"} +
+ + + { + if (file) return Promise.resolve(); + if (String(value || "").trim()) return Promise.resolve(); + return Promise.reject(new Error("请粘贴 Pixiv 原图链接，或上传 .txt 文件")); + }, + }, + ]} + > + + + + + + + + + + + + + + + + + {mutation.isPending ? : null} + + {result ? ( + + + + {result.import_id ? ( + + + + 任务ID: {result.job_id || "-"} + {"executed_inline" in result ? ( + + 前台执行: {result.executed_inline ? "是（可能较慢）" : "否（由 worker 处理）"} + + ) : null} + + {result.executed_inline ? ( + + ) : null} + + ) : null} + + {result.preview.length > 0 ? ( + + `${row.illust_id}_${row.page_index}_${row.ext}`} + columns={previewColumns} + dataSource={result.preview} + pagination={false} + size="small" + /> + + ) : null} + + {result.errors.length > 0 ? ( + + +
`${row.line}_${row.code}`} columns={errorColumns} dataSource={result.errors} pagination={false} size="small" /> + + ) : null} + + ) : null} + + ); +} diff --git a/frontend/src/pages/JobsPage.test.tsx b/frontend/src/pages/JobsPage.test.tsx new file mode 100644 index 0000000..eb62c94 --- /dev/null +++ b/frontend/src/pages/JobsPage.test.tsx @@ -0,0 +1,66 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { JobsPage } from "./JobsPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("JobsPage", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/admin/api/jobs?limit=50&status=failed")) { + return new Response( + JSON.stringify({ + ok: true, + items: [ + { + id: "j1", + type: "proxy_probe", + status: "failed", + priority: 0, + last_error: "forbidden", + locked_by: null, + locked_at: null, + ref_type: null, + ref_id: null, + created_at: "2026-02-11T00:00:00Z", + attempt: 1, + max_attempts: 3, + run_after: null, + updated_at: "2026-02-11T00:00:00Z", + }, + ], + next_cursor: "", + request_id: "req_jobs", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders list", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("任务队列")).toBeInTheDocument(); + expect(await screen.findByText("代理探测")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_jobs/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/JobsPage.tsx b/frontend/src/pages/JobsPage.tsx new file mode 100644 index 0000000..c851d85 --- /dev/null +++ b/frontend/src/pages/JobsPage.tsx @@ -0,0 +1,329 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Alert, Button, Card, Descriptions, Drawer, Select, Skeleton, Space, Table, Tag, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React, { useState } from "react"; + +import { ApiError, apiJson } from "../api/client"; + +type JobItem = { + id: string; + type: string; + status: string; + priority: number; + run_after: string | null; + attempt: number; + max_attempts: number; + last_error: string | null; + locked_by: string | null; + locked_at: string | null; + ref_type: string | null; + ref_id: string | null; + created_at: string; + updated_at: string; +}; + +type JobsListResponse = { + ok: true; + items: JobItem[]; + next_cursor: string; + request_id: string; +}; + +type JobDetailResponse = { + ok: true; + item: JobItem & { payload: unknown; payload_json: string }; + request_id: string; +}; + +type JobActionResponse = { ok: true; job_id: string; status: string; request_id: string }; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +function statusLabel(status: string): string { + switch (status) { + case "pending": + return "等待中"; + case "running": + return "运行中"; + case "paused": + return "已暂停"; + case "canceled": + return "已取消"; + case "completed": + return "已完成"; + case "failed": + return "失败"; + case "dlq": + return "死信"; + default: + return status || "未知"; + } +} + +function typeLabel(type: string): string { + switch (type) { + case "import_images": + return "导入图片"; + case "hydrate_metadata": + return "补全元数据"; + case "proxy_probe": + return "代理探测"; + case "easy_proxies_import": + return "导入 easy-proxies"; + case "heal_url": + return "修复 URL"; + default: + return type || "未知"; + } +} + +function statusTag(status: string) { + const label = statusLabel(status); + const color = + status === "running" + ? "processing" + : status === "pending" + ? "blue" + : status === "completed" + ? "success" + : status === "failed" + ? "error" + : status === "paused" + ? "warning" + : status === "dlq" + ? "magenta" + : "default"; + return {label}; +} + +export function JobsPage() { + const qc = useQueryClient(); + const [status, setStatus] = useState("failed"); + const [type, setType] = useState("all"); + const [detailJobId, setDetailJobId] = useState(null); + const [actionAlert, setActionAlert] = useState<{ type: "success" | "error"; message: string; requestId: string | null } | null>(null); + + const query = useQuery({ + queryKey: ["admin", "jobs", { status, type, limit: 50 }], + queryFn: () => { + const sp = new URLSearchParams({ limit: "50" }); + if (status) sp.set("status", status); + if (type !== "all") sp.set("type", type); + return apiJson(`/admin/api/jobs?${sp.toString()}`); + }, + }); + + const jobDetail = useQuery({ + queryKey: ["admin", "jobs", "detail", detailJobId], + enabled: Boolean(detailJobId), + queryFn: () => apiJson(`/admin/api/jobs/${encodeURIComponent(detailJobId || "")}`), + }); + + const action = useMutation({ + mutationFn: (vars: { jobId: string; action: "retry" | "cancel" | "move-to-dlq" }) => + apiJson(`/admin/api/jobs/${encodeURIComponent(vars.jobId)}/${vars.action}`, { method: "POST" }), + onMutate: () => { + setActionAlert(null); + }, + onSuccess: (data) => { + setActionAlert({ type: "success", message: `操作成功：任务 #${data.job_id} 状态已更新为 ${statusLabel(data.status)}`, requestId: data.request_id }); + qc.invalidateQueries({ queryKey: ["admin", "jobs"] }); + qc.invalidateQueries({ queryKey: ["admin", "jobs", "detail", String(data.job_id)] }); + }, + onError: (err) => { + setActionAlert({ type: "error", message: err instanceof Error ? err.message : "操作失败", requestId: requestIdFromError(err) }); + }, + }); + + const columns: ColumnsType = [ + { + title: "任务ID", + dataIndex: "id", + key: "id", + width: 110, + render: (value: string) => ( + + ), + }, + { title: "类型", dataIndex: "type", key: "type", width: 140, render: (value) => typeLabel(String(value)) }, + { title: "状态", dataIndex: "status", key: "status", width: 110, render: (value) => statusTag(String(value)) }, + { + title: "重试次数", + key: "attempt", + width: 120, + render: (_, row) => `${row.attempt} / ${row.max_attempts}`, + }, + { title: "优先级", dataIndex: "priority", key: "priority", width: 90 }, + { title: "下次执行", dataIndex: "run_after", key: "run_after", width: 180, render: (value) => value || "-" }, + { title: "锁定者", dataIndex: "locked_by", key: "locked_by", width: 140, render: (value) => value || "-" }, + { title: "锁定时间", dataIndex: "locked_at", key: "locked_at", width: 180, render: (value) => value || "-" }, + { + title: "引用", + key: "ref", + width: 160, + render: (_, row) => (row.ref_type ? `${row.ref_type}:${row.ref_id || ""}` : "-"), + }, + { + title: "错误", + dataIndex: "last_error", + key: "last_error", + width: 240, + render: (value) => (value ? String(value) : "-"), + }, + { title: "更新时间", dataIndex: "updated_at", key: "updated_at", width: 180 }, + { + title: "操作", + key: "actions", + width: 240, + render: (_, row) => { + const retryDisabled = row.status === "running"; + const cancelDisabled = row.status === "completed" || row.status === "canceled"; + const dlqDisabled = row.status === "dlq"; + + const isRetryLoading = action.isPending && action.variables?.jobId === row.id && action.variables?.action === "retry"; + const isCancelLoading = action.isPending && action.variables?.jobId === row.id && action.variables?.action === "cancel"; + const isDlqLoading = action.isPending && action.variables?.jobId === row.id && action.variables?.action === "move-to-dlq"; + + return ( + + + + + + ); + }, + }, + ]; + + return ( + + + 任务队列 + + + + 状态: + setType(value)} + options={[ + { value: "all", label: "全部" }, + { value: "import_images", label: "导入图片" }, + { value: "hydrate_metadata", label: "补全元数据" }, + { value: "proxy_probe", label: "代理探测" }, + { value: "easy_proxies_import", label: "导入 easy-proxies" }, + { value: "heal_url", label: "修复 URL" }, + ]} + style={{ width: 200 }} + /> + + + + {actionAlert ? : null} + + + {query.isLoading ? ( + + ) : query.isError ? ( + + ) : !query.data ? ( + + ) : query.data.items.length === 0 ? ( + + ) : ( + <> + 请求ID: {query.data.request_id} + + rowKey={(row) => row.id} + columns={columns} + dataSource={query.data.items} + pagination={false} + size="small" + scroll={{ x: 1600 }} + style={{ marginTop: 12 }} + /> + + )} + + + setDetailJobId(null)} + title={detailJobId ? `任务详情 #${detailJobId}` : "任务详情"} + width={720} + > + {jobDetail.isLoading ? ( + + ) : jobDetail.isError ? ( + + ) : !jobDetail.data ? ( + + ) : ( + + 请求ID: {jobDetail.data.request_id} + + #{jobDetail.data.item.id} + {typeLabel(jobDetail.data.item.type)} + {statusLabel(jobDetail.data.item.status)} + {jobDetail.data.item.priority} + {`${jobDetail.data.item.attempt} / ${jobDetail.data.item.max_attempts}`} + {jobDetail.data.item.run_after || "-"} + {jobDetail.data.item.locked_by || "-"} + {jobDetail.data.item.locked_at || "-"} + {jobDetail.data.item.ref_type || "-"} + {jobDetail.data.item.ref_id || "-"} + {jobDetail.data.item.created_at} + {jobDetail.data.item.updated_at} + + {jobDetail.data.item.last_error || "-"} + + + + + 
+                {JSON.stringify(jobDetail.data.item.payload, null, 2)}
+
+ + + )} + + + ); +} diff --git a/frontend/src/pages/LoginPage.test.tsx b/frontend/src/pages/LoginPage.test.tsx new file mode 100644 index 0000000..ae056dd --- /dev/null +++ b/frontend/src/pages/LoginPage.test.tsx @@ -0,0 +1,86 @@ +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import React from "react"; +import { MemoryRouter } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { getAdminToken } from "../auth/tokenStorage"; +import { LoginPage } from "./LoginPage"; + +describe("LoginPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + localStorage.clear(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/admin/api/login")) { + return new Response(JSON.stringify({ ok: true, token: "tok_admin", request_id: "req_login" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("logs in and stores token", async () => { + render( + + + , + ); + + fireEvent.change(screen.getByPlaceholderText("请输入用户名"), { target: { value: "admin" } }); + fireEvent.change(screen.getByPlaceholderText("请输入密码"), { target: { value: "pw" } }); + fireEvent.click(screen.getByRole("button", { name: /登\s*录/ })); + + await waitFor(() => { + expect(getAdminToken()).toBe("tok_admin"); + }); + expect(await screen.findByText(/请求ID:\s*req_login/)).toBeInTheDocument(); + expect(screen.queryByText("tok_admin")).not.toBeInTheDocument(); + }); + + it("shows error and request_id on failure", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/admin/api/login")) { + return new Response( + JSON.stringify({ ok: false, code: "UNAUTHORIZED", message: "bad", request_id: "req_bad", details: {} }), + { status: 401, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + + render( + + + , + ); + + fireEvent.change(screen.getByPlaceholderText("请输入用户名"), { target: { value: "admin" } }); + fireEvent.change(screen.getByPlaceholderText("请输入密码"), { target: { value: "pw" } }); + fireEvent.click(screen.getByRole("button", { name: /登\s*录/ })); + + expect(await screen.findByText(/请求ID:\s*req_bad/)).toBeInTheDocument(); + expect(getAdminToken()).toBeNull(); + }); +}); + + diff --git a/frontend/src/pages/LoginPage.tsx b/frontend/src/pages/LoginPage.tsx new file mode 100644 index 0000000..3971b53 --- /dev/null +++ b/frontend/src/pages/LoginPage.tsx @@ -0,0 +1,106 @@ +import { Alert, Button, Card, Form, Input, Space, Typography } from "antd"; +import React, { useState } from "react"; +import { useLocation, useNavigate } from "react-router-dom"; + +import { ApiError, apiJson } from "../api/client"; +import { setAdminToken } from "../auth/tokenStorage"; + +type LoginFormValues = { + username: string; + password: string; +}; + +type LoginResponse = { + ok: true; + token: string; + request_id: string; +}; + +export function LoginPage() { + const navigate = useNavigate(); + const location = useLocation(); + const [form] = Form.useForm(); + const [errorMessage, setErrorMessage] = useState(null); + const [requestId, setRequestId] = useState(null); + const [loading, setLoading] = useState(false); + + const searchParams = new URLSearchParams(location.search); + const reason = String(searchParams.get("reason") || "").trim(); + const next = String(searchParams.get("next") || "").trim(); + + const reasonMessage = + reason === "unauthorized" + ? "登录已失效，请重新登录。" + : reason === "missing_token" + ? "请先登录后再访问管理后台。" + : null; + + const onFinish = async (values: LoginFormValues) => { + setLoading(true); + setErrorMessage(null); + setRequestId(null); + + try { + const resp = await apiJson("/admin/api/login", { + method: "POST", + body: JSON.stringify({ username: values.username, password: values.password }), + }); + + setAdminToken(resp.token); + setRequestId(resp.request_id); + form.resetFields(["password"]); + + const nextPath = next && next.startsWith("/admin") && !next.startsWith("/admin/login") ? next : "/admin"; + navigate(nextPath, { replace: true }); + } catch (err: unknown) { + if (err instanceof ApiError) { + setErrorMessage(err.message); + setRequestId(err.body?.request_id ? String(err.body.request_id) : null); + } else if (err instanceof Error) { + setErrorMessage(err.message); + } else { + setErrorMessage("登录失败"); + } + } finally { + setLoading(false); + } + }; + + return ( +
+ + + + 管理后台登录 + + + {reasonMessage ? : null} + {errorMessage ? : null} + {requestId ? 请求ID: {requestId} : null} + + form={form} layout="vertical" onFinish={onFinish}> + + + + + + + + + + + + +
+ ); +} diff --git a/frontend/src/pages/PlaceholderPage.tsx b/frontend/src/pages/PlaceholderPage.tsx new file mode 100644 index 0000000..6787317 --- /dev/null +++ b/frontend/src/pages/PlaceholderPage.tsx @@ -0,0 +1,6 @@ +import { Alert } from "antd"; +import React from "react"; + +export function PlaceholderPage({ title }: { title: string }) { + return ; +} diff --git a/frontend/src/pages/PlaygroundPage.test.tsx b/frontend/src/pages/PlaygroundPage.test.tsx new file mode 100644 index 0000000..596869f --- /dev/null +++ b/frontend/src/pages/PlaygroundPage.test.tsx @@ -0,0 +1,93 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import React from "react"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { PlaygroundPage } from "./PlaygroundPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("PlaygroundPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes("/random") && url.includes("format=image")) { + return new Response( + JSON.stringify({ + ok: false, + code: "NO_MATCH", + message: "No matching image.", + request_id: "req_nomatch", + details: { + hints: { + applied_filters: { r18: 0, r18_strict: 1, orientation: "any", min_width: 0, min_height: 0, min_pixels: 0 }, + suggestions: ["运行元数据补全任务以提升元数据覆盖率"], + }, + }, + }), + { status: 404, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.includes("/random") && url.includes("format=json")) { + return new Response( + JSON.stringify({ ok: true, request_id: "req_play", data: { urls: { proxy: "/i/1.jpg" } } }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("runs and shows request_id", async () => { + const qc = makeClient(); + render( + + + + } /> + + + , + ); + + expect(await screen.findByText("随机接口调试")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "开始请求" })); + expect(await screen.findByText(/请求ID:\s*req_play/)).toBeInTheDocument(); + }); + + it("shows NO_MATCH hints in image mode and keeps message Chinese", async () => { + const qc = makeClient(); + render( + + + + } /> + + + , + ); + + expect(await screen.findByText("随机接口调试")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "开始请求" })); + + expect(await screen.findByText("没有匹配的图片")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_nomatch/)).toBeInTheDocument(); + expect(await screen.findByText("运行元数据补全任务以提升元数据覆盖率")).toBeInTheDocument(); + expect(await screen.findByText("本次筛选条件：")).toBeInTheDocument(); + expect(await screen.findByText(/r18_strict/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/PlaygroundPage.tsx b/frontend/src/pages/PlaygroundPage.tsx new file mode 100644 index 0000000..135dc68 --- /dev/null +++ b/frontend/src/pages/PlaygroundPage.tsx @@ -0,0 +1,542 @@ +import { useMutation } from "@tanstack/react-query"; +import { Alert, Button, Card, Col, Form, Input, InputNumber, Row, Select, Skeleton, Space, Switch, Typography } from "antd"; +import React, { useEffect } from "react"; +import { useLocation } from "react-router-dom"; + +import { ApiError, type ApiErrorBody, apiFetch, apiJson, formatApiErrorMessage } from "../api/client"; + +type PlaygroundFormValues = { + format: "image" | "json" | "redirect"; + attempts: number | null; + seed: string; + strategy: "default" | "quality" | "random"; + quality_samples: number | null; + r18: 0 | 1 | 2; + r18_strict: "default" | "1" | "0"; + adaptive: boolean; + layout: "any" | "portrait" | "landscape" | "square"; + min_width: number | null; + min_height: number | null; + min_pixels: number | null; + min_bookmarks: number | null; + min_views: number | null; + min_comments: number | null; + included_tags: string; + excluded_tags: string; + user_id: number | null; + illust_id: number | null; + ai_type: "any" | "0" | "1"; + illust_type: "any" | "illust" | "manga" | "ugoira"; + created_from: string; + created_to: string; +}; + +type RandomJsonResponse = { + ok: true; + request_id: string; + data: { urls?: { proxy?: string } }; +}; + +type PlaygroundResult = + | { kind: "json"; url: string; request_id: string; payload: RandomJsonResponse } + | { kind: "image"; url: string; request_id: string | null; src: string; headers: Record } + | { kind: "redirect"; url: string; request_id: string | null; location: string | null; note: string | null }; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +function buildRandomUrl(values: PlaygroundFormValues): string { + const sp = new URLSearchParams(); + + if (values.attempts != null) sp.set("attempts", String(values.attempts)); + if (values.seed.trim()) sp.set("seed", values.seed.trim()); + + if (values.strategy !== "default") sp.set("strategy", values.strategy); + if (values.strategy === "quality" && values.quality_samples != null) sp.set("quality_samples", String(values.quality_samples)); + + sp.set("r18", String(values.r18)); + if (values.r18_strict !== "default") sp.set("r18_strict", values.r18_strict); + if (values.adaptive) sp.set("adaptive", "1"); + if (values.layout !== "any") sp.set("layout", values.layout); + + if (values.min_width != null) sp.set("min_width", String(values.min_width)); + if (values.min_height != null) sp.set("min_height", String(values.min_height)); + if (values.min_pixels != null) sp.set("min_pixels", String(values.min_pixels)); + if (values.min_bookmarks != null) sp.set("min_bookmarks", String(values.min_bookmarks)); + if (values.min_views != null) sp.set("min_views", String(values.min_views)); + if (values.min_comments != null) sp.set("min_comments", String(values.min_comments)); + + if (values.included_tags.trim()) sp.set("included_tags", values.included_tags.trim()); + if (values.excluded_tags.trim()) sp.set("excluded_tags", values.excluded_tags.trim()); + + if (values.user_id != null) sp.set("user_id", String(values.user_id)); + if (values.illust_id != null) sp.set("illust_id", String(values.illust_id)); + + if (values.ai_type !== "any") sp.set("ai_type", values.ai_type); + if (values.illust_type !== "any") sp.set("illust_type", values.illust_type); + if (values.created_from.trim()) sp.set("created_from", values.created_from.trim()); + if (values.created_to.trim()) sp.set("created_to", values.created_to.trim()); + + if (values.format === "redirect") { + sp.set("redirect", "1"); + sp.set("format", "image"); + } else { + sp.set("format", values.format); + } + + const qs = sp.toString(); + return qs ? `/random?${qs}` : "/random"; +} + +function pickHeaders(headers: Headers, keys: string[]): Record { + const out: Record = {}; + for (const key of keys) { + const v = headers.get(key); + if (v) out[key] = v; + } + return out; +} + +function isApiErrorBody(value: unknown): value is ApiErrorBody { + if (!value || typeof value !== "object") return false; + const v = value as Record; + return ( + v.ok === false && + typeof v.code === "string" && + typeof v.message === "string" && + typeof v.request_id === "string" && + !!v.details && + typeof v.details === "object" + ); +} + +async function parseApiErrorFromResponse(resp: Response): Promise { + const contentType = resp.headers.get("content-type") || ""; + if (contentType.toLowerCase().includes("application/json")) { + try { + const raw = (await resp.json()) as unknown; + const body = isApiErrorBody(raw) ? raw : null; + const message = formatApiErrorMessage(resp.status, body); + return new ApiError(message, { status: resp.status, body }); + } catch { + // fall through + } + } + return new ApiError(`HTTP ${resp.status}`, { status: resp.status }); +} + +function hintsFromError(err: unknown): { appliedFilters: Record | null; suggestions: string[] } | null { + if (!(err instanceof ApiError)) return null; + const details = err.body?.details; + if (!details || typeof details !== "object") return null; + const hintsRaw = (details as Record)["hints"]; + if (!hintsRaw || typeof hintsRaw !== "object" || Array.isArray(hintsRaw)) return null; + + const hints = hintsRaw as Record; + const appliedFiltersRaw = hints["applied_filters"]; + const suggestionsRaw = hints["suggestions"]; + + const appliedFilters = + appliedFiltersRaw && typeof appliedFiltersRaw === "object" && !Array.isArray(appliedFiltersRaw) + ? (appliedFiltersRaw as Record) + : null; + + const suggestions = Array.isArray(suggestionsRaw) + ? suggestionsRaw.map((v) => String(v || "").trim()).filter((v) => v.length > 0) + : []; + + return appliedFilters || suggestions.length > 0 ? { appliedFilters, suggestions } : null; +} + +async function fetchPlayground(values: PlaygroundFormValues): Promise { + const url = buildRandomUrl(values); + + if (values.format === "json") { + const payload = await apiJson(url); + return { kind: "json", url, request_id: payload.request_id, payload }; + } + + if (values.format === "image") { + const resp = await apiFetch(url, { method: "GET" }); + const reqId = resp.headers.get("x-request-id") || resp.headers.get("x-request_id") || null; + if (!resp.ok) throw await parseApiErrorFromResponse(resp); + + const blob = await resp.blob(); + if (!("createObjectURL" in URL)) { + throw new Error("当前浏览器不支持 URL.createObjectURL"); + } + const src = URL.createObjectURL(blob); + const headers = pickHeaders(resp.headers, [ + "content-type", + "cache-control", + "content-disposition", + "x-origin-url", + "x-request-id", + ]); + + return { kind: "image", url, request_id: reqId, src, headers }; + } + + const resp = await apiFetch(url, { method: "GET", redirect: "manual" }); + const reqId = resp.headers.get("x-request-id") || resp.headers.get("x-request_id") || null; + const location = resp.headers.get("location"); + if (location) return { kind: "redirect", url, request_id: reqId, location, note: null }; + + const fallbackUrl = url.replace(/[?&]redirect=1(&|$)/, "$1"); + const payload = await apiJson(fallbackUrl.replace(/([?&])format=image(&|$)/, "$1format=json$2")); + const proxy = payload.data?.urls?.proxy || null; + return { + kind: "redirect", + url, + request_id: reqId ?? payload.request_id, + location: proxy, + note: proxy ? "未获取到 Location 响应头，已回退为结构化返回中的代理链接。" : "未获取到 Location 响应头。", + }; +} + +async function copyText(text: string): Promise { + if (navigator.clipboard?.writeText) { + await navigator.clipboard.writeText(text); + return; + } + const el = document.createElement("textarea"); + el.value = text; + el.setAttribute("readonly", ""); + el.style.position = "fixed"; + el.style.left = "-9999px"; + document.body.appendChild(el); + el.select(); + document.execCommand("copy"); + document.body.removeChild(el); +} + +export function PlaygroundPage() { + const location = useLocation(); + const [form] = Form.useForm(); + + const m = useMutation({ mutationFn: fetchPlayground }); + + useEffect(() => { + const sp = new URLSearchParams(location.search); + const format = sp.get("format"); + const includedTags = sp.get("included_tags"); + + const updates: Partial = {}; + if (format === "image" || format === "json" || format === "redirect") updates.format = format; + if (includedTags != null) updates.included_tags = includedTags; + + if (Object.keys(updates).length > 0) form.setFieldsValue(updates); + }, [form, location.search]); + + useEffect(() => { + if (m.data?.kind !== "image") return; + const src = m.data.src; + return () => { + URL.revokeObjectURL(src); + }; + }, [m.data]); + + const url = m.data?.url || null; + + const onCopyUrl = async () => { + if (!url) return; + await copyText(url); + }; + + const onCopyCurl = async () => { + if (!url) return; + await copyText(`curl -i ${JSON.stringify(url)}`); + }; + + const onOpen = () => { + if (!url) return; + window.open(url, "_blank", "noopener,noreferrer"); + }; + + return ( + + + 随机接口调试 + + + + + + + form={form} + layout="vertical" + initialValues={{ + format: "json", + attempts: null, + seed: "", + strategy: "default", + quality_samples: null, + r18: 0, + r18_strict: "default", + adaptive: false, + layout: "any", + min_width: null, + min_height: null, + min_pixels: null, + min_bookmarks: null, + min_views: null, + min_comments: null, + included_tags: "", + excluded_tags: "", + user_id: null, + illust_id: null, + ai_type: "any", + illust_type: "any", + created_from: "", + created_to: "", + }} + onFinish={(v) => m.mutate(v)} + > + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {m.isPending ? ( + + ) : m.isError ? ( + { + const rid = requestIdFromError(m.error); + const hints = hintsFromError(m.error); + if (!rid && !hints) return ""; + return ( + + {rid ? 请求ID: {rid} : null} + {hints?.suggestions && hints.suggestions.length > 0 ? ( + <> + 建议： +
    + {hints.suggestions.map((s) => ( +
  • {s}
    • + ))} +
+ + ) : null} + {hints?.appliedFilters ? ( + <> + 本次筛选条件： + 
+                              {JSON.stringify(hints.appliedFilters, null, 2)}
+
+ + ) : null} + + ); + })() + } + /> + ) : m.isSuccess ? ( + + 请求ID: {m.data.request_id || "-"} + 请求链接: {m.data.url} + + {m.data.kind === "json" ? ( + 
+                    {JSON.stringify(m.data.payload, null, 2)}
+
+ ) : m.data.kind === "image" ? ( + <> + 随机图片 + 
{JSON.stringify(m.data.headers, null, 2)}
+ + ) : ( + <> + + 跳转地址: {m.data.location ? m.data.location : "（未提供）"} + + {m.data.note ? : null} + + )} + + ) : ( + + )} + + + + + + + + + + + + + + + + ); +} + diff --git a/frontend/src/pages/ProxiesPage.test.tsx b/frontend/src/pages/ProxiesPage.test.tsx new file mode 100644 index 0000000..98d7646 --- /dev/null +++ b/frontend/src/pages/ProxiesPage.test.tsx @@ -0,0 +1,218 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import React from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ProxiesPage } from "./ProxiesPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("ProxiesPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + let endpointsCalls = 0; + let endpointEnabled = true; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/admin/api/proxy-pools")) { + return new Response( + JSON.stringify({ + ok: true, + items: [{ id: "1", name: "pixiv", description: null, enabled: true }], + request_id: "req_pools_1", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/proxies/endpoints")) { + endpointsCalls += 1; + return new Response( + JSON.stringify({ + ok: true, + items: [ + { + id: "1", + uri_masked: "http://***:***@1.2.3.4:8080", + source: "manual", + source_ref: null, + enabled: endpointEnabled, + latency_ms: 123, + status: "ok", + blacklisted_until: null, + last_error: null, + success_count: 10, + failure_count: 2, + last_ok_at: "2026-02-13T00:00:00Z", + last_fail_at: null, + pools: [{ id: "1", name: "pixiv", pool_enabled: true, member_enabled: true, weight: 1 }], + bindings: { primary_count: 1, override_count: 0 }, + }, + ], + request_id: `req_proxies_${endpointsCalls}`, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/proxies/endpoints/1") && init?.method === "PUT") { + const body = init?.body ? JSON.parse(String(init.body)) : {}; + endpointEnabled = Boolean(body.enabled); + return new Response(JSON.stringify({ ok: true, endpoint_id: "1", enabled: endpointEnabled, request_id: "req_toggle" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.endsWith("/admin/api/proxies/endpoints/1/reset-failures") && init?.method === "POST") { + return new Response(JSON.stringify({ ok: true, endpoint_id: "1", request_id: "req_reset" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.endsWith("/admin/api/proxies/endpoints/import")) { + expect(init?.method).toBe("POST"); + const body = init?.body ? JSON.parse(String(init.body)) : {}; + expect(body.conflict_policy).toBe("overwrite"); + expect(String(body.text || "")).toContain("http://u:pa@ss@1.2.3.4:8080"); + expect(String(body.text || "")).toContain("socks5://5.6.7.8:1080"); + return new Response(JSON.stringify({ ok: true, created: 2, updated: 0, skipped: 0, errors: [], request_id: "req_manual" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.endsWith("/admin/api/proxies/easy-proxies/import")) { + expect(init?.method).toBe("POST"); + const body = init?.body ? JSON.parse(String(init.body)) : {}; + expect(body.base_url).toBe("http://easy.test"); + expect(body.password).toBe("pw_test"); + expect(body.conflict_policy).toBe("skip_non_easy_proxies"); + expect(body.attach_pool_id).toBe(1); + expect(body.attach_weight).toBe(1); + expect(body.recompute_bindings).toBe(true); + expect(body.max_tokens_per_proxy).toBe(2); + expect(body.strict).toBe(false); + return new Response( + JSON.stringify({ ok: true, created: 1, updated: 0, skipped: 0, errors: [], request_id: "req_easy" }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/proxies/probe")) { + expect(init?.method).toBe("POST"); + return new Response(JSON.stringify({ ok: true, job_id: "job_1", request_id: "req_probe" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders list", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("代理管理")).toBeInTheDocument(); + expect(await screen.findByText("http://***:***@1.2.3.4:8080")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_proxies_1/)).toBeInTheDocument(); + }); + + it("imports from easy-proxies", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("代理管理")).toBeInTheDocument(); + fireEvent.change(screen.getByPlaceholderText("http://easy-proxies:15666"), { target: { value: "http://easy.test" } }); + fireEvent.change(screen.getByPlaceholderText("可选"), { target: { value: "pw_test" } }); + fireEvent.click(screen.getByRole("button", { name: "开始导入" })); + + expect(await screen.findByText("外部代理服务导入完成")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_easy/)).toBeInTheDocument(); + }); + + it("imports manual endpoints", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("代理管理")).toBeInTheDocument(); + fireEvent.change(screen.getByPlaceholderText("http://user:pass@1.2.3.4:8080"), { + target: { value: "http://u:pa@ss@1.2.3.4:8080\nsocks5://5.6.7.8:1080\n" }, + }); + + const form = document.querySelector("form"); + expect(form).not.toBeNull(); + fireEvent.submit(form as HTMLFormElement); + + expect(await screen.findByText("手动导入完成")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_manual/)).toBeInTheDocument(); + }); + + it("enqueues proxy probe job", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("代理管理")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "启动健康探测任务" })); + + expect(await screen.findByText("探测任务已入队")).toBeInTheDocument(); + expect(await screen.findByText(/任务ID:\s*job_1/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_probe/)).toBeInTheDocument(); + }); + + it("toggles endpoint enabled", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("代理管理")).toBeInTheDocument(); + expect(await screen.findByText("http://***:***@1.2.3.4:8080")).toBeInTheDocument(); + + fireEvent.click(await screen.findByRole("button", { name: /禁\s*用/ })); + expect(await screen.findByText(/代理节点已禁用：1/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_toggle/)).toBeInTheDocument(); + expect(await screen.findByText("否")).toBeInTheDocument(); + }); + + it("resets endpoint failures", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("代理管理")).toBeInTheDocument(); + fireEvent.click(await screen.findByRole("button", { name: "解除拉黑" })); + + expect(await screen.findByText(/代理节点已重置失败并解除拉黑：1/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_reset/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/ProxiesPage.tsx b/frontend/src/pages/ProxiesPage.tsx new file mode 100644 index 0000000..6f375bb --- /dev/null +++ b/frontend/src/pages/ProxiesPage.tsx @@ -0,0 +1,843 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Alert, Button, Card, Form, Input, InputNumber, Select, Skeleton, Space, Switch, Table, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React, { useEffect, useState } from "react"; + +import { ApiError, apiJson } from "../api/client"; + +type ProxyEndpointItem = { + id: string; + uri_masked: string; + source: string; + source_ref: string | null; + enabled: boolean; + scheme?: string; + host?: string; + port?: number; + invalid_host?: boolean; + latency_ms: number | null; + status: string | null; + blacklisted_until: string | null; + last_error: string | null; + success_count: number; + failure_count: number; + last_ok_at: string | null; + last_fail_at: string | null; + pools: Array<{ + id: string; + name: string; + pool_enabled: boolean; + member_enabled: boolean; + weight: number; + }>; + bindings: { primary_count: number; override_count: number }; +}; + +type ProxiesEndpointsResponse = { + ok: true; + items: ProxyEndpointItem[]; + request_id: string; +}; + +type ProxyPoolItem = { + id: string; + name: string; + description: string | null; + enabled: boolean; +}; + +type ProxyPoolsListResponse = { + ok: true; + items: ProxyPoolItem[]; + request_id: string; +}; + +type ManualImportFormValues = { + text: string; + source: "manual"; + conflict_policy: "overwrite" | "skip"; +}; + +type ManualImportResponse = { + ok: true; + created: number; + updated: number; + skipped: number; + errors: Array<{ line: number; code: string; message: string }>; + request_id: string; +}; + +type EasyProxiesImportFormValues = { + base_url: string; + password: string; + conflict_policy: "overwrite" | "skip_non_easy_proxies"; + bootstrap: boolean; + host_override?: string; + attach_pool_id?: number; + attach_weight?: number; + recompute_bindings?: boolean; + max_tokens_per_proxy?: number; + strict?: boolean; +}; + +type EasyProxiesImportResponse = { + ok: true; + created: number; + updated: number; + skipped: number; + errors: Array<{ code: string; message: string }>; + warnings?: string[]; + attach?: { pool_id: string; endpoints_total: number; created: number; updated: number }; + bindings?: { pool_id: string; recomputed: number } & Record; + request_id: string; +}; + +type ProxiesProbeResponse = { + ok: true; + job_id: string; + request_id: string; +}; + +type UpdateProxyEndpointResponse = { + ok: true; + endpoint_id: string; + enabled: boolean; + request_id: string; +}; + +type ResetProxyEndpointFailuresResponse = { + ok: true; + endpoint_id: string; + request_id: string; +}; + +type CleanupInvalidHostsResponse = { + ok: true; + dry_run: boolean; + invalid_hosts: string[]; + matched: number; + disabled?: number; + memberships_removed?: number; + overrides_cleared?: number; + deleted?: number; + warnings?: string[]; + request_id: string; +}; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +const columns = (actions: { + onToggleEnabled: (row: ProxyEndpointItem) => void; + onResetFailures: (row: ProxyEndpointItem) => void; + updatePendingId: string | null; + resetPendingId: string | null; +}): ColumnsType => [ + { title: "节点ID", dataIndex: "id", key: "id" }, + { title: "代理地址（掩码）", dataIndex: "uri_masked", key: "uri_masked" }, + { + title: "主机", + key: "host", + render: (_, row) => { + const host = String(row.host || "").trim(); + const port = typeof row.port === "number" && Number.isFinite(row.port) ? row.port : null; + const invalid = Boolean(row.invalid_host); + const text = host ? (port ? `${host}:${port}` : host) : "-"; + return invalid ? {text}（占位） : text; + }, + }, + { + title: "来源", + dataIndex: "source", + key: "source", + render: (value: string) => { + const v = String(value || "").trim(); + if (v === "manual") return "手动"; + if (v === "easy_proxies") return "easy-proxies"; + return v || "-"; + }, + }, + { + title: "来源引用", + dataIndex: "source_ref", + key: "source_ref", + render: (value: string | null) => + value ? ( + + {String(value || "")} + + ) : ( + "-" + ), + }, + { title: "启用", dataIndex: "enabled", key: "enabled", render: (value) => (value ? "是" : "否") }, + { + title: "状态", + dataIndex: "status", + key: "status", + render: (value: string | null) => { + const normalized = String(value || "").trim().toLowerCase(); + if (!normalized) return "-"; + if (normalized === "ok") return "正常"; + if (normalized === "fail" || normalized === "failed") return "异常"; + if (normalized === "timeout") return "超时"; + return value; + }, + }, + { + title: "所属代理池", + key: "pools", + render: (_, row) => + (row.pools || []).length + ? (row.pools || []) + .map((pool) => `${pool.name}(#${pool.id}) 权重=${pool.weight} ${pool.pool_enabled && pool.member_enabled ? "启用" : "停用"}`) + .join("，") + : "-", + }, + { + title: "绑定统计", + key: "bindings", + render: (_, row) => + row.bindings ? `主绑定=${row.bindings.primary_count}，覆盖绑定=${row.bindings.override_count}` : "-", + }, + { title: "成功/失败", key: "ok_fail", render: (_, row) => `${row.success_count}/${row.failure_count}` }, + { title: "最近成功", dataIndex: "last_ok_at", key: "last_ok_at", render: (value) => value || "-" }, + { title: "最近失败", dataIndex: "last_fail_at", key: "last_fail_at", render: (value) => value || "-" }, + { title: "延迟(ms)", dataIndex: "latency_ms", key: "latency_ms" }, + { title: "黑名单至", dataIndex: "blacklisted_until", key: "blacklisted_until" }, + { title: "最后错误", dataIndex: "last_error", key: "last_error" }, + { + title: "操作", + key: "actions", + render: (_, row) => ( + + + + + ), + }, +]; + +export function ProxiesPage() { + const queryClient = useQueryClient(); + + const query = useQuery({ + queryKey: ["admin", "proxies", "endpoints"], + queryFn: () => apiJson("/admin/api/proxies/endpoints"), + }); + + const poolsQuery = useQuery({ + queryKey: ["admin", "proxy-pools"], + queryFn: () => apiJson("/admin/api/proxy-pools"), + }); + + const [manualErrorMessage, setManualErrorMessage] = useState(null); + const [manualRequestId, setManualRequestId] = useState(null); + const [manualResult, setManualResult] = useState(null); + const [manualForm] = Form.useForm(); + + const [probeErrorMessage, setProbeErrorMessage] = useState(null); + const [probeRequestId, setProbeRequestId] = useState(null); + const [probeJobId, setProbeJobId] = useState(null); + const [probeUrl, setProbeUrl] = useState("https://www.pixiv.net/robots.txt"); + const [probeTimeoutMs, setProbeTimeoutMs] = useState(8000); + const [probeConcurrency, setProbeConcurrency] = useState(10); + + const [easyErrorMessage, setEasyErrorMessage] = useState(null); + const [easyRequestId, setEasyRequestId] = useState(null); + const [easyResult, setEasyResult] = useState(null); + const [easyForm] = Form.useForm(); + + useEffect(() => { + const enabledPools = (poolsQuery.data?.items || []).filter((p) => Boolean(p.enabled)); + if (enabledPools.length <= 0) return; + const current = easyForm.getFieldValue("attach_pool_id"); + if (typeof current === "number" && Number.isFinite(current) && current > 0) return; + const firstId = Number(enabledPools[0].id); + if (!Number.isFinite(firstId) || firstId <= 0) return; + easyForm.setFieldValue("attach_pool_id", firstId); + }, [poolsQuery.data, easyForm]); + + const [actionMessage, setActionMessage] = useState(null); + const [actionRequestId, setActionRequestId] = useState(null); + const [actionWarningMessage, setActionWarningMessage] = useState(null); + const [actionErrorMessage, setActionErrorMessage] = useState(null); + const [actionErrorRequestId, setActionErrorRequestId] = useState(null); + + const [cleanupRecomputeBindings, setCleanupRecomputeBindings] = useState(true); + const [cleanupDeleteOrphans, setCleanupDeleteOrphans] = useState(true); + const [cleanupMaxTokensPerProxy, setCleanupMaxTokensPerProxy] = useState(2); + const [cleanupStrict, setCleanupStrict] = useState(false); + + const updateEndpoint = useMutation({ + mutationFn: (payload: { endpointId: string; enabled: boolean }) => + apiJson(`/admin/api/proxies/endpoints/${encodeURIComponent(payload.endpointId)}`, { + method: "PUT", + body: JSON.stringify({ enabled: payload.enabled }), + }), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionWarningMessage(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + setActionMessage(`代理节点已${data.enabled ? "启用" : "禁用"}：${data.endpoint_id}`); + setActionRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "proxies", "endpoints"] }); + }, + onError: (err) => { + if (err instanceof ApiError) { + setActionErrorMessage(err.message); + setActionErrorRequestId(requestIdFromError(err)); + return; + } + if (err instanceof Error) { + setActionErrorMessage(err.message); + return; + } + setActionErrorMessage("代理节点更新失败"); + }, + }); + + const resetEndpointFailures = useMutation({ + mutationFn: (payload: { endpointId: string }) => + apiJson( + `/admin/api/proxies/endpoints/${encodeURIComponent(payload.endpointId)}/reset-failures`, + { + method: "POST", + body: JSON.stringify({}), + }, + ), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionWarningMessage(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + setActionMessage(`代理节点已重置失败并解除拉黑：${data.endpoint_id}`); + setActionRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "proxies", "endpoints"] }); + }, + onError: (err) => { + if (err instanceof ApiError) { + setActionErrorMessage(err.message); + setActionErrorRequestId(requestIdFromError(err)); + return; + } + if (err instanceof Error) { + setActionErrorMessage(err.message); + return; + } + setActionErrorMessage("重置失败/解除拉黑失败"); + }, + }); + + const cleanupInvalidHosts = useMutation({ + mutationFn: () => + apiJson("/admin/api/proxies/endpoints/cleanup-invalid-hosts", { + method: "POST", + body: JSON.stringify({ + recompute_bindings: cleanupRecomputeBindings, + delete_orphans: cleanupDeleteOrphans, + max_tokens_per_proxy: Math.max(1, Math.trunc(cleanupMaxTokensPerProxy || 2)), + strict: cleanupStrict, + }), + }), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionWarningMessage(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + const parts = [ + `匹配: ${data.matched}`, + typeof data.disabled === "number" ? `禁用: ${data.disabled}` : null, + typeof data.memberships_removed === "number" ? `移除池成员: ${data.memberships_removed}` : null, + typeof data.overrides_cleared === "number" ? `清理覆盖绑定: ${data.overrides_cleared}` : null, + typeof data.deleted === "number" ? `删除: ${data.deleted}` : null, + ].filter(Boolean); + setActionMessage(`清理完成（${parts.join("，")}）`); + setActionRequestId(data.request_id); + if (Array.isArray(data.warnings) && data.warnings.length) { + setActionWarningMessage(data.warnings.slice(0, 10).join("；")); + } + queryClient.invalidateQueries({ queryKey: ["admin", "proxies", "endpoints"] }); + queryClient.invalidateQueries({ queryKey: ["admin", "bindings"] }); + }, + onError: (err) => { + if (err instanceof ApiError) { + setActionErrorMessage(err.message); + setActionErrorRequestId(requestIdFromError(err)); + return; + } + if (err instanceof Error) { + setActionErrorMessage(err.message); + return; + } + setActionErrorMessage("清理失败"); + }, + }); + + const manualImport = useMutation({ + mutationFn: (values: ManualImportFormValues) => + apiJson("/admin/api/proxies/endpoints/import", { + method: "POST", + body: JSON.stringify(values), + }), + onMutate: () => { + setManualErrorMessage(null); + setManualRequestId(null); + setManualResult(null); + }, + onSuccess: (data) => { + setManualResult(data); + setManualRequestId(data.request_id); + manualForm.setFieldValue("text", ""); + queryClient.invalidateQueries({ queryKey: ["admin", "proxies", "endpoints"] }); + }, + onError: (err) => { + if (err instanceof ApiError) { + setManualErrorMessage(err.message); + setManualRequestId(requestIdFromError(err)); + return; + } + if (err instanceof Error) { + setManualErrorMessage(err.message); + return; + } + setManualErrorMessage("手动导入失败"); + }, + }); + + const probe = useMutation({ + mutationFn: () => + apiJson("/admin/api/proxies/probe", { + method: "POST", + body: JSON.stringify({ + probe_url: String(probeUrl || "").trim() || undefined, + timeout_ms: + Number.isFinite(Number(probeTimeoutMs)) && Number(probeTimeoutMs) > 0 ? Number(probeTimeoutMs) : undefined, + concurrency: + Number.isFinite(Number(probeConcurrency)) && Number(probeConcurrency) > 0 ? Number(probeConcurrency) : undefined, + }), + }), + onMutate: () => { + setProbeErrorMessage(null); + setProbeRequestId(null); + setProbeJobId(null); + }, + onSuccess: (data) => { + setProbeRequestId(data.request_id); + setProbeJobId(data.job_id); + }, + onError: (err) => { + if (err instanceof ApiError) { + setProbeErrorMessage(err.message); + setProbeRequestId(requestIdFromError(err)); + return; + } + if (err instanceof Error) { + setProbeErrorMessage(err.message); + return; + } + setProbeErrorMessage("探测任务入队失败"); + }, + }); + + const easyImport = useMutation({ + mutationFn: (values: EasyProxiesImportFormValues) => { + const payload: Record = { + base_url: String(values.base_url || "").trim(), + password: String(values.password || ""), + conflict_policy: values.conflict_policy, + }; + if (String(values.host_override || "").trim()) payload.host_override = String(values.host_override || "").trim(); + if (values.bootstrap) { + if (typeof values.attach_pool_id === "number") payload.attach_pool_id = values.attach_pool_id; + if (typeof values.attach_weight === "number") payload.attach_weight = values.attach_weight; + payload.recompute_bindings = values.recompute_bindings !== false; + if (payload.recompute_bindings) { + if (typeof values.max_tokens_per_proxy === "number") payload.max_tokens_per_proxy = values.max_tokens_per_proxy; + payload.strict = values.strict ?? false; + } + } + return apiJson("/admin/api/proxies/easy-proxies/import", { + method: "POST", + body: JSON.stringify(payload), + }); + }, + onMutate: () => { + setEasyErrorMessage(null); + setEasyRequestId(null); + setEasyResult(null); + }, + onSuccess: (data) => { + setEasyResult(data); + setEasyRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "proxies", "endpoints"] }); + }, + onError: (err) => { + if (err instanceof ApiError) { + setEasyErrorMessage(err.message); + setEasyRequestId(requestIdFromError(err)); + return; + } + if (err instanceof Error) { + setEasyErrorMessage(err.message); + return; + } + setEasyErrorMessage("从 easy-proxies 导入失败"); + }, + }); + + return ( + + + 代理管理 + + + {actionMessage ? : null} + {actionRequestId ? 请求ID: {actionRequestId} : null} + {actionWarningMessage ? : null} + {actionErrorMessage ? : null} + {actionErrorRequestId ? 请求ID: {actionErrorRequestId} : null} + + + + + 重算 token 绑定 + setCleanupRecomputeBindings(Boolean(v))} /> + + + 删除无引用节点 + setCleanupDeleteOrphans(Boolean(v))} /> + + + 单代理最多绑定令牌数 + setCleanupMaxTokensPerProxy(Number(v) || 2)} + /> + + + 严格容量 + setCleanupStrict(Boolean(v))} /> + + + + + 作用：禁用并从代理池移除无效占位主机（如 0.0.0.0/localhost），可选重算绑定与删除无引用记录。 + + + + + {manualErrorMessage ? : null} + {manualRequestId ? 请求ID: {manualRequestId} : null} + + + form={manualForm} + layout="vertical" + initialValues={{ text: "", source: "manual", conflict_policy: "overwrite" }} + onFinish={(values) => manualImport.mutate(values)} + > + + + + + + + + + + + Boolean(p.enabled)) + .map((p) => ({ value: Number(p.id), label: `${p.name}(#${p.id})` }))} + style={{ minWidth: 260 }} + /> + + + + + + + + p.recompute_bindings !== c.recompute_bindings}> + {({ getFieldValue: get2 }) => + get2("recompute_bindings") ? ( + <> + + + + + + + + ) : null + } + + + ) : null + } + + + + + + {easyImport.isPending ? : null} + {easyResult ? ( + <> + + {Array.isArray(easyResult.warnings) && easyResult.warnings.length > 0 ? ( + + ) : null} + + ) : null} + + + + + + + + + + 探测URL: + setProbeUrl(e.target.value)} + placeholder="留空使用默认" + style={{ width: 360 }} + /> + 超时(ms): + setProbeTimeoutMs(typeof v === "number" ? v : 8000)} /> + 并发: + setProbeConcurrency(typeof v === "number" ? v : 10)} /> + + + {probe.isPending ? : null} + {probeErrorMessage ? : null} + {probeJobId ? ( + + ) : null} + {probeRequestId ? 请求ID: {probeRequestId} : null} + + {query.isLoading ? ( + + ) : query.isError ? ( + + ) : !query.data ? ( + + ) : query.data.items.length === 0 ? ( + + ) : ( + <> + 请求ID: {query.data.request_id} + + rowKey={(row) => row.id} + columns={columns({ + onToggleEnabled: (row) => updateEndpoint.mutate({ endpointId: row.id, enabled: !row.enabled }), + updatePendingId: updateEndpoint.isPending ? updateEndpoint.variables?.endpointId ?? null : null, + onResetFailures: (row) => resetEndpointFailures.mutate({ endpointId: row.id }), + resetPendingId: resetEndpointFailures.isPending ? resetEndpointFailures.variables?.endpointId ?? null : null, + })} + dataSource={query.data.items} + pagination={false} + size="small" + scroll={{ x: 1500 }} + style={{ marginTop: 12 }} + /> + + )} + + + ); +} diff --git a/frontend/src/pages/ProxyPoolsPage.test.tsx b/frontend/src/pages/ProxyPoolsPage.test.tsx new file mode 100644 index 0000000..74a9ce7 --- /dev/null +++ b/frontend/src/pages/ProxyPoolsPage.test.tsx @@ -0,0 +1,97 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import React from "react"; +import { MemoryRouter } from "react-router-dom"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { ProxyPoolsPage } from "./ProxyPoolsPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("ProxyPoolsPage", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/admin/api/proxy-pools") && (!init || init.method === "GET" || !init.method)) { + return new Response( + JSON.stringify({ + ok: true, + items: [{ id: "1", name: "默认代理池", description: null, enabled: true }], + request_id: "req_pools", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/proxies/endpoints")) { + return new Response( + JSON.stringify({ + ok: true, + items: [ + { + id: "10", + uri_masked: "http://1.2.3.4:8080", + enabled: true, + pools: [{ id: "1", name: "默认代理池", pool_enabled: true, member_enabled: true, weight: 2 }], + }, + ], + request_id: "req_eps", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/proxy-pools/1/endpoints") && init?.method === "POST") { + return new Response( + JSON.stringify({ + ok: true, + pool_id: "1", + created: 0, + updated: 1, + removed: 0, + request_id: "req_set", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders pools and saves endpoint membership", async () => { + const qc = makeClient(); + render( + + + + + , + ); + + expect(await screen.findByText("代理池管理")).toBeInTheDocument(); + expect(await screen.findByText("默认代理池")).toBeInTheDocument(); + + fireEvent.click(await screen.findByTestId("pool-config-1")); + expect(await screen.findByText(/配置代理池节点/)).toBeInTheDocument(); + + fireEvent.click(await screen.findByTestId("pool-endpoints-save")); + + await waitFor(() => { + const calls = (fetch as unknown as ReturnType).mock.calls; + const matched = calls.find((c) => String(c[0]).endsWith("/admin/api/proxy-pools/1/endpoints")); + expect(matched).toBeTruthy(); + const init = matched?.[1] as RequestInit | undefined; + expect(init?.method).toBe("POST"); + expect(String(init?.body)).toContain("\"endpoint_id\":10"); + expect(String(init?.body)).toContain("\"weight\":2"); + }); + }); +}); + diff --git a/frontend/src/pages/ProxyPoolsPage.tsx b/frontend/src/pages/ProxyPoolsPage.tsx new file mode 100644 index 0000000..e624d9d --- /dev/null +++ b/frontend/src/pages/ProxyPoolsPage.tsx @@ -0,0 +1,495 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Alert, Button, Card, Form, Input, InputNumber, Modal, Skeleton, Space, Switch, Table, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React, { useEffect, useMemo, useState } from "react"; +import { useNavigate } from "react-router-dom"; + +import { ApiError, apiJson } from "../api/client"; + +type ProxyPoolItem = { + id: string; + name: string; + description: string | null; + enabled: boolean; +}; + +type ProxyPoolsListResponse = { + ok: true; + items: ProxyPoolItem[]; + request_id: string; +}; + +type CreateProxyPoolResponse = { + ok: true; + pool_id: string; + request_id: string; +}; + +type UpdateProxyPoolResponse = { + ok: true; + pool_id: string; + request_id: string; +}; + +type ProxyEndpointListItem = { + id: string; + uri_masked: string; + enabled: boolean; + pools: Array<{ + id: string; + name: string; + pool_enabled: boolean; + member_enabled: boolean; + weight: number; + }>; +}; + +type ProxyEndpointsResponse = { + ok: true; + items: ProxyEndpointListItem[]; + request_id: string; +}; + +type SetPoolEndpointsResponse = { + ok: true; + pool_id: string; + created: number; + updated: number; + removed: number; + request_id: string; +}; + +type CreatePoolFormValues = { + name: string; + description: string; + enabled: boolean; +}; + +type EditPoolFormValues = { + name: string; + description: string; + enabled: boolean; +}; + +type MemberConfig = { enabled: boolean; weight: number }; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +function messageFromError(err: unknown): string { + if (err instanceof ApiError) return err.message; + if (err instanceof Error) return err.message; + return "未知错误"; +} + +export function ProxyPoolsPage() { + const navigate = useNavigate(); + const qc = useQueryClient(); + + const pools = useQuery({ + queryKey: ["admin", "proxy-pools"], + queryFn: () => apiJson("/admin/api/proxy-pools"), + }); + + const endpoints = useQuery({ + queryKey: ["admin", "proxies", "endpoints"], + queryFn: () => apiJson("/admin/api/proxies/endpoints"), + }); + + const [createOpen, setCreateOpen] = useState(false); + const [createForm] = Form.useForm(); + + const [editOpen, setEditOpen] = useState(false); + const [editingPool, setEditingPool] = useState(null); + const [editForm] = Form.useForm(); + + const [configOpen, setConfigOpen] = useState(false); + const [configPool, setConfigPool] = useState(null); + const [selectedEndpointIds, setSelectedEndpointIds] = useState([]); + const [memberConfig, setMemberConfig] = useState>({}); + + const [actionAlert, setActionAlert] = useState<{ type: "success" | "error"; message: string; requestId: string | null } | null>(null); + + const createPool = useMutation({ + mutationFn: (values: CreatePoolFormValues) => + apiJson("/admin/api/proxy-pools", { + method: "POST", + body: JSON.stringify({ + name: values.name, + description: values.description.trim() ? values.description.trim() : null, + enabled: Boolean(values.enabled), + }), + }), + onMutate: () => setActionAlert(null), + onSuccess: (data) => { + setCreateOpen(false); + createForm.resetFields(); + setActionAlert({ type: "success", message: `代理池创建成功：#${data.pool_id}`, requestId: data.request_id }); + qc.invalidateQueries({ queryKey: ["admin", "proxy-pools"] }); + }, + onError: (err) => { + setActionAlert({ type: "error", message: messageFromError(err), requestId: requestIdFromError(err) }); + }, + }); + + const updatePool = useMutation({ + mutationFn: (vars: { poolId: string; values: EditPoolFormValues }) => + apiJson(`/admin/api/proxy-pools/${encodeURIComponent(vars.poolId)}`, { + method: "PUT", + body: JSON.stringify({ + name: vars.values.name, + description: vars.values.description.trim() ? vars.values.description.trim() : null, + enabled: Boolean(vars.values.enabled), + }), + }), + onMutate: () => setActionAlert(null), + onSuccess: (data) => { + setEditOpen(false); + setEditingPool(null); + setActionAlert({ type: "success", message: `代理池已更新：#${data.pool_id}`, requestId: data.request_id }); + qc.invalidateQueries({ queryKey: ["admin", "proxy-pools"] }); + qc.invalidateQueries({ queryKey: ["admin", "proxies", "endpoints"] }); + }, + onError: (err) => { + setActionAlert({ type: "error", message: messageFromError(err), requestId: requestIdFromError(err) }); + }, + }); + + const setPoolEndpoints = useMutation({ + mutationFn: (vars: { poolId: string; endpointIds: string[]; config: Record }) => { + const items = vars.endpointIds + .map((id) => { + const cfg = vars.config[id] || { enabled: true, weight: 1 }; + return { endpoint_id: Number(id), enabled: Boolean(cfg.enabled), weight: Number(cfg.weight) || 1 }; + }) + .filter((v) => Number.isFinite(v.endpoint_id) && v.endpoint_id > 0); + + return apiJson(`/admin/api/proxy-pools/${encodeURIComponent(vars.poolId)}/endpoints`, { + method: "POST", + body: JSON.stringify({ items }), + }); + }, + onMutate: () => setActionAlert(null), + onSuccess: (data) => { + setActionAlert({ + type: "success", + message: `节点配置已保存：新增=${data.created} 更新=${data.updated} 移除=${data.removed}`, + requestId: data.request_id, + }); + qc.invalidateQueries({ queryKey: ["admin", "proxies", "endpoints"] }); + setConfigOpen(false); + setConfigPool(null); + }, + onError: (err) => { + setActionAlert({ type: "error", message: messageFromError(err), requestId: requestIdFromError(err) }); + }, + }); + + const endpointRows = endpoints.data?.items || []; + + const endpointSelection = useMemo(() => { + return { + selectedRowKeys: selectedEndpointIds, + onChange: (keys: React.Key[]) => { + const ids = keys.map((k) => String(k)); + setSelectedEndpointIds(ids); + setMemberConfig((prev) => { + const next = { ...prev }; + for (const id of ids) { + if (!next[id]) next[id] = { enabled: true, weight: 1 }; + } + return next; + }); + }, + }; + }, [selectedEndpointIds]); + + useEffect(() => { + if (!configOpen || !configPool || !endpoints.data) return; + + const poolId = configPool.id; + const selected: string[] = []; + const cfg: Record = {}; + for (const ep of endpoints.data.items) { + const membership = (ep.pools || []).find((p) => String(p.id) === String(poolId)); + if (!membership) continue; + selected.push(String(ep.id)); + cfg[String(ep.id)] = { enabled: Boolean(membership.member_enabled), weight: Number(membership.weight) || 1 }; + } + + setSelectedEndpointIds(selected); + setMemberConfig(cfg); + }, [configOpen, configPool, endpoints.data]); + + const openEdit = (pool: ProxyPoolItem) => { + setEditingPool(pool); + editForm.setFieldsValue({ + name: pool.name, + description: pool.description || "", + enabled: Boolean(pool.enabled), + }); + setEditOpen(true); + }; + + const openConfig = (pool: ProxyPoolItem) => { + setConfigPool(pool); + setConfigOpen(true); + }; + + const poolColumns: ColumnsType = [ + { title: "ID", dataIndex: "id", key: "id", width: 90, render: (value) => `#${value}` }, + { title: "名称", dataIndex: "name", key: "name", width: 220 }, + { title: "启用", dataIndex: "enabled", key: "enabled", width: 90, render: (value) => (value ? "是" : "否") }, + { title: "描述", dataIndex: "description", key: "description", render: (value) => value || "-" }, + { + title: "操作", + key: "actions", + width: 280, + render: (_, row) => ( + + + + + + ), + }, + ]; + + const endpointColumns: ColumnsType = [ + { title: "节点ID", dataIndex: "id", key: "id", width: 90, render: (value) => `#${value}` }, + { title: "代理地址（掩码）", dataIndex: "uri_masked", key: "uri_masked", width: 320 }, + { title: "节点启用", dataIndex: "enabled", key: "enabled", width: 90, render: (v) => (v ? "是" : "否") }, + { + title: "成员启用", + key: "member_enabled", + width: 110, + render: (_, row) => { + const id = String(row.id); + const selected = selectedEndpointIds.includes(id); + const checked = memberConfig[id]?.enabled ?? true; + return ( + setMemberConfig((prev) => ({ ...prev, [id]: { ...(prev[id] || { enabled: true, weight: 1 }), enabled: value } }))} + /> + ); + }, + }, + { + title: "权重", + key: "weight", + width: 100, + render: (_, row) => { + const id = String(row.id); + const selected = selectedEndpointIds.includes(id); + const weight = memberConfig[id]?.weight ?? 1; + return ( + setMemberConfig((prev) => ({ ...prev, [id]: { ...(prev[id] || { enabled: true, weight: 1 }), weight: Number(value || 1) } }))} + /> + ); + }, + }, + ]; + + return ( + + + 代理池管理 + + + + + + + + + + {actionAlert ? ( + + ) : null} + + + {pools.isLoading ? ( + + ) : pools.isError ? ( + + ) : !pools.data || pools.data.items.length === 0 ? ( + + ) : ( + <> + 请求ID: {pools.data.request_id} + + rowKey={(row) => row.id} + columns={poolColumns} + dataSource={pools.data.items} + pagination={false} + size="small" + style={{ marginTop: 12 }} + /> + + )} + + + { + setCreateOpen(false); + createForm.resetFields(); + }} + footer={null} + destroyOnClose + > + + form={createForm} + layout="vertical" + initialValues={{ name: "", description: "", enabled: true }} + onFinish={(values) => createPool.mutate(values)} + > + + + + + + + + + + + + + + + + + { + setEditOpen(false); + setEditingPool(null); + }} + footer={null} + destroyOnClose + > + + form={editForm} + layout="vertical" + initialValues={{ name: "", description: "", enabled: true }} + onFinish={(values) => { + if (!editingPool) return; + updatePool.mutate({ poolId: editingPool.id, values }); + }} + > + + + + + + + + + + + + + + + + + { + setConfigOpen(false); + setConfigPool(null); + }} + width={900} + footer={ + + + {configPool ? ( + + 当前代理池ID: #{configPool.id}（可在系统设置中填写默认代理池ID） + + ) : null} + + + + + + + } + destroyOnClose + > + {endpoints.isLoading ? ( + + ) : endpoints.isError ? ( + + ) : ( + <> + 请求ID: {endpoints.data?.request_id} + + + rowKey={(row) => row.id} + columns={endpointColumns} + dataSource={endpointRows} + pagination={false} + size="small" + style={{ marginTop: 12 }} + scroll={{ x: 880 }} + rowSelection={endpointSelection} + /> + + )} + + + ); +} + diff --git a/frontend/src/pages/RecommendationPage.test.tsx b/frontend/src/pages/RecommendationPage.test.tsx new file mode 100644 index 0000000..bd12a28 --- /dev/null +++ b/frontend/src/pages/RecommendationPage.test.tsx @@ -0,0 +1,119 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import React from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { RecommendationPage } from "./RecommendationPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("RecommendationPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/admin/api/settings") && init?.method === "PUT") { + const body = init.body ? JSON.parse(String(init.body)) : {}; + expect(body.settings.random.strategy).toBe("quality"); + expect(body.settings.random.quality_samples).toBe(5); + expect(body.settings.random.recommendation.pick_mode).toBe("weighted"); + expect(body.settings.random.recommendation.temperature).toBe(1); + expect(body.settings.random.recommendation.score_weights.bookmark).toBe(4); + expect(body.settings.random.recommendation.multipliers.ai).toBe(0.5); + expect(body.settings.random.recommendation.multipliers.manga).toBe(0); + return new Response(JSON.stringify({ ok: true, updated: 3, request_id: "req_save" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.endsWith("/admin/api/settings")) { + return new Response( + JSON.stringify({ + ok: true, + settings: { + random: { + strategy: "quality", + quality_samples: 5, + recommendation: { + pick_mode: "weighted", + temperature: 1, + score_weights: { bookmark: 4, view: 0.5, comment: 2, pixels: 1, bookmark_rate: 3 }, + multipliers: { + ai: 1, + non_ai: 1, + unknown_ai: 1, + illust: 1, + manga: 1, + ugoira: 1, + unknown_illust_type: 1, + }, + }, + }, + }, + request_id: "req_settings", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.startsWith("/random?")) { + return new Response( + JSON.stringify({ + ok: true, + request_id: "req_preview", + data: { debug: { picked_by: "quality_weighted" } }, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("推荐策略")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_settings/)).toBeInTheDocument(); + }); + + it("saves and previews", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("推荐策略")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_settings/)).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: /AI 倍率设为 0.5/ })); + fireEvent.click(screen.getByRole("button", { name: /漫画倍率设为 0/ })); + fireEvent.click(screen.getByRole("button", { name: /保存推荐配置/ })); + + expect(await screen.findByText("保存成功")).toBeInTheDocument(); + expect(await screen.findByText(/更新条目数:\s*3/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_save/)).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: /预览一次随机结果/ })); + expect(await screen.findByText(/请求ID:\s*req_preview/)).toBeInTheDocument(); + expect(await screen.findByText(/quality_weighted/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/RecommendationPage.tsx b/frontend/src/pages/RecommendationPage.tsx new file mode 100644 index 0000000..17d3dab --- /dev/null +++ b/frontend/src/pages/RecommendationPage.tsx @@ -0,0 +1,449 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Alert, Button, Card, Form, Input, InputNumber, Select, Skeleton, Space, Typography } from "antd"; +import React, { useEffect, useMemo, useState } from "react"; + +import { ApiError, apiJson } from "../api/client"; + +type SettingsResponse = { + ok: true; + settings: Record; + request_id: string; +}; + +type SettingsUpdateResponse = { + ok: true; + updated: number; + request_id: string; +}; + +type RandomPreviewResponse = { + ok: true; + request_id: string; + data: Record; +}; + +type FormValues = { + random_strategy: "quality" | "random"; + random_quality_samples: number; + pick_mode: "weighted" | "best"; + temperature: number; + + w_bookmark: number; + w_view: number; + w_comment: number; + w_pixels: number; + w_bookmark_rate: number; + + m_ai: number; + m_non_ai: number; + m_unknown_ai: number; + m_illust: number; + m_manga: number; + m_ugoira: number; + m_unknown_illust_type: number; + + preview_seed: string; +}; + +const STRATEGY_VALUES = new Set(["quality", "random"]); +const PICK_MODE_VALUES = new Set(["weighted", "best"]); + +const DEFAULTS: Omit = { + random_strategy: "quality", + random_quality_samples: 5, + pick_mode: "weighted", + temperature: 1.0, + w_bookmark: 4.0, + w_view: 0.5, + w_comment: 2.0, + w_pixels: 1.0, + w_bookmark_rate: 3.0, + m_ai: 1.0, + m_non_ai: 1.0, + m_unknown_ai: 1.0, + m_illust: 1.0, + m_manga: 1.0, + m_ugoira: 1.0, + m_unknown_illust_type: 1.0, +}; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +function asObject(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) return {}; + return value as Record; +} + +function asInt(value: unknown, fallback: number): number { + if (typeof value === "number" && Number.isFinite(value)) return Math.trunc(value); + if (typeof value === "string") { + const parsed = Number.parseInt(value.trim(), 10); + if (Number.isFinite(parsed)) return parsed; + } + return fallback; +} + +function asFloat(value: unknown, fallback: number): number { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string") { + const parsed = Number.parseFloat(value.trim()); + if (Number.isFinite(parsed)) return parsed; + } + return fallback; +} + +function asLowerEnum(value: unknown, allowed: Set, fallback: T): T { + if (typeof value === "string") { + const normalized = value.trim().toLowerCase(); + if (allowed.has(normalized as T)) return normalized as T; + } + return fallback; +} + +function buildPreviewUrl(seed: string): string { + const sp = new URLSearchParams(); + sp.set("format", "json"); + sp.set("attempts", "1"); + if (seed.trim()) sp.set("seed", seed.trim()); + return `/random?${sp.toString()}`; +} + +export function RecommendationPage() { + const queryClient = useQueryClient(); + const [form] = Form.useForm(); + + const query = useQuery({ + queryKey: ["admin", "settings"], + queryFn: () => apiJson("/admin/api/settings"), + }); + + const [saveErrorMessage, setSaveErrorMessage] = useState(null); + const [saveRequestId, setSaveRequestId] = useState(null); + const [saveUpdated, setSaveUpdated] = useState(null); + + const [previewErrorMessage, setPreviewErrorMessage] = useState(null); + const [previewRequestId, setPreviewRequestId] = useState(null); + const [previewUrl, setPreviewUrl] = useState(null); + const [previewBody, setPreviewBody] = useState(null); + + useEffect(() => { + if (!query.data) return; + const settings = asObject(query.data.settings); + const random = asObject(settings.random); + const recommendation = asObject(random.recommendation); + const scoreWeights = asObject(recommendation.score_weights); + const multipliers = asObject(recommendation.multipliers); + + form.setFieldsValue({ + random_strategy: asLowerEnum(random.strategy, STRATEGY_VALUES, DEFAULTS.random_strategy), + random_quality_samples: asInt(random.quality_samples, DEFAULTS.random_quality_samples), + pick_mode: asLowerEnum(recommendation.pick_mode, PICK_MODE_VALUES, DEFAULTS.pick_mode), + temperature: asFloat(recommendation.temperature, DEFAULTS.temperature), + w_bookmark: asFloat(scoreWeights.bookmark, DEFAULTS.w_bookmark), + w_view: asFloat(scoreWeights.view, DEFAULTS.w_view), + w_comment: asFloat(scoreWeights.comment, DEFAULTS.w_comment), + w_pixels: asFloat(scoreWeights.pixels, DEFAULTS.w_pixels), + w_bookmark_rate: asFloat(scoreWeights.bookmark_rate, DEFAULTS.w_bookmark_rate), + m_ai: asFloat(multipliers.ai, DEFAULTS.m_ai), + m_non_ai: asFloat(multipliers.non_ai, DEFAULTS.m_non_ai), + m_unknown_ai: asFloat(multipliers.unknown_ai, DEFAULTS.m_unknown_ai), + m_illust: asFloat(multipliers.illust, DEFAULTS.m_illust), + m_manga: asFloat(multipliers.manga, DEFAULTS.m_manga), + m_ugoira: asFloat(multipliers.ugoira, DEFAULTS.m_ugoira), + m_unknown_illust_type: asFloat(multipliers.unknown_illust_type, DEFAULTS.m_unknown_illust_type), + preview_seed: "", + }); + }, [form, query.data]); + + const qualityFormulaText = useMemo(() => { + return [ + "score =", + " w_bookmark * ln(1 + bookmark_count)", + "+ w_view * ln(1 + view_count)", + "+ w_comment * ln(1 + comment_count)", + "+ w_pixels * ln(1 + (width*height)/1_000_000)", + "+ w_bookmark_rate * ln(1 + (bookmark_count/view_count)*1000)", + "", + "最终得分会再乘以“类别倍率”（AI/插画/漫画/动图等）。", + "倍率=0 表示彻底不返回该类别。", + ].join("\n"); + }, []); + + const save = useMutation({ + mutationFn: (values: FormValues) => + apiJson("/admin/api/settings", { + method: "PUT", + body: JSON.stringify({ + settings: { + random: { + strategy: values.random_strategy, + quality_samples: values.random_quality_samples, + recommendation: { + pick_mode: values.pick_mode, + temperature: values.temperature, + score_weights: { + bookmark: values.w_bookmark, + view: values.w_view, + comment: values.w_comment, + pixels: values.w_pixels, + bookmark_rate: values.w_bookmark_rate, + }, + multipliers: { + ai: values.m_ai, + non_ai: values.m_non_ai, + unknown_ai: values.m_unknown_ai, + illust: values.m_illust, + manga: values.m_manga, + ugoira: values.m_ugoira, + unknown_illust_type: values.m_unknown_illust_type, + }, + }, + }, + }, + }), + }), + onMutate: () => { + setSaveErrorMessage(null); + setSaveRequestId(null); + setSaveUpdated(null); + }, + onSuccess: (data) => { + setSaveUpdated(data.updated); + setSaveRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "settings"] }); + }, + onError: (err) => { + if (err instanceof ApiError) { + setSaveErrorMessage(err.message); + setSaveRequestId(requestIdFromError(err)); + return; + } + if (err instanceof Error) { + setSaveErrorMessage(err.message); + return; + } + setSaveErrorMessage("保存失败"); + }, + }); + + const preview = useMutation({ + mutationFn: async (values: FormValues) => { + const url = buildPreviewUrl(values.preview_seed); + setPreviewUrl(url); + const data = await apiJson(url); + return { url, data }; + }, + onMutate: () => { + setPreviewErrorMessage(null); + setPreviewRequestId(null); + setPreviewBody(null); + setPreviewUrl(null); + }, + onSuccess: ({ url, data }) => { + setPreviewUrl(url); + setPreviewBody(data); + setPreviewRequestId(data.request_id); + }, + onError: (err) => { + if (err instanceof ApiError) { + setPreviewErrorMessage(err.message); + setPreviewRequestId(requestIdFromError(err)); + return; + } + if (err instanceof Error) { + setPreviewErrorMessage(err.message); + return; + } + setPreviewErrorMessage("预览失败"); + }, + }); + + const onResetDefaults = () => { + form.setFieldsValue({ ...DEFAULTS, preview_seed: form.getFieldValue("preview_seed") || "" }); + }; + + const onAiHalf = () => { + form.setFieldsValue({ m_ai: 0.5 }); + }; + + const onDisableManga = () => { + form.setFieldsValue({ m_manga: 0 }); + }; + + const onOnlyIllust = () => { + form.setFieldsValue({ m_illust: 1.0, m_manga: 0, m_ugoira: 0, m_unknown_illust_type: 0 }); + }; + + return ( + + + 推荐策略 + + + 这里管理的是 /random 默认的“质量优先推荐”策略：权重/倍率/随机程度都会实时生效（保存后无需重启）。 + + + + + + + + + + + {save.isPending ? : null} + {saveErrorMessage ? : null} + {saveUpdated !== null ? ( + + ) : null} + {saveRequestId ? 请求ID: {saveRequestId} : null} + + {query.isLoading ? ( + + ) : query.isError ? ( + + ) : !query.data ? ( + + ) : ( + + 请求ID: {query.data.request_id} +
save.mutate(values)}> + + 选择路径 + + + + + + + + + + 评分公式（score_weights） + + + 
{qualityFormulaText}
+ + + + + + + + + + + + + + + + + + + + + 类别倍率（multipliers） + + + 提示：倍率=0 会直接剔除该类别；倍率越大，被选中的概率越高。建议范围 0~3（通常不需要很大）。 + + + + + + + + + + + + + + + + + + + + + + + + + + + 预览（/random） + + + 预览使用的是“已保存并生效”的配置（不是未保存的草稿）。建议先保存，再预览。 + + + + + + + + + {preview.isPending ? : null} + {previewErrorMessage ? : null} + {previewRequestId ? 请求ID: {previewRequestId} : null} + {previewUrl ? 请求链接: {previewUrl} : null} + {previewBody ? ( + 
+                {JSON.stringify(previewBody, null, 2)}
+
+ ) : null} + + + )} + + ); +} diff --git a/frontend/src/pages/SettingsPage.test.tsx b/frontend/src/pages/SettingsPage.test.tsx new file mode 100644 index 0000000..ff20efd --- /dev/null +++ b/frontend/src/pages/SettingsPage.test.tsx @@ -0,0 +1,101 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import React from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { SettingsPage } from "./SettingsPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("SettingsPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/admin/api/settings") && init?.method === "PUT") { + const body = init.body ? JSON.parse(String(init.body)) : {}; + expect(body.settings.proxy.enabled).toBe(true); + expect(body.settings.proxy.fail_closed).toBe(false); + expect(body.settings.proxy.route_mode).toBe("pixiv_only"); + expect(body.settings.proxy.allowlist_domains).toEqual(["pixiv.net"]); + expect(body.settings.proxy.default_pool_id).toBe(""); + expect(body.settings.random.default_attempts).toBe(3); + expect(body.settings.random.default_r18_strict).toBe(true); + expect(body.settings.random.fail_cooldown_ms).toBe(600000); + expect(body.settings.random.strategy).toBe("quality"); + expect(body.settings.random.quality_samples).toBe(5); + expect(body.settings.security.hide_origin_url_in_public_json).toBe(true); + expect(body.settings.rate_limit.pixiv_hydrate_min_interval_ms).toBe(800); + expect(body.settings.rate_limit.pixiv_hydrate_jitter_ms).toBe(200); + return new Response(JSON.stringify({ ok: true, updated: 9, request_id: "req_save" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.endsWith("/admin/api/settings")) { + return new Response( + JSON.stringify({ + ok: true, + settings: { + proxy: { + enabled: true, + fail_closed: false, + route_mode: "pixiv_only", + allowlist_domains: ["pixiv.net"], + default_pool_id: "", + }, + random: { default_attempts: 3, default_r18_strict: true, fail_cooldown_ms: 600000, strategy: "quality", quality_samples: 5 }, + security: { hide_origin_url_in_public_json: true }, + rate_limit: {}, + }, + request_id: "req_settings", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("系统设置")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_settings/)).toBeInTheDocument(); + }); + + it("saves settings", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("系统设置")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_settings/)).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: /保存设置/ })); + + expect(await screen.findByText("保存成功")).toBeInTheDocument(); + expect(await screen.findByText(/更新条目数:\s*9/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_save/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx new file mode 100644 index 0000000..50b008b --- /dev/null +++ b/frontend/src/pages/SettingsPage.tsx @@ -0,0 +1,327 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Alert, Button, Card, Form, InputNumber, Select, Skeleton, Space, Switch, Typography } from "antd"; +import React, { useEffect, useState } from "react"; + +import { ApiError, apiJson } from "../api/client"; + +type ProxyPoolItem = { + id: string; + name: string; + description: string | null; + enabled: boolean; +}; + +type ProxyPoolsListResponse = { + ok: true; + items: ProxyPoolItem[]; + request_id: string; +}; + +type SettingsResponse = { + ok: true; + settings: Record; + request_id: string; +}; + +type SettingsUpdateResponse = { + ok: true; + updated: number; + request_id: string; +}; + +type SettingsFormValues = { + proxy_enabled: boolean; + proxy_fail_closed: boolean; + proxy_route_mode: "pixiv_only" | "all" | "allowlist" | "off"; + proxy_allowlist_domains: string[]; + proxy_default_pool_id: number; + random_default_attempts: number; + random_default_r18_strict: boolean; + random_fail_cooldown_ms: number; + random_strategy: "quality" | "random"; + random_quality_samples: number; + security_hide_origin_url_in_public_json: boolean; + pixiv_hydrate_min_interval_ms: number; + pixiv_hydrate_jitter_ms: number; +}; + +const RANDOM_STRATEGY_VALUES = new Set(["quality", "random"]); + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +function asObject(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) return {}; + return value as Record; +} + +function asBool(value: unknown, fallback: boolean): boolean { + if (typeof value === "boolean") return value; + if (typeof value === "number" && (value === 0 || value === 1)) return Boolean(value); + if (typeof value === "string") { + const normalized = value.trim().toLowerCase(); + if (normalized === "true" || normalized === "1" || normalized === "yes" || normalized === "on") return true; + if (normalized === "false" || normalized === "0" || normalized === "no" || normalized === "off") return false; + } + return fallback; +} + +function asInt(value: unknown, fallback: number): number { + if (typeof value === "number" && Number.isFinite(value)) return Math.trunc(value); + if (typeof value === "string") { + const parsed = Number.parseInt(value.trim(), 10); + if (Number.isFinite(parsed)) return parsed; + } + return fallback; +} + +function asLowerEnum(value: unknown, allowed: Set, fallback: T): T { + if (typeof value === "string") { + const normalized = value.trim().toLowerCase(); + if (allowed.has(normalized as T)) return normalized as T; + } + return fallback; +} + +function asStrList(value: unknown): string[] { + if (!Array.isArray(value)) return []; + const out: string[] = []; + const seen = new Set(); + for (const item of value) { + if (typeof item !== "string") continue; + const trimmed = item.trim(); + if (!trimmed || trimmed.length > 200 || seen.has(trimmed)) continue; + seen.add(trimmed); + out.push(trimmed); + } + return out; +} + +export function SettingsPage() { + const queryClient = useQueryClient(); + const [form] = Form.useForm(); + const query = useQuery({ + queryKey: ["admin", "settings"], + queryFn: () => apiJson("/admin/api/settings"), + }); + + const poolsQuery = useQuery({ + queryKey: ["admin", "proxy-pools"], + queryFn: () => apiJson("/admin/api/proxy-pools"), + }); + + const [saveErrorMessage, setSaveErrorMessage] = useState(null); + const [saveRequestId, setSaveRequestId] = useState(null); + const [saveUpdated, setSaveUpdated] = useState(null); + + useEffect(() => { + if (!query.data) return; + const settings = asObject(query.data.settings); + const proxy = asObject(settings.proxy); + const random = asObject(settings.random); + const security = asObject(settings.security); + const rateLimit = asObject(settings.rate_limit); + + const routeModeRaw = String(proxy.route_mode || "pixiv_only").trim().toLowerCase(); + const routeMode: SettingsFormValues["proxy_route_mode"] = + routeModeRaw === "all" || routeModeRaw === "allowlist" || routeModeRaw === "off" ? routeModeRaw : "pixiv_only"; + + form.setFieldsValue({ + proxy_enabled: asBool(proxy.enabled, false), + proxy_fail_closed: asBool(proxy.fail_closed, false), + proxy_route_mode: routeMode, + proxy_allowlist_domains: asStrList(proxy.allowlist_domains), + proxy_default_pool_id: asInt(proxy.default_pool_id, 0), + random_default_attempts: asInt(random.default_attempts, 3), + random_default_r18_strict: asBool(random.default_r18_strict, true), + random_fail_cooldown_ms: asInt(random.fail_cooldown_ms, 600_000), + random_strategy: asLowerEnum(random.strategy, RANDOM_STRATEGY_VALUES, "quality"), + random_quality_samples: asInt(random.quality_samples, 5), + security_hide_origin_url_in_public_json: asBool(security.hide_origin_url_in_public_json, true), + pixiv_hydrate_min_interval_ms: asInt(rateLimit.pixiv_hydrate_min_interval_ms, 800), + pixiv_hydrate_jitter_ms: asInt(rateLimit.pixiv_hydrate_jitter_ms, 200), + }); + }, [form, query.data]); + + const save = useMutation({ + mutationFn: (values: SettingsFormValues) => + apiJson("/admin/api/settings", { + method: "PUT", + body: JSON.stringify({ + settings: { + proxy: { + enabled: values.proxy_enabled, + fail_closed: values.proxy_fail_closed, + route_mode: values.proxy_route_mode, + allowlist_domains: values.proxy_allowlist_domains, + default_pool_id: values.proxy_default_pool_id > 0 ? values.proxy_default_pool_id : "", + }, + random: { + default_attempts: values.random_default_attempts, + default_r18_strict: values.random_default_r18_strict, + fail_cooldown_ms: values.random_fail_cooldown_ms, + strategy: values.random_strategy, + quality_samples: values.random_quality_samples, + }, + security: { hide_origin_url_in_public_json: values.security_hide_origin_url_in_public_json }, + rate_limit: { + pixiv_hydrate_min_interval_ms: Math.max(0, Math.trunc(values.pixiv_hydrate_min_interval_ms || 0)), + pixiv_hydrate_jitter_ms: Math.max(0, Math.trunc(values.pixiv_hydrate_jitter_ms || 0)), + }, + }, + }), + }), + onMutate: () => { + setSaveErrorMessage(null); + setSaveRequestId(null); + setSaveUpdated(null); + }, + onSuccess: (data) => { + setSaveUpdated(data.updated); + setSaveRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "settings"] }); + }, + onError: (err) => { + if (err instanceof ApiError) { + setSaveErrorMessage(err.message); + setSaveRequestId(requestIdFromError(err)); + return; + } + if (err instanceof Error) { + setSaveErrorMessage(err.message); + return; + } + setSaveErrorMessage("保存失败"); + }, + }); + + return ( + + + 系统设置 + + + + + + + {save.isPending ? : null} + {saveErrorMessage ? : null} + {saveUpdated !== null ? : null} + {saveRequestId ? 请求ID: {saveRequestId} : null} + + {query.isLoading ? ( + + ) : query.isError ? ( + + ) : !query.data ? ( + + ) : ( + + 请求ID: {query.data.request_id} +
save.mutate(values)}> + + 代理设置 + + + + + + + + + + + + + + + + + + + + 安全设置 + + + + + + + 补全任务设置 + + + + + + + + + + )} + + ); +} diff --git a/frontend/src/pages/TagsPage.test.tsx b/frontend/src/pages/TagsPage.test.tsx new file mode 100644 index 0000000..44cd3e8 --- /dev/null +++ b/frontend/src/pages/TagsPage.test.tsx @@ -0,0 +1,84 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import React from "react"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { PlaygroundPage } from "./PlaygroundPage"; +import { TagsPage } from "./TagsPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("TagsPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/tags?limit=50")) { + return new Response( + JSON.stringify({ + ok: true, + items: [{ name: "tag1", translated_name: "t1", count_images: 5 }], + next_cursor: "", + request_id: "req_tags", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders list", async () => { + const qc = makeClient(); + render( + + + + } /> + } /> + + + , + ); + + expect(await screen.findByText("标签列表")).toBeInTheDocument(); + expect(await screen.findByText("tag1")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_tags/)).toBeInTheDocument(); + }); + + it("navigates to playground with included_tags prefill", async () => { + const qc = makeClient(); + render( + + + + } /> + } /> + + + , + ); + + expect(await screen.findByText("tag1")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /按此标签随机一张/ })); + expect(await screen.findByText("随机接口调试")).toBeInTheDocument(); + + await waitFor(() => { + const input = screen.getByLabelText(/包含标签/) as HTMLInputElement; + expect(input.value).toBe("tag1"); + }); + }); +}); diff --git a/frontend/src/pages/TagsPage.tsx b/frontend/src/pages/TagsPage.tsx new file mode 100644 index 0000000..91b35a1 --- /dev/null +++ b/frontend/src/pages/TagsPage.tsx @@ -0,0 +1,97 @@ +import { useQuery } from "@tanstack/react-query"; +import { Alert, Button, Card, Skeleton, Space, Table, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React from "react"; +import { useNavigate } from "react-router-dom"; + +import { ApiError, apiJson } from "../api/client"; + +type TagItem = { + name: string; + translated_name: string | null; + count_images: number | null; +}; + +type TagsListResponse = { + ok: true; + items: TagItem[]; + next_cursor: string; + request_id: string; +}; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +const baseColumns: ColumnsType = [ + { title: "标签", dataIndex: "name", key: "name" }, + { title: "翻译", dataIndex: "translated_name", key: "translated_name" }, + { title: "图片数量", dataIndex: "count_images", key: "count_images" }, +]; + +export function TagsPage() { + const navigate = useNavigate(); + + const query = useQuery({ + queryKey: ["public", "tags", { limit: 50 }], + queryFn: () => apiJson("/tags?limit=50"), + }); + + const columns: ColumnsType = [ + ...baseColumns, + { + title: "操作", + key: "actions", + render: (_, row) => ( + + ), + }, + ]; + + return ( + + + 标签列表 + + + {query.isLoading ? ( + + ) : query.isError ? ( + + ) : !query.data ? ( + + ) : query.data.items.length === 0 ? ( + + ) : ( + + 请求ID: {query.data.request_id} + + rowKey={(row) => row.name} + columns={columns} + dataSource={query.data.items} + pagination={false} + size="small" + style={{ marginTop: 12 }} + /> + + )} + + ); +} + diff --git a/frontend/src/pages/TokensPage.test.tsx b/frontend/src/pages/TokensPage.test.tsx new file mode 100644 index 0000000..46c8941 --- /dev/null +++ b/frontend/src/pages/TokensPage.test.tsx @@ -0,0 +1,183 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, fireEvent, render, screen, within } from "@testing-library/react"; +import React from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { TokensPage } from "./TokensPage"; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("TokensPage", () => { + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + let listCalls = 0; + let tokenLabel: string | null = "acc1"; + let tokenEnabled = true; + let tokenWeight = 1.0; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/admin/api/tokens") && init?.method === "POST") { + const body = init?.body ? JSON.parse(String(init.body)) : {}; + expect(body.refresh_token).toBe("rt_test"); + return new Response(JSON.stringify({ ok: true, token_id: "2", request_id: "req_create" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.endsWith("/admin/api/tokens")) { + listCalls += 1; + return new Response( + JSON.stringify({ + ok: true, + items: [ + { + id: "1", + label: tokenLabel, + enabled: tokenEnabled, + refresh_token_masked: "***", + weight: tokenWeight, + error_count: 0, + backoff_until: null, + last_ok_at: null, + last_fail_at: null, + last_error_code: null, + last_error_msg: null, + }, + ], + request_id: `req_tokens_${listCalls}`, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url.endsWith("/admin/api/tokens/1") && init?.method === "PUT") { + const body = init?.body ? JSON.parse(String(init.body)) : {}; + if ("label" in body) tokenLabel = body.label ?? null; + if ("enabled" in body) tokenEnabled = Boolean(body.enabled); + if ("weight" in body) tokenWeight = Number(body.weight); + return new Response(JSON.stringify({ ok: true, token_id: "1", request_id: "req_update" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.endsWith("/admin/api/tokens/1/test-refresh")) { + expect(init?.method).toBe("POST"); + return new Response( + JSON.stringify({ + ok: true, + expires_in: 123, + user_id: "u1", + proxy: { endpoint_id: "10", pool_id: "1" }, + request_id: "req_test_refresh", + }), + { + status: 200, + headers: { "Content-Type": "application/json" }, + }, + ); + } + if (url.endsWith("/admin/api/tokens/1/reset-failures")) { + expect(init?.method).toBe("POST"); + return new Response(JSON.stringify({ ok: true, token_id: "1", request_id: "req_reset" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response(JSON.stringify({ ok: false, code: "NOT_FOUND", message: "not found", request_id: "req_x", details: {} }), { + status: 404, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + }); + + it("renders list", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("Pixiv 令牌管理")).toBeInTheDocument(); + expect(await screen.findByText("acc1")).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_tokens_1/)).toBeInTheDocument(); + }); + + it("creates token", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("Pixiv 令牌管理")).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: /新增令牌/ })); + const dialog = await screen.findByRole("dialog", { name: /新增令牌/ }); + fireEvent.change(within(dialog).getByPlaceholderText("必填"), { target: { value: "rt_test" } }); + + const form = dialog.querySelector("form"); + expect(form).not.toBeNull(); + fireEvent.submit(form as HTMLFormElement); + + expect(await screen.findByText(/令牌创建成功：2/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_create/)).toBeInTheDocument(); + }); + + it("tests refresh and resets failures", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("Pixiv 令牌管理")).toBeInTheDocument(); + + fireEvent.click(await screen.findByRole("button", { name: /测试刷新/ })); + expect(await screen.findByText(/令牌刷新成功/)).toBeInTheDocument(); + expect(await screen.findByText(/代理 #10，代理池 #1/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_test_refresh/)).toBeInTheDocument(); + + fireEvent.click(await screen.findByRole("button", { name: /重置失败计数/ })); + expect(await screen.findByText(/已重置失败计数：1/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_reset/)).toBeInTheDocument(); + }); + + it("updates token", async () => { + const qc = makeClient(); + render( + + + , + ); + + expect(await screen.findByText("Pixiv 令牌管理")).toBeInTheDocument(); + expect(await screen.findByText("acc1")).toBeInTheDocument(); + + fireEvent.click(await screen.findByRole("button", { name: /编\s*辑/ })); + const dialog = await screen.findByRole("dialog", { name: /编辑令牌/ }); + fireEvent.change(within(dialog).getByPlaceholderText("例如：主账号"), { target: { value: "acc2" } }); + + const form = dialog.querySelector("form"); + expect(form).not.toBeNull(); + fireEvent.submit(form as HTMLFormElement); + + expect(await screen.findByText(/令牌已更新：1/)).toBeInTheDocument(); + expect(await screen.findByText(/请求ID:\s*req_update/)).toBeInTheDocument(); + expect(await screen.findByText("acc2")).toBeInTheDocument(); + + fireEvent.click(await screen.findByRole("button", { name: /禁\s*用/ })); + expect(await screen.findByText(/令牌已更新：1/)).toBeInTheDocument(); + expect(await screen.findByText("否")).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/TokensPage.tsx b/frontend/src/pages/TokensPage.tsx new file mode 100644 index 0000000..372eb74 --- /dev/null +++ b/frontend/src/pages/TokensPage.tsx @@ -0,0 +1,458 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Alert, Button, Card, Form, Input, InputNumber, Modal, Popconfirm, Skeleton, Space, Switch, Table, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import React from "react"; + +import { ApiError, apiJson } from "../api/client"; + +type TokenItem = { + id: string; + label: string | null; + enabled: boolean; + refresh_token_masked: string | null; + weight: number | null; + error_count: number | null; + backoff_until: string | null; + last_ok_at: string | null; + last_fail_at: string | null; +}; + +type TokensListResponse = { + ok: true; + items: TokenItem[]; + request_id: string; +}; + +type CreateTokenFormValues = { + label: string; + refresh_token: string; + enabled: boolean; + weight: number; +}; + +type CreateTokenResponse = { + ok: true; + token_id: string; + request_id: string; +}; + +type UpdateTokenResponse = { + ok: true; + token_id: string; + request_id: string; +}; + +type DeleteTokenResponse = { + ok: true; + token_id: string; + request_id: string; +}; + +type TestRefreshResponse = { + ok: true; + expires_in: number; + user_id: string | null; + proxy: { endpoint_id: string; pool_id: string } | null; + request_id: string; +}; + +type ResetFailuresResponse = { + ok: true; + token_id: string; + request_id: string; +}; + +function requestIdFromError(err: unknown): string | null { + if (!(err instanceof ApiError)) return null; + return err.body?.request_id ? String(err.body.request_id) : null; +} + +function messageFromError(err: unknown): string { + if (err instanceof ApiError) return err.message; + if (err instanceof Error) return err.message; + return "未知错误"; +} + +const columns = (actions: { + onEdit: (row: TokenItem) => void; + onTestRefresh: (id: string) => void; + onResetFailures: (id: string) => void; + onToggleEnabled: (row: TokenItem) => void; + onDelete: (id: string) => void; + testPendingId: string | null; + resetPendingId: string | null; + updatePendingId: string | null; + deletePendingId: string | null; +}): ColumnsType => [ + { title: "标签", dataIndex: "label", key: "label" }, + { title: "启用", dataIndex: "enabled", key: "enabled", render: (value) => (value ? "是" : "否") }, + { title: "掩码令牌", dataIndex: "refresh_token_masked", key: "refresh_token_masked" }, + { title: "权重", dataIndex: "weight", key: "weight" }, + { title: "错误次数", dataIndex: "error_count", key: "error_count" }, + { title: "退避截止", dataIndex: "backoff_until", key: "backoff_until" }, + { title: "最近成功", dataIndex: "last_ok_at", key: "last_ok_at" }, + { title: "最近失败", dataIndex: "last_fail_at", key: "last_fail_at" }, + { + title: "操作", + key: "actions", + render: (_, row) => ( + + + + + + actions.onDelete(row.id)} + > + + + + ), + }, +]; + +type EditTokenFormValues = { + label: string; + enabled: boolean; + weight: number; +}; + +export function TokensPage() { + const queryClient = useQueryClient(); + const [createOpen, setCreateOpen] = React.useState(false); + const [createForm] = Form.useForm(); + const [editOpen, setEditOpen] = React.useState(false); + const [editForm] = Form.useForm(); + const [editingToken, setEditingToken] = React.useState(null); + + const [actionMessage, setActionMessage] = React.useState(null); + const [actionRequestId, setActionRequestId] = React.useState(null); + const [actionErrorMessage, setActionErrorMessage] = React.useState(null); + const [actionErrorRequestId, setActionErrorRequestId] = React.useState(null); + + const query = useQuery({ + queryKey: ["admin", "tokens"], + queryFn: () => apiJson("/admin/api/tokens"), + }); + + const createToken = useMutation({ + mutationFn: (values: CreateTokenFormValues) => + apiJson("/admin/api/tokens", { + method: "POST", + body: JSON.stringify({ + label: values.label || null, + refresh_token: values.refresh_token, + enabled: Boolean(values.enabled), + weight: values.weight, + }), + }), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + setCreateOpen(false); + setActionMessage(`令牌创建成功：${data.token_id}`); + setActionRequestId(data.request_id); + createForm.resetFields(); + queryClient.invalidateQueries({ queryKey: ["admin", "tokens"] }); + }, + onError: (err) => { + setActionErrorMessage(messageFromError(err)); + setActionErrorRequestId(requestIdFromError(err)); + }, + }); + + const updateToken = useMutation({ + mutationFn: (payload: { tokenId: string; body: Record }) => + apiJson(`/admin/api/tokens/${encodeURIComponent(payload.tokenId)}`, { + method: "PUT", + body: JSON.stringify(payload.body), + }), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + setEditOpen(false); + setEditingToken(null); + editForm.resetFields(); + + setActionMessage(`令牌已更新：${data.token_id}`); + setActionRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "tokens"] }); + }, + onError: (err) => { + setActionErrorMessage(messageFromError(err)); + setActionErrorRequestId(requestIdFromError(err)); + queryClient.invalidateQueries({ queryKey: ["admin", "tokens"] }); + }, + }); + + const testRefresh = useMutation({ + mutationFn: (tokenId: string) => + apiJson(`/admin/api/tokens/${encodeURIComponent(tokenId)}/test-refresh`, { method: "POST" }), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + const routeInfo = data.proxy ? `（经代理 #${data.proxy.endpoint_id}，代理池 #${data.proxy.pool_id}）` : "（直连）"; + setActionMessage(`令牌刷新成功，expires_in=${data.expires_in}${routeInfo}`); + setActionRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "tokens"] }); + }, + onError: (err) => { + setActionErrorMessage(messageFromError(err)); + setActionErrorRequestId(requestIdFromError(err)); + queryClient.invalidateQueries({ queryKey: ["admin", "tokens"] }); + }, + }); + + const resetFailures = useMutation({ + mutationFn: (tokenId: string) => + apiJson(`/admin/api/tokens/${encodeURIComponent(tokenId)}/reset-failures`, { method: "POST" }), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + setActionMessage(`已重置失败计数：${data.token_id}`); + setActionRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "tokens"] }); + }, + onError: (err) => { + setActionErrorMessage(messageFromError(err)); + setActionErrorRequestId(requestIdFromError(err)); + }, + }); + + const deleteToken = useMutation({ + mutationFn: (tokenId: string) => + apiJson(`/admin/api/tokens/${encodeURIComponent(tokenId)}`, { method: "DELETE" }), + onMutate: () => { + setActionMessage(null); + setActionRequestId(null); + setActionErrorMessage(null); + setActionErrorRequestId(null); + }, + onSuccess: (data) => { + setActionMessage(`令牌已删除：${data.token_id}`); + setActionRequestId(data.request_id); + queryClient.invalidateQueries({ queryKey: ["admin", "tokens"] }); + queryClient.invalidateQueries({ queryKey: ["admin", "bindings"] }); + }, + onError: (err) => { + setActionErrorMessage(messageFromError(err)); + setActionErrorRequestId(requestIdFromError(err)); + queryClient.invalidateQueries({ queryKey: ["admin", "tokens"] }); + }, + }); + + const openEdit = (row: TokenItem) => { + setEditingToken(row); + setEditOpen(true); + editForm.setFieldsValue({ + label: row.label || "", + enabled: Boolean(row.enabled), + weight: row.weight != null ? row.weight : 1.0, + }); + }; + + const closeEdit = () => { + setEditOpen(false); + setEditingToken(null); + editForm.resetFields(); + }; + + return ( + + + Pixiv 令牌管理 + + + {actionMessage ? : null} + {actionRequestId ? 请求ID: {actionRequestId} : null} + {actionErrorMessage ? : null} + {actionErrorRequestId ? 请求ID: {actionErrorRequestId} : null} + + + + + + { + setCreateOpen(false); + createForm.resetFields(); + }} + footer={null} + destroyOnClose + > + + form={createForm} + layout="vertical" + initialValues={{ label: "", refresh_token: "", enabled: true, weight: 1.0 }} + onFinish={(values) => createToken.mutate(values)} + > + + + + + + + + + + + + + + + + + + + + + + + + + + + + form={editForm} + layout="vertical" + initialValues={{ label: "", enabled: true, weight: 1.0 }} + onFinish={(values) => { + const tokenId = editingToken?.id; + if (!tokenId) return; + updateToken.mutate({ + tokenId, + body: { + label: values.label.trim() ? values.label.trim() : null, + enabled: Boolean(values.enabled), + weight: values.weight, + }, + }); + }} + > + + + + + + + + + + + + + + + + + + + + + + {query.isLoading ? ( + + ) : query.isError ? ( + + ) : !query.data ? ( + + ) : query.data.items.length === 0 ? ( + + ) : ( + + 请求ID: {query.data.request_id} + + rowKey={(row) => row.id} + columns={columns({ + onEdit: (row) => openEdit(row), + onTestRefresh: (id) => testRefresh.mutate(id), + onResetFailures: (id) => resetFailures.mutate(id), + onToggleEnabled: (row) => + updateToken.mutate({ + tokenId: row.id, + body: { enabled: !row.enabled }, + }), + onDelete: (id) => deleteToken.mutate(id), + testPendingId: testRefresh.isPending ? testRefresh.variables ?? null : null, + resetPendingId: resetFailures.isPending ? resetFailures.variables ?? null : null, + updatePendingId: updateToken.isPending ? updateToken.variables?.tokenId ?? null : null, + deletePendingId: deleteToken.isPending ? deleteToken.variables ?? null : null, + })} + dataSource={query.data.items} + pagination={false} + size="small" + style={{ marginTop: 12 }} + /> + + )} + + ); +} diff --git a/frontend/src/test/setup.ts b/frontend/src/test/setup.ts new file mode 100644 index 0000000..925377e --- /dev/null +++ b/frontend/src/test/setup.ts @@ -0,0 +1,23 @@ +import "@testing-library/jest-dom/vitest"; + +if (!window.matchMedia) { + Object.defineProperty(window, "matchMedia", { + writable: true, + value: (query: string) => ({ + matches: false, + media: query, + onchange: null, + addListener: () => {}, + removeListener: () => {}, + addEventListener: () => {}, + removeEventListener: () => {}, + dispatchEvent: () => false, + }), + }); +} + +const originalGetComputedStyle = window.getComputedStyle; +window.getComputedStyle = (elt: Element, pseudoElt?: string | null) => { + if (pseudoElt) return originalGetComputedStyle(elt); + return originalGetComputedStyle(elt); +}; diff --git a/frontend/src/vite-env.d.ts b/frontend/src/vite-env.d.ts new file mode 100644 index 0000000..ed77210 --- /dev/null +++ b/frontend/src/vite-env.d.ts @@ -0,0 +1,2 @@ +/// + diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json new file mode 100644 index 0000000..717bd84 --- /dev/null +++ b/frontend/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client", "node"] + }, + "include": ["src", "vite.config.ts"] +} diff --git a/frontend/tsconfig.node.json b/frontend/tsconfig.node.json new file mode 100644 index 0000000..c88b91b --- /dev/null +++ b/frontend/tsconfig.node.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "composite": true, + "skipLibCheck": true, + "module": "ESNext", + "moduleResolution": "Bundler", + "allowSyntheticDefaultImports": true + }, + "include": ["vite.config.ts"] +} + diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts new file mode 100644 index 0000000..18910fe --- /dev/null +++ b/frontend/vite.config.ts @@ -0,0 +1,30 @@ +import react from "@vitejs/plugin-react"; +import { loadEnv } from "vite"; +import { defineConfig } from "vitest/config"; + +export default defineConfig(({ mode }) => { + const env = loadEnv(mode, process.cwd(), ""); + const apiTarget = (env.VITE_API_PROXY_TARGET || "http://localhost:8000").trim(); + const buildMinifyRaw = (process.env.VITE_BUILD_MINIFY || env.VITE_BUILD_MINIFY || "").trim().toLowerCase(); + const buildMinify = buildMinifyRaw ? !["0", "false", "no", "off"].includes(buildMinifyRaw) : true; + + return { + base: mode === "production" ? "/admin/" : "/", + plugins: [react()], + server: { + proxy: { + "/admin/api": apiTarget, + "/metrics": apiTarget, + }, + }, + build: { + minify: buildMinify ? "esbuild" : false, + }, + test: { + environment: "jsdom", + setupFiles: ["./src/test/setup.ts"], + pool: "threads", + poolOptions: { threads: { singleThread: true } }, + }, + }; +}); diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..080ac72 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,3 @@ +-r requirements.in +pytest==8.4.1 + diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..5bc1723 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,106 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --no-emit-index-url --output-file=requirements-dev.txt requirements-dev.in +# +aiosqlite==0.22.1 + # via -r requirements.in +alembic==1.18.3 + # via -r requirements.in +annotated-doc==0.0.4 + # via fastapi +annotated-types==0.7.0 + # via pydantic +anyio==4.12.1 + # via + # httpx + # starlette + # watchfiles +certifi==2026.1.4 + # via + # httpcore + # httpx +cffi==2.0.0 + # via cryptography +click==8.3.1 + # via uvicorn +colorama==0.4.6 + # via + # click + # pytest + # uvicorn +cryptography==42.0.5 + # via -r requirements.in +fastapi==0.128.6 + # via -r requirements.in +greenlet==3.3.1 + # via sqlalchemy +h11==0.16.0 + # via + # httpcore + # uvicorn +httpcore==1.0.9 + # via httpx +httptools==0.7.1 + # via uvicorn +httpx==0.28.1 + # via -r requirements.in +idna==3.11 + # via + # anyio + # httpx +iniconfig==2.3.0 + # via pytest +mako==1.3.10 + # via alembic +markupsafe==3.0.3 + # via mako +packaging==26.0 + # via pytest +pluggy==1.6.0 + # via pytest +prometheus-client==0.24.1 + # via -r requirements.in +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via fastapi +pydantic-core==2.41.5 + # via pydantic +pygments==2.19.2 + # via pytest +pytest==8.4.1 + # via -r requirements-dev.in +python-dotenv==1.2.1 + # via uvicorn +python-multipart==0.0.22 + # via -r requirements.in +pyyaml==6.0.3 + # via uvicorn +sqlalchemy==2.0.46 + # via + # -r requirements.in + # alembic +starlette==0.52.1 + # via fastapi +typing-extensions==4.15.0 + # via + # alembic + # anyio + # fastapi + # pydantic + # pydantic-core + # sqlalchemy + # starlette + # typing-inspection +typing-inspection==0.4.2 + # via + # fastapi + # pydantic +uvicorn[standard]==0.40.0 + # via -r requirements.in +watchfiles==1.1.1 + # via uvicorn +websockets==16.0 + # via uvicorn diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..8528d2f --- /dev/null +++ b/requirements.in @@ -0,0 +1,9 @@ +fastapi==0.128.6 +uvicorn[standard]==0.40.0 +sqlalchemy==2.0.46 +aiosqlite==0.22.1 +alembic==1.18.3 +httpx==0.28.1 +cryptography==42.0.5 +python-multipart==0.0.22 +prometheus-client==0.24.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2b13d92 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,95 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --no-emit-index-url --output-file=requirements.txt requirements.in +# +aiosqlite==0.22.1 + # via -r requirements.in +alembic==1.18.3 + # via -r requirements.in +annotated-doc==0.0.4 + # via fastapi +annotated-types==0.7.0 + # via pydantic +anyio==4.12.1 + # via + # httpx + # starlette + # watchfiles +certifi==2026.1.4 + # via + # httpcore + # httpx +cffi==2.0.0 + # via cryptography +click==8.3.1 + # via uvicorn +colorama==0.4.6 + # via + # click + # uvicorn +cryptography==42.0.5 + # via -r requirements.in +fastapi==0.128.6 + # via -r requirements.in +greenlet==3.3.1 + # via sqlalchemy +h11==0.16.0 + # via + # httpcore + # uvicorn +httpcore==1.0.9 + # via httpx +httptools==0.7.1 + # via uvicorn +httpx==0.28.1 + # via -r requirements.in +idna==3.11 + # via + # anyio + # httpx +mako==1.3.10 + # via alembic +markupsafe==3.0.3 + # via mako +prometheus-client==0.24.1 + # via -r requirements.in +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via fastapi +pydantic-core==2.41.5 + # via pydantic +python-dotenv==1.2.1 + # via uvicorn +python-multipart==0.0.22 + # via -r requirements.in +pyyaml==6.0.3 + # via uvicorn +sqlalchemy==2.0.46 + # via + # -r requirements.in + # alembic +starlette==0.52.1 + # via fastapi +typing-extensions==4.15.0 + # via + # alembic + # anyio + # fastapi + # pydantic + # pydantic-core + # sqlalchemy + # starlette + # typing-inspection +typing-inspection==0.4.2 + # via + # fastapi + # pydantic +uvicorn[standard]==0.40.0 + # via -r requirements.in +watchfiles==1.1.1 + # via uvicorn +websockets==16.0 + # via uvicorn diff --git a/scripts/perf/bench_random_asgi.py b/scripts/perf/bench_random_asgi.py new file mode 100644 index 0000000..850e281 --- /dev/null +++ b/scripts/perf/bench_random_asgi.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import platform +import random +import statistics +import sys +import tempfile +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import httpx + +REPO_ROOT = Path(__file__).resolve().parents[2] +BACKEND_DIR = REPO_ROOT / "backend" +if str(BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(BACKEND_DIR)) + +from app.db.models.base import Base +from app.main import create_app + + +@dataclass(frozen=True, slots=True) +class BenchResult: + total: int + ok: int + no_match: int + other_error: int + durations_s: list[float] + + +def _p(sorted_values: list[float], p: float) -> float: + if not sorted_values: + return 0.0 + p = max(0.0, min(float(p), 100.0)) + if p == 0.0: + return sorted_values[0] + if p == 100.0: + return sorted_values[-1] + k = (len(sorted_values) - 1) * (p / 100.0) + f = int(k) + c = min(f + 1, len(sorted_values) - 1) + if f == c: + return sorted_values[f] + d0 = sorted_values[f] * (c - k) + d1 = sorted_values[c] * (k - f) + return d0 + d1 + + +async def _migrate_and_seed(*, app, seed_images: int) -> None: + engine = app.state.engine + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + seed_images_i = max(1, int(seed_images)) + values: list[tuple[Any, ...]] = [] + for i in range(seed_images_i): + illust_id = 500_000_000 + i + random_key = (i + 0.5) / seed_images_i + values.append( + ( + illust_id, + 0, + "jpg", + f"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/{illust_id}_p0.jpg", + f"/i/seed_{illust_id}.jpg", + float(random_key), + 1000, + 1000, + 1.0, + 3, + 0, + 0, + 1, + 0, + None, + None, + None, + None, + None, + None, + None, + None, + None, + now, + now, + ) + ) + + await conn.exec_driver_sql( + """ +INSERT INTO images ( + illust_id, page_index, ext, original_url, proxy_path, random_key, + width, height, aspect_ratio, orientation, x_restrict, ai_type, + status, fail_count, created_import_id, + last_fail_at, last_ok_at, last_error_code, last_error_msg, + user_id, user_name, title, created_at_pixiv, + added_at, updated_at +) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?); +""".strip(), + values, + ) + + +async def _run_bench(*, app, total_requests: int, concurrency: int, endpoint: str) -> BenchResult: + total_i = max(1, int(total_requests)) + conc_i = max(1, int(concurrency)) + + transport = httpx.ASGITransport(app=app) + timeout = httpx.Timeout(30.0, connect=10.0) + client = httpx.AsyncClient(transport=transport, base_url="http://bench.local", timeout=timeout) + + semaphore = asyncio.Semaphore(conc_i) + durations_s: list[float] = [] + ok = 0 + no_match = 0 + other_error = 0 + + async def one(i: int) -> None: + nonlocal ok, no_match, other_error + _ = i + async with semaphore: + started = time.perf_counter() + try: + resp = await client.get(endpoint) + if resp.status_code == 200: + ok += 1 + elif resp.status_code == 404: + no_match += 1 + else: + other_error += 1 + except Exception: + other_error += 1 + finally: + durations_s.append(time.perf_counter() - started) + + try: + tasks = [asyncio.create_task(one(i)) for i in range(total_i)] + await asyncio.gather(*tasks) + finally: + await client.aclose() + + return BenchResult( + total=total_i, + ok=ok, + no_match=no_match, + other_error=other_error, + durations_s=durations_s, + ) + + +def _build_report(*, args, result: BenchResult, elapsed_s: float) -> dict[str, Any]: + ds = sorted(float(x) for x in result.durations_s if x is not None and x >= 0) + mean = statistics.fmean(ds) if ds else 0.0 + p50 = _p(ds, 50.0) + p90 = _p(ds, 90.0) + p99 = _p(ds, 99.0) + + report: dict[str, Any] = { + "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "endpoint": args.endpoint, + "requests": { + "total": result.total, + "concurrency": args.concurrency, + "ok": result.ok, + "no_match": result.no_match, + "other_error": result.other_error, + }, + "latency_s": { + "min": ds[0] if ds else 0.0, + "p50": p50, + "p90": p90, + "p99": p99, + "max": ds[-1] if ds else 0.0, + "mean": mean, + }, + "throughput": { + "elapsed_s": float(elapsed_s), + "rps": float(result.total / elapsed_s) if elapsed_s > 0 else 0.0, + }, + "seed_images": int(args.seed_images), + "env": { + "python": sys.version.split()[0], + "platform": platform.platform(), + "machine": platform.machine(), + }, + } + return report + + +async def main_async(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="In-process /random load test using httpx ASGITransport.") + parser.add_argument("--requests", type=int, default=500) + parser.add_argument("--concurrency", type=int, default=50) + parser.add_argument("--seed-images", type=int, default=5000) + parser.add_argument("--endpoint", type=str, default="/random?format=json&attempts=1") + parser.add_argument("--output", type=str, default="") + args = parser.parse_args(argv) + + with tempfile.TemporaryDirectory(prefix="new_pixiv_bench_") as td: + db_path = Path(td) / "bench.db" + db_url = "sqlite+aiosqlite:///" + db_path.as_posix() + + os.environ["APP_ENV"] = os.environ.get("APP_ENV") or "dev" + os.environ["DATABASE_URL"] = db_url + os.environ["SECRET_KEY"] = os.environ.get("SECRET_KEY") or "dev-secret-key" + os.environ["ADMIN_USERNAME"] = os.environ.get("ADMIN_USERNAME") or "admin" + + app = create_app() + try: + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpcore").setLevel(logging.WARNING) + + await _migrate_and_seed(app=app, seed_images=args.seed_images) + + # Warm-up: one request to prime import paths/DB connections. + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://bench.local", + ) as c: + await c.get("/healthz") + + started = time.perf_counter() + result = await _run_bench( + app=app, + total_requests=args.requests, + concurrency=args.concurrency, + endpoint=args.endpoint, + ) + elapsed_s = time.perf_counter() - started + + report = _build_report(args=args, result=result, elapsed_s=elapsed_s) + + out_path = (args.output or "").strip() + if out_path: + out = Path(out_path) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"[bench_random_asgi] wrote report: {out}") + else: + print(json.dumps(report, ensure_ascii=False, indent=2)) + finally: + engine = getattr(app.state, "engine", None) + if engine is not None: + await engine.dispose() + + return 0 + + +def main(argv: list[str] | None = None) -> int: + return asyncio.run(main_async(argv)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_all.ps1 b/scripts/test_all.ps1 new file mode 100644 index 0000000..b2637d1 --- /dev/null +++ b/scripts/test_all.ps1 @@ -0,0 +1,21 @@ +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +Set-Location $repoRoot + +Write-Host "== test_all: backend ==" +& (Join-Path $PSScriptRoot "test_backend.ps1") + +Write-Host "== test_all: frontend ==" +& (Join-Path $PSScriptRoot "test_frontend.ps1") + +if (Test-Path "deploy\\docker-compose.yml") { + if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + throw "docker not found but deploy/docker-compose.yml exists" + } + Write-Host "== test_all: docker compose config ==" + docker compose -f deploy/docker-compose.yml config | Out-Null +} + +Write-Host "== test_all: OK ==" + diff --git a/scripts/test_backend.ps1 b/scripts/test_backend.ps1 new file mode 100644 index 0000000..8c1af85 --- /dev/null +++ b/scripts/test_backend.ps1 @@ -0,0 +1,22 @@ +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +Set-Location $repoRoot + +if (-not (Test-Path "backend")) { + Write-Host "[test_backend] backend/ not found -> skip" + exit 0 +} + +Write-Host "[test_backend] python compileall" +py -3.11 -m compileall -q backend + +if (Test-Path "backend\\tests") { + Write-Host "[test_backend] pytest" + py -3.11 -m pytest -q backend\\tests +} else { + Write-Host "[test_backend] backend/tests not found -> skip pytest" +} + +Write-Host "[test_backend] OK" + diff --git a/scripts/test_frontend.ps1 b/scripts/test_frontend.ps1 new file mode 100644 index 0000000..f7e196a --- /dev/null +++ b/scripts/test_frontend.ps1 @@ -0,0 +1,32 @@ +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +Set-Location $repoRoot + +if (-not (Test-Path "frontend\\package.json")) { + Write-Host "[test_frontend] frontend/package.json not found -> skip" + exit 0 +} + +if (-not (Get-Command npm -ErrorAction SilentlyContinue)) { + throw "npm not found but frontend/package.json exists" +} + +Write-Host "[test_frontend] npm -C frontend run lint (if present)" +try { npm -C frontend run lint } catch { Write-Host "[test_frontend] lint missing or failed: $($_.Exception.Message)"; throw } + +Write-Host "[test_frontend] npm -C frontend run typecheck (if present)" +try { npm -C frontend run typecheck } catch { Write-Host "[test_frontend] typecheck missing or failed: $($_.Exception.Message)"; throw } + +Write-Host "[test_frontend] npm -C frontend test (if present)" +if (-not $env:VITEST_MAX_THREADS) { $env:VITEST_MAX_THREADS = "1" } +if (-not $env:VITEST_MIN_THREADS) { $env:VITEST_MIN_THREADS = "1" } +Write-Host "[test_frontend] vitest threads: min=$($env:VITEST_MIN_THREADS) max=$($env:VITEST_MAX_THREADS)" +try { npm -C frontend test } catch { Write-Host "[test_frontend] test missing or failed: $($_.Exception.Message)"; throw } + +Write-Host "[test_frontend] npm -C frontend run build (if present)" +if (-not $env:VITE_BUILD_MINIFY) { $env:VITE_BUILD_MINIFY = "false" } +Write-Host "[test_frontend] vite build minify=$($env:VITE_BUILD_MINIFY)" +try { npm -C frontend run build } catch { Write-Host "[test_frontend] build missing or failed: $($_.Exception.Message)"; throw } + +Write-Host "[test_frontend] OK" diff --git a/scripts/verify_backend_install.ps1 b/scripts/verify_backend_install.ps1 new file mode 100644 index 0000000..c22c428 --- /dev/null +++ b/scripts/verify_backend_install.ps1 @@ -0,0 +1,36 @@ +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +Set-Location $repoRoot + +if (-not (Test-Path "requirements-dev.txt")) { + throw "requirements-dev.txt not found at repo root" +} + +if (-not (Test-Path "backend")) { + throw "backend/ not found" +} + +$venvRoot = Join-Path $repoRoot ".venv" +$venvPath = Join-Path $venvRoot "backend_verify" + +if (Test-Path $venvPath) { + Remove-Item -Recurse -Force $venvPath +} + +Write-Host "[verify_backend_install] create venv: $venvPath" +py -3.11 -m venv $venvPath + +$python = Join-Path $venvPath "Scripts\\python.exe" + +Write-Host "[verify_backend_install] pip install -U pip" +& $python -m pip install -U pip + +Write-Host "[verify_backend_install] pip install -r requirements-dev.txt" +& $python -m pip install -r requirements-dev.txt + +Write-Host "[verify_backend_install] pytest" +& $python -m pytest -q backend\\tests + +Write-Host "[verify_backend_install] OK" +