458 lines
18 KiB
Python
458 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app.core.security import create_jwt
|
|
from app.db.models.base import Base
|
|
from app.main import create_app
|
|
from app.worker import build_default_dispatcher, poll_and_execute_jobs
|
|
|
|
|
|
def test_admin_imports_json_happy_path(tmp_path: Path, monkeypatch) -> None:
|
|
db_path = tmp_path / "admin_imports.db"
|
|
db_url = "sqlite+aiosqlite:///" + db_path.as_posix()
|
|
|
|
monkeypatch.setenv("APP_ENV", "dev")
|
|
monkeypatch.setenv("DATABASE_URL", db_url)
|
|
monkeypatch.setenv("SECRET_KEY", "secret_test")
|
|
monkeypatch.setenv("ADMIN_USERNAME", "admin")
|
|
|
|
app = create_app()
|
|
|
|
async def _migrate() -> None:
|
|
async with app.state.engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
asyncio.run(_migrate())
|
|
|
|
token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600)
|
|
with TestClient(app) as client:
|
|
text = "\n".join(
|
|
[
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg",
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg",
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p1.png",
|
|
"https://www.pixiv.net/artworks/12345678",
|
|
]
|
|
)
|
|
|
|
resp = client.post(
|
|
"/admin/api/imports",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
json={"text": text, "dry_run": False, "hydrate_on_import": False, "source": "manual"},
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["ok"] is True
|
|
assert body["accepted"] == 2
|
|
assert body["deduped"] == 1
|
|
assert len(body["errors"]) == 1
|
|
assert body["import_id"].isdigit()
|
|
assert body["job_id"].isdigit()
|
|
executed_inline = bool(body.get("executed_inline"))
|
|
assert body["request_id"] == "req_test"
|
|
assert resp.headers["X-Request-Id"] == "req_test"
|
|
|
|
if not executed_inline:
|
|
async def _run_worker() -> None:
|
|
dispatcher = build_default_dispatcher(app.state.engine)
|
|
ran = await poll_and_execute_jobs(app.state.engine, dispatcher, worker_id="test-worker", max_jobs=10)
|
|
assert ran >= 1
|
|
|
|
asyncio.run(_run_worker())
|
|
|
|
async def _fetch_import_counts() -> tuple[int, int, int, int]:
|
|
async with app.state.engine.connect() as conn:
|
|
result = await conn.exec_driver_sql("SELECT total, accepted, success, failed FROM imports")
|
|
row = result.fetchone()
|
|
assert row is not None
|
|
return (int(row[0]), int(row[1]), int(row[2]), int(row[3]))
|
|
|
|
assert asyncio.run(_fetch_import_counts()) == (4, 2, 2, 1)
|
|
|
|
|
|
def test_admin_imports_multipart_happy_path(tmp_path: Path, monkeypatch) -> None:
|
|
db_path = tmp_path / "admin_imports_multipart.db"
|
|
db_url = "sqlite+aiosqlite:///" + db_path.as_posix()
|
|
|
|
monkeypatch.setenv("APP_ENV", "dev")
|
|
monkeypatch.setenv("DATABASE_URL", db_url)
|
|
monkeypatch.setenv("SECRET_KEY", "secret_test")
|
|
monkeypatch.setenv("ADMIN_USERNAME", "admin")
|
|
|
|
app = create_app()
|
|
|
|
async def _migrate() -> None:
|
|
async with app.state.engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
asyncio.run(_migrate())
|
|
|
|
token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600)
|
|
with TestClient(app) as client:
|
|
text = "\n".join(
|
|
[
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg",
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg",
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p1.png",
|
|
"https://www.pixiv.net/artworks/12345678",
|
|
]
|
|
)
|
|
|
|
resp = client.post(
|
|
"/admin/api/imports",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
data={"dry_run": "false", "hydrate_on_import": "false", "source": "manual"},
|
|
files={"file": ("urls.txt", text.encode("utf-8"), "text/plain")},
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["ok"] is True
|
|
assert body["accepted"] == 2
|
|
assert body["deduped"] == 1
|
|
assert len(body["errors"]) == 1
|
|
assert body["import_id"].isdigit()
|
|
assert body["job_id"].isdigit()
|
|
executed_inline = bool(body.get("executed_inline"))
|
|
assert body["request_id"] == "req_test"
|
|
assert resp.headers["X-Request-Id"] == "req_test"
|
|
|
|
import_id = int(body["import_id"])
|
|
|
|
if not executed_inline:
|
|
async def _run_worker() -> None:
|
|
dispatcher = build_default_dispatcher(app.state.engine)
|
|
ran = await poll_and_execute_jobs(app.state.engine, dispatcher, worker_id="test-worker", max_jobs=10)
|
|
assert ran >= 1
|
|
|
|
asyncio.run(_run_worker())
|
|
|
|
async def _fetch_import_counts() -> tuple[int, int, int, int]:
|
|
async with app.state.engine.connect() as conn:
|
|
result = await conn.exec_driver_sql(
|
|
"SELECT total, accepted, success, failed FROM imports WHERE id = ?",
|
|
(import_id,),
|
|
)
|
|
row = result.fetchone()
|
|
assert row is not None
|
|
return (int(row[0]), int(row[1]), int(row[2]), int(row[3]))
|
|
|
|
assert asyncio.run(_fetch_import_counts()) == (4, 2, 2, 1)
|
|
|
|
|
|
def test_admin_imports_multipart_pixiv_batch_downloader_json_imports_metadata(tmp_path: Path, monkeypatch) -> None:
|
|
db_path = tmp_path / "admin_imports_pbd_json.db"
|
|
db_url = "sqlite+aiosqlite:///" + db_path.as_posix()
|
|
|
|
monkeypatch.setenv("APP_ENV", "dev")
|
|
monkeypatch.setenv("DATABASE_URL", db_url)
|
|
monkeypatch.setenv("SECRET_KEY", "secret_test")
|
|
monkeypatch.setenv("ADMIN_USERNAME", "admin")
|
|
|
|
app = create_app()
|
|
|
|
async def _migrate() -> None:
|
|
async with app.state.engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
asyncio.run(_migrate())
|
|
|
|
token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600)
|
|
with TestClient(app) as client:
|
|
payload = (
|
|
b"["
|
|
+ b"{"
|
|
+ b"\"id\":\"12345678_p0\","
|
|
+ b"\"index\":0,"
|
|
+ b"\"original\":\"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg\","
|
|
+ b"\"fullWidth\":1000,\"fullHeight\":1500,"
|
|
+ b"\"xRestrict\":0,\"aiType\":1,\"type\":0,"
|
|
+ b"\"userId\":\"42\",\"user\":\"alice\",\"title\":\"hello\","
|
|
+ b"\"date\":\"2023-01-01T00:00:00+09:00\","
|
|
+ b"\"bmk\":12,\"viewCount\":34,\"commentCount\":5,"
|
|
+ b"\"tags\":[\"loli\",\"cute\"]"
|
|
+ b"}"
|
|
+ b"]"
|
|
)
|
|
|
|
resp = client.post(
|
|
"/admin/api/imports",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
data={"dry_run": "false", "hydrate_on_import": "true", "source": "manual"},
|
|
files={"file": ("result.json", payload, "application/json")},
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["ok"] is True
|
|
assert body["accepted"] == 1
|
|
assert body["deduped"] == 0
|
|
assert body["errors"] == []
|
|
assert body["import_id"].isdigit()
|
|
assert body["job_id"].isdigit()
|
|
executed_inline = bool(body.get("executed_inline"))
|
|
|
|
import_id = int(body["import_id"])
|
|
|
|
if not executed_inline:
|
|
async def _run_worker() -> None:
|
|
dispatcher = build_default_dispatcher(app.state.engine)
|
|
ran = await poll_and_execute_jobs(app.state.engine, dispatcher, worker_id="test-worker", max_jobs=10)
|
|
assert ran >= 1
|
|
|
|
asyncio.run(_run_worker())
|
|
|
|
async def _fetch_image_and_tag_counts() -> tuple[int, int, int]:
|
|
async with app.state.engine.connect() as conn:
|
|
img = await conn.exec_driver_sql(
|
|
"SELECT width,height,orientation,ai_type,x_restrict,user_id,bookmark_count FROM images WHERE illust_id=12345678 AND page_index=0"
|
|
)
|
|
row = img.fetchone()
|
|
assert row is not None
|
|
tag_count = int((await conn.exec_driver_sql("SELECT COUNT(*) FROM tags")).scalar_one())
|
|
link_count = int((await conn.exec_driver_sql("SELECT COUNT(*) FROM image_tags")).scalar_one())
|
|
return (int(row[0]), int(row[1]), tag_count, link_count)
|
|
|
|
assert asyncio.run(_fetch_image_and_tag_counts()) == (1000, 1500, 2, 2)
|
|
|
|
async def _hydrate_job_count() -> int:
|
|
async with app.state.engine.connect() as conn:
|
|
return int((await conn.exec_driver_sql("SELECT COUNT(*) FROM jobs WHERE type='hydrate_metadata'")).scalar_one())
|
|
|
|
assert asyncio.run(_hydrate_job_count()) == 0
|
|
|
|
async def _fetch_import_counts() -> tuple[int, int, int, int]:
|
|
async with app.state.engine.connect() as conn:
|
|
result = await conn.exec_driver_sql(
|
|
"SELECT total, accepted, success, failed FROM imports WHERE id = ?",
|
|
(import_id,),
|
|
)
|
|
row = result.fetchone()
|
|
assert row is not None
|
|
return (int(row[0]), int(row[1]), int(row[2]), int(row[3]))
|
|
|
|
assert asyncio.run(_fetch_import_counts()) == (1, 1, 1, 0)
|
|
|
|
|
|
def test_admin_imports_dry_run_preview_does_not_write_db(tmp_path: Path, monkeypatch) -> None:
|
|
db_path = tmp_path / "admin_imports_dry_run.db"
|
|
db_url = "sqlite+aiosqlite:///" + db_path.as_posix()
|
|
|
|
monkeypatch.setenv("APP_ENV", "dev")
|
|
monkeypatch.setenv("DATABASE_URL", db_url)
|
|
monkeypatch.setenv("SECRET_KEY", "secret_test")
|
|
monkeypatch.setenv("ADMIN_USERNAME", "admin")
|
|
|
|
app = create_app()
|
|
|
|
async def _migrate() -> None:
|
|
async with app.state.engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
asyncio.run(_migrate())
|
|
|
|
token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600)
|
|
with TestClient(app) as client:
|
|
text = "\n".join(
|
|
[
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg",
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg",
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p1.png",
|
|
"https://www.pixiv.net/artworks/12345678",
|
|
]
|
|
)
|
|
|
|
resp = client.post(
|
|
"/admin/api/imports",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
json={"text": text, "dry_run": True, "hydrate_on_import": False, "source": "manual"},
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["ok"] is True
|
|
assert body["accepted"] == 2
|
|
assert body["deduped"] == 1
|
|
assert len(body["errors"]) == 1
|
|
assert len(body["preview"]) == 2
|
|
assert body["import_id"] == ""
|
|
assert body["job_id"] == ""
|
|
assert body["request_id"] == "req_test"
|
|
|
|
async def _count_images() -> int:
|
|
async with app.state.engine.connect() as conn:
|
|
result = await conn.exec_driver_sql("SELECT COUNT(*) FROM images")
|
|
return int(result.scalar_one())
|
|
|
|
assert asyncio.run(_count_images()) == 0
|
|
|
|
|
|
def test_admin_imports_rollback_disable_and_delete(tmp_path: Path, monkeypatch) -> None:
|
|
db_path = tmp_path / "admin_imports_rollback.db"
|
|
db_url = "sqlite+aiosqlite:///" + db_path.as_posix()
|
|
|
|
monkeypatch.setenv("APP_ENV", "dev")
|
|
monkeypatch.setenv("DATABASE_URL", db_url)
|
|
monkeypatch.setenv("SECRET_KEY", "secret_test")
|
|
monkeypatch.setenv("ADMIN_USERNAME", "admin")
|
|
|
|
app = create_app()
|
|
|
|
async def _migrate() -> None:
|
|
async with app.state.engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
asyncio.run(_migrate())
|
|
|
|
token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600)
|
|
with TestClient(app) as client:
|
|
text = "\n".join(
|
|
[
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p0.jpg",
|
|
"https://i.pximg.net/img-original/img/2023/01/01/00/00/00/12345678_p1.png",
|
|
]
|
|
)
|
|
|
|
create_resp = client.post(
|
|
"/admin/api/imports",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
json={"text": text, "dry_run": False, "hydrate_on_import": False, "source": "manual"},
|
|
)
|
|
assert create_resp.status_code == 200
|
|
create_body = create_resp.json()
|
|
import_id = int(create_body["import_id"])
|
|
executed_inline = bool(create_body.get("executed_inline"))
|
|
|
|
if not executed_inline:
|
|
async def _run_worker() -> None:
|
|
dispatcher = build_default_dispatcher(app.state.engine)
|
|
ran = await poll_and_execute_jobs(app.state.engine, dispatcher, worker_id="test-worker", max_jobs=10)
|
|
assert ran >= 1
|
|
|
|
asyncio.run(_run_worker())
|
|
|
|
disable_resp = client.post(
|
|
f"/admin/api/imports/{import_id}/rollback",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
json={"mode": "disable"},
|
|
)
|
|
assert disable_resp.status_code == 200
|
|
assert disable_resp.json()["updated"] == 2
|
|
|
|
async def _count_status(status: int) -> int:
|
|
async with app.state.engine.connect() as conn:
|
|
result = await conn.exec_driver_sql(
|
|
"SELECT COUNT(*) FROM images WHERE created_import_id = ? AND status = ?",
|
|
(import_id, status),
|
|
)
|
|
return int(result.scalar_one())
|
|
|
|
assert asyncio.run(_count_status(2)) == 2
|
|
|
|
delete_resp = client.post(
|
|
f"/admin/api/imports/{import_id}/rollback",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
json={"mode": "delete"},
|
|
)
|
|
assert delete_resp.status_code == 200
|
|
assert delete_resp.json()["updated"] == 2
|
|
assert asyncio.run(_count_status(4)) == 2
|
|
|
|
|
|
def test_admin_imports_invalid_body_returns_400(tmp_path: Path, monkeypatch) -> None:
|
|
db_path = tmp_path / "admin_imports_invalid_body.db"
|
|
db_url = "sqlite+aiosqlite:///" + db_path.as_posix()
|
|
|
|
monkeypatch.setenv("APP_ENV", "dev")
|
|
monkeypatch.setenv("DATABASE_URL", db_url)
|
|
monkeypatch.setenv("SECRET_KEY", "secret_test")
|
|
monkeypatch.setenv("ADMIN_USERNAME", "admin")
|
|
|
|
app = create_app()
|
|
|
|
async def _migrate() -> None:
|
|
async with app.state.engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
asyncio.run(_migrate())
|
|
|
|
token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600)
|
|
with TestClient(app) as client:
|
|
resp = client.post(
|
|
"/admin/api/imports",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
json={"text": "", "dry_run": False},
|
|
)
|
|
assert resp.status_code == 400
|
|
body = resp.json()
|
|
assert body["ok"] is False
|
|
assert body["code"] == "BAD_REQUEST"
|
|
assert body["request_id"] == "req_test"
|
|
|
|
|
|
def test_admin_imports_rollback_not_found_returns_404(tmp_path: Path, monkeypatch) -> None:
|
|
db_path = tmp_path / "admin_imports_rollback_not_found.db"
|
|
db_url = "sqlite+aiosqlite:///" + db_path.as_posix()
|
|
|
|
monkeypatch.setenv("APP_ENV", "dev")
|
|
monkeypatch.setenv("DATABASE_URL", db_url)
|
|
monkeypatch.setenv("SECRET_KEY", "secret_test")
|
|
monkeypatch.setenv("ADMIN_USERNAME", "admin")
|
|
|
|
app = create_app()
|
|
|
|
async def _migrate() -> None:
|
|
async with app.state.engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
asyncio.run(_migrate())
|
|
|
|
token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600)
|
|
with TestClient(app) as client:
|
|
resp = client.post(
|
|
"/admin/api/imports/999/rollback",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
json={"mode": "disable"},
|
|
)
|
|
assert resp.status_code == 404
|
|
body = resp.json()
|
|
assert body["ok"] is False
|
|
assert body["code"] == "NOT_FOUND"
|
|
assert body["request_id"] == "req_test"
|
|
|
|
|
|
def test_admin_imports_rollback_invalid_id_returns_400(tmp_path: Path, monkeypatch) -> None:
|
|
db_path = tmp_path / "admin_imports_rollback_invalid_id.db"
|
|
db_url = "sqlite+aiosqlite:///" + db_path.as_posix()
|
|
|
|
monkeypatch.setenv("APP_ENV", "dev")
|
|
monkeypatch.setenv("DATABASE_URL", db_url)
|
|
monkeypatch.setenv("SECRET_KEY", "secret_test")
|
|
monkeypatch.setenv("ADMIN_USERNAME", "admin")
|
|
|
|
app = create_app()
|
|
|
|
async def _migrate() -> None:
|
|
async with app.state.engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
asyncio.run(_migrate())
|
|
|
|
token = create_jwt(secret_key="secret_test", subject="admin", ttl_s=3600)
|
|
with TestClient(app) as client:
|
|
resp = client.post(
|
|
"/admin/api/imports/0/rollback",
|
|
headers={"Authorization": f"Bearer {token}", "X-Request-Id": "req_test"},
|
|
json={"mode": "disable"},
|
|
)
|
|
assert resp.status_code == 400
|
|
body = resp.json()
|
|
assert body["ok"] is False
|
|
assert body["code"] == "BAD_REQUEST"
|
|
assert body["request_id"] == "req_test"
|