diff --git a/.gitignore b/.gitignore index 0a35f53..f6c10f1 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,6 @@ wrangler.toml # local db exports — may contain real email data *-data.sql *-backup.sql + +# Real-mail test fixtures (pulled from your own D1, contain private data) +test/fixtures/*.eml diff --git a/AGENTS.md b/AGENTS.md index a75d418..4a45b7f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,7 @@ - `src/mcp/server.ts` — remote MCP server at `/mcp` (`@hono/mcp` Streamable HTTP): `list_addresses`, `list_codes`, `get_latest_code`, `wait_for_code`. - `web/` — React 18 + Vite + Tailwind + shadcn/ui frontend. Built to `web/dist`, served via Cloudflare `ASSETS` binding. Pages: Login (doubles as first-admin setup), Inbox, API Keys, Users & Access (admin). - `migrations/` — D1 schema, applied with `wrangler d1 migrations apply`. Tables: `raw_mails`, `code_mails` (+ `category`), `users`, `api_keys`, `grants`. +- `test/` — `*.test.ts` unit tests (workers pool). `test/mail/*.mail.ts` real-mail regression suite (node env, `vitest.mail.config.mts`): `pipeline.mail.ts` asserts promo-filter verdicts and LLM-input content deterministically; `classify.mail.ts` runs the live LLM eval (`EVAL_LLM=1`). Fixtures come from your own D1 via `scripts/pull-fixtures.mjs` into gitignored `test/fixtures/*.eml`; expectations live in the committed `test/fixtures/manifest.json`. ## Commands @@ -27,7 +28,10 @@ pnpm run db:migrate:remote # apply D1 migrations to live D1 pnpm run build:web # vite build → web/dist pnpm run deploy # build:web + db:migrate:remote + wrangler deploy # Misc -pnpm run test # vitest with @cloudflare/vitest-pool-workers +pnpm run test # unit tests (vitest with @cloudflare/vitest-pool-workers) +pnpm run fixtures:pull # pull real-mail fixtures from remote D1 into test/fixtures/*.eml (gitignored) +pnpm run test:mail # deterministic pipeline regression on real-mail fixtures (no LLM, free) +pnpm run test:eval # LLM classification eval on fixtures (real provider calls; EVAL_RUNS=n, default 5) pnpm run cf-typegen # regenerate Worker type bindings ``` diff --git a/CLAUDE.md b/CLAUDE.md index 0df5e9d..7e225ea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,10 @@ pnpm run dev # Worker backend :8787 pnpm run dev:web # React frontend :5173 (proxy /api → :8787) pnpm run db:migrate:local # apply D1 migrations locally pnpm run deploy # build:web + db:migrate:remote + wrangler deploy -pnpm run test +pnpm run test # unit tests (workers pool) +pnpm run fixtures:pull # pull real mails from remote D1 → test/fixtures/*.eml (gitignored) +pnpm run test:mail # deterministic regression on real-mail fixtures (free, no LLM) +pnpm run test:eval # live LLM classification eval on fixtures (costs API calls) ``` ## Architecture Boundaries — Never Violate diff --git a/package.json b/package.json index 4157c93..fa6a108 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,9 @@ "dev": "wrangler dev", "start": "wrangler dev", "test": "vitest", + "test:mail": "vitest run --config vitest.mail.config.mts", + "test:eval": "EVAL_LLM=1 vitest run --config vitest.mail.config.mts", + "fixtures:pull": "node scripts/pull-fixtures.mjs", "cf-typegen": "wrangler types", "db:migrate:local": "wrangler d1 migrations apply inbox-d1 --local", "db:migrate:remote": "wrangler d1 migrations apply inbox-d1 --remote" diff --git a/scripts/pull-fixtures.mjs b/scripts/pull-fixtures.mjs new file mode 100644 index 0000000..4baae11 --- /dev/null +++ b/scripts/pull-fixtures.mjs @@ -0,0 +1,42 @@ +#!/usr/bin/env node +/* + * 从远程 D1 的 raw_mails 拉取 manifest 里列出的真实邮件, 存成 test/fixtures/.eml。 + * .eml 含真实地址/令牌, 已 gitignore — 只存在于本地, 供 test:mail / test:eval 使用。 + * + * 用法: pnpm run fixtures:pull (远程 D1, 需要 wrangler 登录) + * pnpm run fixtures:pull --local (本地 D1) + */ +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const fixturesDir = join(root, "test", "fixtures"); +const manifest = JSON.parse(readFileSync(join(fixturesDir, "manifest.json"), "utf8")); + +const where = process.argv.includes("--local") ? "--local" : "--remote"; +const ids = manifest.fixtures.map((f) => f.rawMailId); +const sql = `SELECT id, raw FROM raw_mails WHERE id IN (${ids.join(",")})`; + +console.log(`Fetching ${ids.length} fixtures from ${where} D1 ...`); +const out = execFileSync( + "npx", + ["wrangler", "d1", "execute", "inbox-d1", where, "--json", "--command", sql], + { cwd: root, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }, +); +const rows = JSON.parse(out)[0].results; +const byId = new Map(rows.map((r) => [r.id, r.raw])); + +let missing = 0; +for (const fixture of manifest.fixtures) { + const raw = byId.get(fixture.rawMailId); + if (!raw) { + console.warn(`✗ ${fixture.name}: raw_mails id ${fixture.rawMailId} not found`); + missing++; + continue; + } + writeFileSync(join(fixturesDir, `${fixture.name}.eml`), raw); + console.log(`✓ ${fixture.name}.eml (${raw.length} bytes)`); +} +process.exit(missing ? 1 : 0); diff --git a/src/services/classify.ts b/src/services/classify.ts index efb0fcc..2a4496a 100644 --- a/src/services/classify.ts +++ b/src/services/classify.ts @@ -15,7 +15,7 @@ export interface ExtractedMail { category?: MailCategory; } -function extractJsonFromText(rawText: string): Record | null { +export function extractJsonFromText(rawText: string): Record | null { let candidate = rawText.trim(); const jsonMatch = candidate.match(/```json\s*([\s\S]*?)\s*```/); if (jsonMatch && jsonMatch[1]) { @@ -33,7 +33,7 @@ function normaliseBaseUrl(baseUrl: string): string { return baseUrl.replace(/\/+$/, ""); } -async function callProvider(config: ProviderConfig, prompt: string): Promise { +export async function callProvider(config: ProviderConfig, prompt: string): Promise { const base = normaliseBaseUrl(config.baseUrl); let endpoint: string; let body: unknown; @@ -111,7 +111,7 @@ async function callProvider(config: ProviderConfig, prompt: string): Promise + toml.match(new RegExp(`^${key}\\s*=\\s*"([^"]+)"`, "m"))?.[1] ?? ""; + return { + baseUrl: grab("AI_BASE_URL"), + apiKey: grab("AI_API_KEY"), + format: grab("AI_API_FORMAT") as ProviderConfig["format"], + model: grab("AI_MODEL"), + }; +} + +describe.skipIf(!enabled)("LLM classification eval", () => { + const config = enabled ? loadProviderConfig() : null; + const fixtures = loadManifest().filter((f) => f.eval); + + for (const fixture of fixtures) { + const expected = fixture.eval!; + it.skipIf(!hasFixture(fixture))( + `${fixture.name} → codeExist=${expected.codeExist}${expected.categories ? ` category∈[${expected.categories}]` : ""}${expected.advisory ? " (advisory)" : ""}`, + { timeout: 120_000 }, + async () => { + const raw = readFixture(fixture); + // 被头部过滤器拦下的邮件在生产中不会进 LLM, eval 也跳过 + if (isPromotionalEmail(parseRawHeaders(raw), raw)) return; + + const prompt = buildPrompt(raw); + const failures: string[] = []; + + for (let run = 1; run <= RUNS; run++) { + const response = await callProvider(config!, prompt); + const parsed = response ? extractJsonFromText(response) : null; + if (!parsed) { + failures.push(`run ${run}: provider/parse failure`); + continue; + } + + const codeExist = parsed.codeExist === 1 ? 1 : 0; + const category = coerceCategory(parsed.category); + const code = typeof parsed.code === "string" ? parsed.code : ""; + const summary = `run ${run}: codeExist=${codeExist} category=${category} code=${code.slice(0, 60)}`; + + if (codeExist !== expected.codeExist) { + failures.push(`${summary} — expected codeExist=${expected.codeExist}`); + continue; + } + if (expected.codeExist === 1) { + if (expected.categories && !expected.categories.includes(category)) { + failures.push(`${summary} — category not in [${expected.categories}]`); + } + if (expected.codeContains && !code.toLowerCase().includes(expected.codeContains.toLowerCase())) { + failures.push(`${summary} — code missing "${expected.codeContains}"`); + } + } + } + + if (failures.length > 0) { + const report = `${fixture.name}: ${failures.length}/${RUNS} runs off-expectation\n ${failures.join("\n ")}`; + if (expected.advisory) { + console.warn(`[advisory] ${report}`); + } else { + expect.fail(report); + } + } + }, + ); + } +}); diff --git a/test/mail/helpers.ts b/test/mail/helpers.ts new file mode 100644 index 0000000..992d8d6 --- /dev/null +++ b/test/mail/helpers.ts @@ -0,0 +1,62 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { extractMailBodies, stripHtmlTags } from "../../src/services/mime"; + +export const FIXTURES_DIR = join(__dirname, "..", "fixtures"); + +export interface FixtureEval { + codeExist: 0 | 1; + categories?: string[]; + codeContains?: string; + advisory?: boolean; +} + +export interface Fixture { + name: string; + rawMailId: number; + note?: string; + promotional: boolean; + contentIncludes?: string[]; + eval?: FixtureEval; +} + +export function loadManifest(): Fixture[] { + const manifest = JSON.parse(readFileSync(join(FIXTURES_DIR, "manifest.json"), "utf8")); + return manifest.fixtures as Fixture[]; +} + +export function fixturePath(fixture: Fixture): string { + return join(FIXTURES_DIR, `${fixture.name}.eml`); +} + +export function hasFixture(fixture: Fixture): boolean { + return existsSync(fixturePath(fixture)); +} + +export function readFixture(fixture: Fixture): string { + return readFileSync(fixturePath(fixture), "utf8"); +} + +// 把原始邮件的顶层头部解析成 Headers, 模拟 Email Workers 传入的 message.headers。 +// 值里的非 Latin-1 字符替换掉 (Headers 只接受 ByteString; 真实 MTA 会用 RFC2047 编码)。 +export function parseRawHeaders(rawEmail: string): Headers { + const headerBlock = rawEmail.split(/\r?\n\r?\n/)[0] ?? ""; + const unfolded = headerBlock.replace(/\r?\n[ \t]+/g, " "); + const headers = new Headers(); + for (const line of unfolded.split(/\r?\n/)) { + const match = line.match(/^([\w-]+):\s*(.*)$/); + if (!match) continue; + try { + headers.append(match[1], match[2].replace(/[^\x00-\xff]/g, "?")); + } catch { + // 个别非法头直接跳过, 与 workerd 的容错行为一致 + } + } + return headers; +} + +// 与 classify.ts 的 buildPrompt 完全一致的正文选取逻辑 +export function emailContentForLlm(rawEmail: string): string { + const { textBody, htmlBody } = extractMailBodies(rawEmail); + return textBody ?? (htmlBody ? stripHtmlTags(htmlBody) : rawEmail); +} diff --git a/test/mail/pipeline.mail.ts b/test/mail/pipeline.mail.ts new file mode 100644 index 0000000..8b2fb3c --- /dev/null +++ b/test/mail/pipeline.mail.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { isPromotionalEmail } from "../../src/services/mime"; +import { emailContentForLlm, hasFixture, loadManifest, parseRawHeaders, readFixture } from "./helpers"; + +/* + * 确定性 pipeline 回归测试 (不调 LLM, 免费, 秒级): + * 对 D1 里挑出的真实邮件断言 1) 广告过滤器判定 2) LLM 输入正文包含关键片段。 + * fixture 缺失时跳过并提示 — 先跑 `pnpm run fixtures:pull`。 + */ + +const fixtures = loadManifest(); + +describe("mail pipeline fixtures", () => { + for (const fixture of fixtures) { + describe(fixture.name, () => { + it.skipIf(!hasFixture(fixture))( + `promotional filter → ${fixture.promotional}`, + () => { + const raw = readFixture(fixture); + expect(isPromotionalEmail(parseRawHeaders(raw), raw)).toBe(fixture.promotional); + }, + ); + + if (fixture.contentIncludes?.length) { + it.skipIf(!hasFixture(fixture))( + `LLM input contains: ${fixture.contentIncludes.join(", ")}`, + () => { + const content = emailContentForLlm(readFixture(fixture)).toLowerCase(); + for (const fragment of fixture.contentIncludes!) { + expect(content).toContain(fragment.toLowerCase()); + } + }, + ); + } + }); + } + + it("reminds you to pull fixtures when none exist", () => { + const present = fixtures.filter(hasFixture).length; + if (present === 0) { + console.warn("No fixture .eml files found — run `pnpm run fixtures:pull` first."); + } + expect(fixtures.length).toBeGreaterThan(0); + }); +}); diff --git a/vitest.mail.config.mts b/vitest.mail.config.mts new file mode 100644 index 0000000..3ebd673 --- /dev/null +++ b/vitest.mail.config.mts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +// 真实邮件 fixture 测试 (test/mail/*.mail.ts) 跑在 node 环境: +// 需要 fs 读 .eml, LLM eval 需要出网 — 都不适合 workers pool。 +// 主测试 (pnpm test, *.test.ts) 仍走 vitest.config.mts 的 workers pool。 +export default defineConfig({ + test: { + include: ["test/mail/**/*.mail.ts"], + environment: "node", + }, +});