test: add real-mail fixture evals
This commit is contained in:
TooonyChen committed
1 parent
b6a1ad7d9d
commit
83927b8b2b
11 files changed
+353
-6
No files matched your search
@@ -178,3 +178,6 @@ wrangler.toml
|
||||
# local db exports — may contain real email data
|
||||
*-data.sql
|
||||
*-backup.sql
|
||||
|
||||
# Real-mail test fixtures (pulled from your own D1, contain private data)
|
||||
test/fixtures/*.eml
|
||||
@@ -11,6 +11,7 @@
|
||||
- `src/mcp/server.ts` — remote MCP server at `/mcp` (`@hono/mcp` Streamable HTTP): `list_addresses`, `list_codes`, `get_latest_code`, `wait_for_code`.
|
||||
- `web/` — React 18 + Vite + Tailwind + shadcn/ui frontend. Built to `web/dist`, served via Cloudflare `ASSETS` binding. Pages: Login (doubles as first-admin setup), Inbox, API Keys, Users & Access (admin).
|
||||
- `migrations/` — D1 schema, applied with `wrangler d1 migrations apply`. Tables: `raw_mails`, `code_mails` (+ `category`), `users`, `api_keys`, `grants`.
|
||||
- `test/` — `*.test.ts` unit tests (workers pool). `test/mail/*.mail.ts` real-mail regression suite (node env, `vitest.mail.config.mts`): `pipeline.mail.ts` asserts promo-filter verdicts and LLM-input content deterministically; `classify.mail.ts` runs the live LLM eval (`EVAL_LLM=1`). Fixtures come from your own D1 via `scripts/pull-fixtures.mjs` into gitignored `test/fixtures/*.eml`; expectations live in the committed `test/fixtures/manifest.json`.
|
||||
|
||||
## Commands
|
||||
|
||||
@@ -27,7 +28,10 @@ pnpm run db:migrate:remote # apply D1 migrations to live D1
|
||||
pnpm run build:web # vite build → web/dist
|
||||
pnpm run deploy # build:web + db:migrate:remote + wrangler deploy
|
||||
# Misc
|
||||
pnpm run test # vitest with @cloudflare/vitest-pool-workers
|
||||
pnpm run test # unit tests (vitest with @cloudflare/vitest-pool-workers)
|
||||
pnpm run fixtures:pull # pull real-mail fixtures from remote D1 into test/fixtures/*.eml (gitignored)
|
||||
pnpm run test:mail # deterministic pipeline regression on real-mail fixtures (no LLM, free)
|
||||
pnpm run test:eval # LLM classification eval on fixtures (real provider calls; EVAL_RUNS=n, default 5)
|
||||
pnpm run cf-typegen # regenerate Worker type bindings
|
||||
```
|
||||
|
||||
|
||||
@@ -13,7 +13,10 @@ pnpm run dev # Worker backend :8787
|
||||
pnpm run dev:web # React frontend :5173 (proxy /api → :8787)
|
||||
pnpm run db:migrate:local # apply D1 migrations locally
|
||||
pnpm run deploy # build:web + db:migrate:remote + wrangler deploy
|
||||
pnpm run test
|
||||
pnpm run test # unit tests (workers pool)
|
||||
pnpm run fixtures:pull # pull real mails from remote D1 → test/fixtures/*.eml (gitignored)
|
||||
pnpm run test:mail # deterministic regression on real-mail fixtures (free, no LLM)
|
||||
pnpm run test:eval # live LLM classification eval on fixtures (costs API calls)
|
||||
```
|
||||
|
||||
## Architecture Boundaries — Never Violate
|
||||
|
||||
@@ -9,6 +9,9 @@
|
||||
"dev": "wrangler dev",
|
||||
"start": "wrangler dev",
|
||||
"test": "vitest",
|
||||
"test:mail": "vitest run --config vitest.mail.config.mts",
|
||||
"test:eval": "EVAL_LLM=1 vitest run --config vitest.mail.config.mts",
|
||||
"fixtures:pull": "node scripts/pull-fixtures.mjs",
|
||||
"cf-typegen": "wrangler types",
|
||||
"db:migrate:local": "wrangler d1 migrations apply inbox-d1 --local",
|
||||
"db:migrate:remote": "wrangler d1 migrations apply inbox-d1 --remote"
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* 从远程 D1 的 raw_mails 拉取 manifest 里列出的真实邮件, 存成 test/fixtures/<name>.eml。
|
||||
* .eml 含真实地址/令牌, 已 gitignore — 只存在于本地, 供 test:mail / test:eval 使用。
|
||||
*
|
||||
* 用法: pnpm run fixtures:pull (远程 D1, 需要 wrangler 登录)
|
||||
* pnpm run fixtures:pull --local (本地 D1)
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const fixturesDir = join(root, "test", "fixtures");
|
||||
const manifest = JSON.parse(readFileSync(join(fixturesDir, "manifest.json"), "utf8"));
|
||||
|
||||
const where = process.argv.includes("--local") ? "--local" : "--remote";
|
||||
const ids = manifest.fixtures.map((f) => f.rawMailId);
|
||||
const sql = `SELECT id, raw FROM raw_mails WHERE id IN (${ids.join(",")})`;
|
||||
|
||||
console.log(`Fetching ${ids.length} fixtures from ${where} D1 ...`);
|
||||
const out = execFileSync(
|
||||
"npx",
|
||||
["wrangler", "d1", "execute", "inbox-d1", where, "--json", "--command", sql],
|
||||
{ cwd: root, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 },
|
||||
);
|
||||
const rows = JSON.parse(out)[0].results;
|
||||
const byId = new Map(rows.map((r) => [r.id, r.raw]));
|
||||
|
||||
let missing = 0;
|
||||
for (const fixture of manifest.fixtures) {
|
||||
const raw = byId.get(fixture.rawMailId);
|
||||
if (!raw) {
|
||||
console.warn(`✗ ${fixture.name}: raw_mails id ${fixture.rawMailId} not found`);
|
||||
missing++;
|
||||
continue;
|
||||
}
|
||||
writeFileSync(join(fixturesDir, `${fixture.name}.eml`), raw);
|
||||
console.log(`✓ ${fixture.name}.eml (${raw.length} bytes)`);
|
||||
}
|
||||
process.exit(missing ? 1 : 0);
|
||||
@@ -15,7 +15,7 @@ export interface ExtractedMail {
|
||||
category?: MailCategory;
|
||||
}
|
||||
|
||||
function extractJsonFromText(rawText: string): Record<string, unknown> | null {
|
||||
export function extractJsonFromText(rawText: string): Record<string, unknown> | null {
|
||||
let candidate = rawText.trim();
|
||||
const jsonMatch = candidate.match(/```json\s*([\s\S]*?)\s*```/);
|
||||
if (jsonMatch && jsonMatch[1]) {
|
||||
@@ -33,7 +33,7 @@ function normaliseBaseUrl(baseUrl: string): string {
|
||||
return baseUrl.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
async function callProvider(config: ProviderConfig, prompt: string): Promise<string | null> {
|
||||
export async function callProvider(config: ProviderConfig, prompt: string): Promise<string | null> {
|
||||
const base = normaliseBaseUrl(config.baseUrl);
|
||||
let endpoint: string;
|
||||
let body: unknown;
|
||||
@@ -111,7 +111,7 @@ async function callProvider(config: ProviderConfig, prompt: string): Promise<str
|
||||
return typeof text === "string" ? text : null;
|
||||
}
|
||||
|
||||
function buildPrompt(rawEmail: string): string {
|
||||
export function buildPrompt(rawEmail: string): string {
|
||||
const { textBody, htmlBody } = extractMailBodies(rawEmail);
|
||||
const emailContent = textBody ?? (htmlBody ? stripHtmlTags(htmlBody) : rawEmail);
|
||||
|
||||
@@ -151,7 +151,7 @@ If there is no code, clickable link, or this is an advertisement email, return:
|
||||
`;
|
||||
}
|
||||
|
||||
function coerceCategory(value: unknown): MailCategory {
|
||||
export function coerceCategory(value: unknown): MailCategory {
|
||||
if (typeof value === "string" && (MAIL_CATEGORIES as readonly string[]).includes(value)) {
|
||||
return value as MailCategory;
|
||||
}
|
||||
|
||||
Vendored
+78
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"$comment": "真实邮件回归测试清单。.eml 文件含真实地址/令牌, 已 gitignore — 用 `pnpm run fixtures:pull` 从你自己的 D1 拉取。promotional: isPromotionalEmail 的期望结果; contentIncludes: LLM 输入 (emailContent) 必须包含的片段; eval: LLM 分类期望, advisory=true 只警告不判失败。",
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "cmhk-registration",
|
||||
"rawMailId": 2650,
|
||||
"note": "multipart + 中文, 明文数字验证码",
|
||||
"promotional": false,
|
||||
"contentIncludes": ["668445"],
|
||||
"eval": { "codeExist": 1, "categories": ["registration"], "codeContains": "668445" }
|
||||
},
|
||||
{
|
||||
"name": "netflix-forwarded-html",
|
||||
"rawMailId": 2651,
|
||||
"note": "Yandex 手动转发: 单一 text/html 裸片段, 只有按钮链接 (stripHtmlTags 保留 href 的回归用例)",
|
||||
"promotional": false,
|
||||
"contentIncludes": ["netflix.com/account/travel/verify"],
|
||||
"eval": { "codeExist": 1, "categories": ["login_code"], "codeContains": "travel/verify" }
|
||||
},
|
||||
{
|
||||
"name": "bandwagon-device-verification",
|
||||
"rawMailId": 2502,
|
||||
"note": "text/plain 明文数字码",
|
||||
"promotional": false,
|
||||
"contentIncludes": ["596352"],
|
||||
"eval": { "codeExist": 1, "categories": ["login_code", "account_security"], "codeContains": "596352" }
|
||||
},
|
||||
{
|
||||
"name": "18birdies-verify",
|
||||
"rawMailId": 2453,
|
||||
"note": "html-only 注册验证链接",
|
||||
"promotional": false,
|
||||
"contentIncludes": ["18birdies.com"],
|
||||
"eval": { "codeExist": 1, "categories": ["registration"], "codeContains": "18birdies.com" }
|
||||
},
|
||||
{
|
||||
"name": "subcard-password-reset",
|
||||
"rawMailId": 2577,
|
||||
"note": "改密码确认, RFC2047 编码主题",
|
||||
"promotional": false,
|
||||
"contentIncludes": ["subcard"],
|
||||
"eval": { "codeExist": 1, "categories": ["password_reset", "account_security"] }
|
||||
},
|
||||
{
|
||||
"name": "disney-new-login",
|
||||
"rawMailId": 2552,
|
||||
"note": "新设备登录提醒 (account_security)",
|
||||
"promotional": false,
|
||||
"contentIncludes": ["disneyplus.com"],
|
||||
"eval": { "codeExist": 1, "categories": ["account_security", "login_code"], "advisory": true }
|
||||
},
|
||||
{
|
||||
"name": "ubereats-promo",
|
||||
"rawMailId": 2639,
|
||||
"note": "促销: List-Unsubscribe + List-ID, 必须被头部过滤器拦下",
|
||||
"promotional": true
|
||||
},
|
||||
{
|
||||
"name": "trip-promo",
|
||||
"rawMailId": 2647,
|
||||
"note": "促销 (无 list 头, 会进 LLM): 期望 LLM 判无码",
|
||||
"promotional": false,
|
||||
"eval": { "codeExist": 0, "advisory": true }
|
||||
},
|
||||
{
|
||||
"name": "linkedin-invitations",
|
||||
"rawMailId": 2648,
|
||||
"note": "社交通知: List-Unsubscribe + 非事务主题, 会被头部过滤器拦下",
|
||||
"promotional": true
|
||||
},
|
||||
{
|
||||
"name": "starbucks-promo",
|
||||
"rawMailId": 2635,
|
||||
"note": "促销, RFC2047 主题; List-Unsubscribe + 非事务主题, 会被头部过滤器拦下",
|
||||
"promotional": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ProviderConfig } from "../../src/types";
|
||||
import { buildPrompt, callProvider, coerceCategory, extractJsonFromText } from "../../src/services/classify";
|
||||
import { isPromotionalEmail } from "../../src/services/mime";
|
||||
import { hasFixture, loadManifest, parseRawHeaders, readFixture } from "./helpers";
|
||||
|
||||
/*
|
||||
* LLM 分类 eval (真实调用 AI provider, 花钱, 结果非严格确定):
|
||||
* EVAL_LLM=1 pnpm run test:eval — 每个 fixture 跑 EVAL_RUNS 次 (默认 5)
|
||||
* EVAL_RUNS=3 pnpm run test:eval — 降低次数
|
||||
* provider 配置优先取环境变量 AI_*, 否则从 wrangler.toml [vars] 读。
|
||||
* eval.advisory=true 的用例只告警不判失败 (LLM 判断类, 允许漂移观察)。
|
||||
*/
|
||||
|
||||
const enabled = process.env.EVAL_LLM === "1";
|
||||
const RUNS = Number(process.env.EVAL_RUNS ?? 5);
|
||||
|
||||
function loadProviderConfig(): ProviderConfig {
|
||||
const fromEnv = {
|
||||
baseUrl: process.env.AI_BASE_URL,
|
||||
apiKey: process.env.AI_API_KEY,
|
||||
format: process.env.AI_API_FORMAT,
|
||||
model: process.env.AI_MODEL,
|
||||
};
|
||||
if (fromEnv.baseUrl && fromEnv.apiKey && fromEnv.format && fromEnv.model) {
|
||||
return fromEnv as ProviderConfig;
|
||||
}
|
||||
const toml = readFileSync(join(__dirname, "..", "..", "wrangler.toml"), "utf8");
|
||||
const grab = (key: string): string =>
|
||||
toml.match(new RegExp(`^${key}\\s*=\\s*"([^"]+)"`, "m"))?.[1] ?? "";
|
||||
return {
|
||||
baseUrl: grab("AI_BASE_URL"),
|
||||
apiKey: grab("AI_API_KEY"),
|
||||
format: grab("AI_API_FORMAT") as ProviderConfig["format"],
|
||||
model: grab("AI_MODEL"),
|
||||
};
|
||||
}
|
||||
|
||||
describe.skipIf(!enabled)("LLM classification eval", () => {
|
||||
const config = enabled ? loadProviderConfig() : null;
|
||||
const fixtures = loadManifest().filter((f) => f.eval);
|
||||
|
||||
for (const fixture of fixtures) {
|
||||
const expected = fixture.eval!;
|
||||
it.skipIf(!hasFixture(fixture))(
|
||||
`${fixture.name} → codeExist=${expected.codeExist}${expected.categories ? ` category∈[${expected.categories}]` : ""}${expected.advisory ? " (advisory)" : ""}`,
|
||||
{ timeout: 120_000 },
|
||||
async () => {
|
||||
const raw = readFixture(fixture);
|
||||
// 被头部过滤器拦下的邮件在生产中不会进 LLM, eval 也跳过
|
||||
if (isPromotionalEmail(parseRawHeaders(raw), raw)) return;
|
||||
|
||||
const prompt = buildPrompt(raw);
|
||||
const failures: string[] = [];
|
||||
|
||||
for (let run = 1; run <= RUNS; run++) {
|
||||
const response = await callProvider(config!, prompt);
|
||||
const parsed = response ? extractJsonFromText(response) : null;
|
||||
if (!parsed) {
|
||||
failures.push(`run ${run}: provider/parse failure`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const codeExist = parsed.codeExist === 1 ? 1 : 0;
|
||||
const category = coerceCategory(parsed.category);
|
||||
const code = typeof parsed.code === "string" ? parsed.code : "";
|
||||
const summary = `run ${run}: codeExist=${codeExist} category=${category} code=${code.slice(0, 60)}`;
|
||||
|
||||
if (codeExist !== expected.codeExist) {
|
||||
failures.push(`${summary} — expected codeExist=${expected.codeExist}`);
|
||||
continue;
|
||||
}
|
||||
if (expected.codeExist === 1) {
|
||||
if (expected.categories && !expected.categories.includes(category)) {
|
||||
failures.push(`${summary} — category not in [${expected.categories}]`);
|
||||
}
|
||||
if (expected.codeContains && !code.toLowerCase().includes(expected.codeContains.toLowerCase())) {
|
||||
failures.push(`${summary} — code missing "${expected.codeContains}"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
const report = `${fixture.name}: ${failures.length}/${RUNS} runs off-expectation\n ${failures.join("\n ")}`;
|
||||
if (expected.advisory) {
|
||||
console.warn(`[advisory] ${report}`);
|
||||
} else {
|
||||
expect.fail(report);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { extractMailBodies, stripHtmlTags } from "../../src/services/mime";
|
||||
|
||||
export const FIXTURES_DIR = join(__dirname, "..", "fixtures");
|
||||
|
||||
export interface FixtureEval {
|
||||
codeExist: 0 | 1;
|
||||
categories?: string[];
|
||||
codeContains?: string;
|
||||
advisory?: boolean;
|
||||
}
|
||||
|
||||
export interface Fixture {
|
||||
name: string;
|
||||
rawMailId: number;
|
||||
note?: string;
|
||||
promotional: boolean;
|
||||
contentIncludes?: string[];
|
||||
eval?: FixtureEval;
|
||||
}
|
||||
|
||||
export function loadManifest(): Fixture[] {
|
||||
const manifest = JSON.parse(readFileSync(join(FIXTURES_DIR, "manifest.json"), "utf8"));
|
||||
return manifest.fixtures as Fixture[];
|
||||
}
|
||||
|
||||
export function fixturePath(fixture: Fixture): string {
|
||||
return join(FIXTURES_DIR, `${fixture.name}.eml`);
|
||||
}
|
||||
|
||||
export function hasFixture(fixture: Fixture): boolean {
|
||||
return existsSync(fixturePath(fixture));
|
||||
}
|
||||
|
||||
export function readFixture(fixture: Fixture): string {
|
||||
return readFileSync(fixturePath(fixture), "utf8");
|
||||
}
|
||||
|
||||
// 把原始邮件的顶层头部解析成 Headers, 模拟 Email Workers 传入的 message.headers。
|
||||
// 值里的非 Latin-1 字符替换掉 (Headers 只接受 ByteString; 真实 MTA 会用 RFC2047 编码)。
|
||||
export function parseRawHeaders(rawEmail: string): Headers {
|
||||
const headerBlock = rawEmail.split(/\r?\n\r?\n/)[0] ?? "";
|
||||
const unfolded = headerBlock.replace(/\r?\n[ \t]+/g, " ");
|
||||
const headers = new Headers();
|
||||
for (const line of unfolded.split(/\r?\n/)) {
|
||||
const match = line.match(/^([\w-]+):\s*(.*)$/);
|
||||
if (!match) continue;
|
||||
try {
|
||||
headers.append(match[1], match[2].replace(/[^\x00-\xff]/g, "?"));
|
||||
} catch {
|
||||
// 个别非法头直接跳过, 与 workerd 的容错行为一致
|
||||
}
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
// 与 classify.ts 的 buildPrompt 完全一致的正文选取逻辑
|
||||
export function emailContentForLlm(rawEmail: string): string {
|
||||
const { textBody, htmlBody } = extractMailBodies(rawEmail);
|
||||
return textBody ?? (htmlBody ? stripHtmlTags(htmlBody) : rawEmail);
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isPromotionalEmail } from "../../src/services/mime";
|
||||
import { emailContentForLlm, hasFixture, loadManifest, parseRawHeaders, readFixture } from "./helpers";
|
||||
|
||||
/*
|
||||
* 确定性 pipeline 回归测试 (不调 LLM, 免费, 秒级):
|
||||
* 对 D1 里挑出的真实邮件断言 1) 广告过滤器判定 2) LLM 输入正文包含关键片段。
|
||||
* fixture 缺失时跳过并提示 — 先跑 `pnpm run fixtures:pull`。
|
||||
*/
|
||||
|
||||
const fixtures = loadManifest();
|
||||
|
||||
describe("mail pipeline fixtures", () => {
|
||||
for (const fixture of fixtures) {
|
||||
describe(fixture.name, () => {
|
||||
it.skipIf(!hasFixture(fixture))(
|
||||
`promotional filter → ${fixture.promotional}`,
|
||||
() => {
|
||||
const raw = readFixture(fixture);
|
||||
expect(isPromotionalEmail(parseRawHeaders(raw), raw)).toBe(fixture.promotional);
|
||||
},
|
||||
);
|
||||
|
||||
if (fixture.contentIncludes?.length) {
|
||||
it.skipIf(!hasFixture(fixture))(
|
||||
`LLM input contains: ${fixture.contentIncludes.join(", ")}`,
|
||||
() => {
|
||||
const content = emailContentForLlm(readFixture(fixture)).toLowerCase();
|
||||
for (const fragment of fixture.contentIncludes!) {
|
||||
expect(content).toContain(fragment.toLowerCase());
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
it("reminds you to pull fixtures when none exist", () => {
|
||||
const present = fixtures.filter(hasFixture).length;
|
||||
if (present === 0) {
|
||||
console.warn("No fixture .eml files found — run `pnpm run fixtures:pull` first.");
|
||||
}
|
||||
expect(fixtures.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
// 真实邮件 fixture 测试 (test/mail/*.mail.ts) 跑在 node 环境:
|
||||
// 需要 fs 读 .eml, LLM eval 需要出网 — 都不适合 workers pool。
|
||||
// 主测试 (pnpm test, *.test.ts) 仍走 vitest.config.mts 的 workers pool。
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ["test/mail/**/*.mail.ts"],
|
||||
environment: "node",
|
||||
},
|
||||
});
|
||||
Reference in new issue
Block a user