From bc76275d0420fabf511a7d2cb30a6bc6164a443a Mon Sep 17 00:00:00 2001 From: TooonyChen <26252739+TooonyChen@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:12:21 +1200 Subject: [PATCH] fix: stop treating List-Unsubscribe alone as promotional for transactional mail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gmail/Yahoo bulk-sender rules (2024) force large senders (Netflix via SES, etc.) to add List-Unsubscribe to verification emails too, so it can no longer be an unconditional bulk signal — real access-code mail was being filtered before the LLM and never reached code_mails. List-ID/List-Post, Precedence bulk/list, and ESP campaign X-headers remain unconditional. List-Unsubscribe alone is now exempted when the decoded subject looks transactional (code/verify/OTP/sign-in/reset/验证码/…); genuine ads that slip through still get codeExist=0 from the classifier. --- AGENTS.md | 2 +- src/services/mime.ts | 17 ++++++++++++++- test/mime.test.ts | 52 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 737ab0b..a75d418 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,7 +42,7 @@ pnpm run dev ## Architecture Constraints -- **Promotional filter runs before LLM.** `isPromotionalEmail()` checks `List-Unsubscribe`, `List-ID`, `Precedence: bulk/list`, and known ESP X-headers. If matched, raw email is still saved to `raw_mails` but LLM is skipped. +- **Promotional filter runs before LLM.** `isPromotionalEmail()` treats `List-ID`/`List-Post`, `Precedence: bulk/list`, and known ESP campaign X-headers as unconditional bulk signals. `List-Unsubscribe` alone is a weak signal: transactional-looking subjects (code/verify/OTP/sign-in/reset/验证码 …) are exempted, because Gmail/Yahoo bulk-sender rules force senders like Netflix/SES to add it to verification mail too. If matched, raw email is still saved to `raw_mails` but LLM is skipped. - **Two DB tables, strict separation.** Every email → `raw_mails`. Only emails with extracted codes/links → `code_mails`. Never skip `raw_mails` insert. - **`visibleMails()` in `src/services/mail.ts` is the only permission layer.** REST routes and MCP tools must query mails through it — never add a second filter path. Grants are enforced in SQL (GLOB address pattern + category allowlist). - **Sensitive categories are default-deny.** `password_reset`, `account_security` require `allow_sensitive = 1` on the grant; `legacy` (pre-v2 mails) is always admin-only. Raw email bodies (`raw`/`textBody`/`htmlBody`) are admin-only on every interface. diff --git a/src/services/mime.ts b/src/services/mime.ts index 65c170e..8ef3437 100644 --- a/src/services/mime.ts +++ b/src/services/mime.ts @@ -108,8 +108,14 @@ export function stripHtmlTags(text: string): string { .trim(); } +// 主题看起来像验证码/事务性邮件 (verification, OTP, sign-in, reset, 验证码…)。 +// 只用来豁免 List-Unsubscribe 这一条弱信号, 不影响下面强信号的判定。 +const TRANSACTIONAL_SUBJECT_RE = + /\b(codes?|otp|2fa|mfa|pin|passcodes?|passwords?|verif\w*|confirm\w*|activat\w*|authenticat\w*|one[- ]?time|single[- ]?use|magic link|sign[- ]?in|log[- ]?in|reset)\b|验证|驗證|校验|校驗|认证|認證|登录|登錄|登入|一次性|动态密码|動態密碼|認証|ワンタイム|パスワード|인증|확인/i; + export function isPromotionalEmail(headers: Headers, rawEmail: string): boolean { - if (headers.get("List-Unsubscribe") || headers.get("List-ID") || headers.get("List-Post")) { + // 强信号: 真正的邮件列表 / 群发 + if (headers.get("List-ID") || headers.get("List-Post")) { return true; } @@ -118,6 +124,7 @@ export function isPromotionalEmail(headers: Headers, rawEmail: string): boolean return true; } + // 强信号: 营销平台的 campaign 头 const rawHeaders = rawEmail.slice(0, rawEmail.search(/\r?\n\r?\n/) + 1 || 4000); if ( /^X-Campaign(-ID)?:/im.test(rawHeaders) || @@ -130,6 +137,14 @@ export function isPromotionalEmail(headers: Headers, rawEmail: string): boolean return true; } + // 弱信号: List-Unsubscribe 单独出现不足以定罪 —— Gmail/Yahoo 2024 起的 + // bulk sender 规则迫使大发件方 (Netflix/SES 等) 给验证码等事务邮件也带上它。 + // 主题像事务邮件时豁免, 放行给 LLM (真广告在 prompt 里仍会被判 codeExist=0)。 + if (headers.get("List-Unsubscribe")) { + const subject = decodeMimeHeader(headers.get("Subject")) ?? ""; + return !TRANSACTIONAL_SUBJECT_RE.test(subject); + } + return false; } diff --git a/test/mime.test.ts b/test/mime.test.ts index dfe7849..53d1a91 100644 --- a/test/mime.test.ts +++ b/test/mime.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { decodeMimeHeader, decodeQuotedPrintable, extractMailBodies } from "../src/services/mime"; +import { + decodeMimeHeader, + decodeQuotedPrintable, + extractMailBodies, + isPromotionalEmail, +} from "../src/services/mime"; describe("mime decoding", () => { it("decodes UTF-8 base64 body parts as Unicode text", () => { @@ -42,3 +47,48 @@ describe("mime decoding", () => { expect(decodeMimeHeader("=?UTF-8?B?6aqM6K+B56CB?=")).toBe("验证码"); }); }); + +describe("isPromotionalEmail", () => { + it("does not flag transactional mail that carries List-Unsubscribe (Gmail/Yahoo bulk sender rules)", () => { + const headers = new Headers({ + "List-Unsubscribe": "", + "List-Unsubscribe-Post": "List-Unsubscribe=One-Click", + Subject: "Your Netflix temporary access code", + }); + expect(isPromotionalEmail(headers, "Subject: x\r\n\r\nbody")).toBe(false); + }); + + it("exempts RFC 2047 encoded CJK verification subjects", () => { + const headers = new Headers({ + "List-Unsubscribe": "", + Subject: "=?UTF-8?B?6aqM6K+B56CB?=", // 验证码 + }); + expect(isPromotionalEmail(headers, "Subject: x\r\n\r\nbody")).toBe(false); + }); + + it("still flags List-Unsubscribe mail with a promotional subject", () => { + const headers = new Headers({ + "List-Unsubscribe": "", + Subject: "Summer sale: 50% off everything this weekend!", + }); + expect(isPromotionalEmail(headers, "Subject: x\r\n\r\nbody")).toBe(true); + }); + + it("keeps strong signals unconditional even when the subject looks transactional", () => { + const listMail = new Headers({ "List-ID": "", Subject: "your code" }); + expect(isPromotionalEmail(listMail, "Subject: x\r\n\r\nbody")).toBe(true); + + const bulkMail = new Headers({ Precedence: "bulk", Subject: "your verification code" }); + expect(isPromotionalEmail(bulkMail, "Subject: x\r\n\r\nbody")).toBe(true); + + const campaignMail = new Headers({ Subject: "your login code" }); + expect( + isPromotionalEmail(campaignMail, "X-Campaign-ID: 42\r\nSubject: x\r\n\r\nbody"), + ).toBe(true); + }); + + it("keeps plain mail without any bulk signal non-promotional", () => { + const headers = new Headers({ Subject: "Hello" }); + expect(isPromotionalEmail(headers, "Subject: Hello\r\n\r\nbody")).toBe(false); + }); +});