fix(backups): back up the tables added since the format shipped

BACKUP_TABLES is a hardcoded list, and the migrations that added
email_templates, calendar_events (0020) and auto_reply_deliveries (0022)
never updated it. Those three tables are absent from every backup, and
because restoreDatabaseRecords only touches the listed tables, restoring
into an empty database silently loses calendar events and email templates
with nothing reporting a problem.

Add the three tables, and add assertBackupTablesCoverDatabase, which
compares sqlite_master against BACKUP_TABLES and aborts the backup naming
any unlisted table. A backup that fails loudly is recoverable; one that
silently omits a table is only discovered during a restore, when the data
is already gone. The three tables are appended, so the reverse-order delete
still removes children before parents and the forward-order insert still
writes them after users and mailboxes, which they reference.

Extending the list would by itself have broken restores of existing
backups: isDatabaseBackupDocument required every BACKUP_TABLES entry to be
present, and documents written before this change have only the original
nineteen. Split out REQUIRED_BACKUP_TABLES so tables added later are
optional on read, and fill absent ones with an empty array before
validation. Documents missing an original table are still rejected.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/lib/backups/export.ts

@@ -1,14 +1,48 @@
 import type { DatabaseBackupDocument, DatabaseBackupTable, DatabaseRecord } from "./types";
 import { mergeLegacyMessageBodies } from "./utils";
 
-const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings"];
+const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings", "email_templates", "calendar_events", "auto_reply_deliveries"];
+/**
+ * Tables every backup document must contain. Tables added to BACKUP_TABLES
+ * after the format shipped are absent from older documents, so they stay
+ * optional here and are filled in as empty on restore.
+ */
+const REQUIRED_BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings"];
 const INSERT_BATCH_SIZE = 50;
 
 export function getD1ExportConfigurationStatus(_env?: CloudflareEnv) {
 	return { configured: true, missing: [] };
 }
 
+/**
+ * Tables D1 manages itself, which are intentionally absent from BACKUP_TABLES.
+ */
+const INTERNAL_TABLE_PATTERNS = ["sqlite_%", "_cf%"];
+const INTERNAL_TABLES = ["d1_migrations"];
+
+/**
+ * Fails the backup when the database contains a table BACKUP_TABLES does not
+ * list. Without this, a migration that adds a table silently produces backups
+ * that omit it, and the omission only surfaces during a restore.
+ */
+export async function assertBackupTablesCoverDatabase(db: D1Database): Promise<void> {
+	const conditions = [
+		...INTERNAL_TABLE_PATTERNS.map((pattern) => `name NOT LIKE '${pattern}'`),
+		...INTERNAL_TABLES.map((name) => `name <> '${name}'`),
+	].join(" AND ");
+	const result = await db
+		.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND ${conditions}`)
+		.all<{ name: string }>();
+	const covered = new Set<string>(BACKUP_TABLES);
+	const unlisted = result.results.map((row) => row.name).filter((name) => !covered.has(name));
+	if (unlisted.length === 0) return;
+	throw new Error(
+		`Backup aborted: ${unlisted.join(", ")} not listed in BACKUP_TABLES, so this backup would omit them. Add them to BACKUP_TABLES in src/lib/backups/export.ts.`,
+	);
+}
+
 export async function exportDatabaseRecords(db: D1Database): Promise<Uint8Array> {
+	await assertBackupTablesCoverDatabase(db);
 	const tables = {} as Record<DatabaseBackupTable, DatabaseRecord[]>;
 	for (const table of BACKUP_TABLES) {
 		const result = await db.prepare(`SELECT * FROM ${table}`).all<DatabaseRecord>();
@@ -21,6 +55,7 @@ export async function exportDatabaseRecords(db: D1Database): Promise<Uint8Array>
 export async function restoreDatabaseRecords(db: D1Database, content: ArrayBuffer): Promise<void> {
 	const document = parseDatabaseBackup(content);
 	mergeLegacyMessageBodies(document);
+	fillMissingBackupTables(document);
 	validateDatabaseBackup(document);
 	for (const table of [...BACKUP_TABLES].reverse()) await db.prepare(`DELETE FROM ${table}`).run();
 	for (const table of BACKUP_TABLES) {
@@ -42,7 +77,12 @@ function parseDatabaseBackup(content: ArrayBuffer): DatabaseBackupDocument {
 function isDatabaseBackupDocument(value: unknown): value is DatabaseBackupDocument {
 	if (!value || typeof value !== "object") return false;
 	const document = value as Partial<DatabaseBackupDocument>;
-	return document.format === "mailflare-database-backup" && document.version === 1 && !!document.tables && BACKUP_TABLES.every((table) => Array.isArray(document.tables?.[table]));
+	if (document.format !== "mailflare-database-backup" || document.version !== 1 || !document.tables) return false;
+	if (!REQUIRED_BACKUP_TABLES.every((table) => Array.isArray(document.tables?.[table]))) return false;
+	return BACKUP_TABLES.every((table) => {
+		const rows = document.tables?.[table];
+		return rows === undefined || Array.isArray(rows);
+	});
 }
 
 function createInsertStatement(db: D1Database, table: DatabaseBackupTable, row: DatabaseRecord) {
@@ -53,6 +93,13 @@ function createInsertStatement(db: D1Database, table: DatabaseBackupTable, row:
 	return db.prepare(`INSERT INTO ${table} (${identifiers}) VALUES (${placeholders})`).bind(...columns.map((column) => row[column]));
 }
 
+/** Backups written before a table joined BACKUP_TABLES simply omit it. */
+function fillMissingBackupTables(document: DatabaseBackupDocument): void {
+	for (const table of BACKUP_TABLES) {
+		if (!document.tables[table]) document.tables[table] = [];
+	}
+}
+
 function validateDatabaseBackup(document: DatabaseBackupDocument): void {
 	for (const table of BACKUP_TABLES) {
 		for (const row of document.tables[table]) {

src/lib/backups/types.d.ts

@@ -12,6 +12,6 @@ export type BackupWorkflowBinding = {
 	}): Promise<unknown>;
 };
 
-export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings";
+export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings" | "email_templates" | "calendar_events" | "auto_reply_deliveries";
 export type DatabaseRecord = Record<string, string | number | null>;
 export type DatabaseBackupDocument = { format: "mailflare-database-backup"; version: 1; createdAt: string; tables: Record<DatabaseBackupTable, DatabaseRecord[]>; };