diff --git a/src/server/db/migrate.test.ts b/src/server/db/migrate.test.ts index e509598..9369543 100644 --- a/src/server/db/migrate.test.ts +++ b/src/server/db/migrate.test.ts @@ -1,5 +1,6 @@ import Database from 'better-sqlite3'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createHash } from 'node:crypto'; import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; @@ -29,6 +30,26 @@ function applyMigrationSql(sqlite: Database.Database, sqlText: string) { } } +function recordAppliedMigrations( + sqlite: Database.Database, + journalEntries: MigrationJournalEntry[], +) { + sqlite.exec(` + CREATE TABLE IF NOT EXISTS "__drizzle_migrations" ( + id SERIAL PRIMARY KEY, + hash text NOT NULL, + created_at numeric + ) + `); + + const insert = sqlite.prepare('INSERT INTO "__drizzle_migrations" ("hash", "created_at") VALUES (?, ?)'); + for (const entry of journalEntries) { + const sqlText = readFileSync(join(migrationsDir, `${entry.tag}.sql`), 'utf8'); + const hash = createHash('sha256').update(sqlText).digest('hex'); + insert.run(hash, entry.when); + } +} + describe('sqlite migrate bootstrap', () => { afterEach(() => { delete process.env.DATA_DIR; @@ -186,4 +207,44 @@ describe('sqlite migrate bootstrap', () => { sqlite.close(); }); + + it('replays missing migrations before marking a duplicate-column migration as applied', async () => { + const dataDir = mkdtempSync(join(tmpdir(), 'metapi-migrate-partial-journal-')); + const dbPath = join(dataDir, 'hub.db'); + const sqlite = new Database(dbPath); + const journalEntries = readMigrationJournalEntries(); + const appliedEntries = journalEntries.filter((entry) => entry.tag !== '0006_site_disabled_models' && entry.tag !== '0007_account_token_group'); + + for (const entry of appliedEntries) { + const sqlText = readFileSync(join(migrationsDir, `${entry.tag}.sql`), 'utf8'); + applyMigrationSql(sqlite, sqlText); + } + recordAppliedMigrations(sqlite, appliedEntries); + + // Simulate legacy compatibility code adding token_group before the formal 0007 migration ran. + sqlite.exec('ALTER TABLE account_tokens ADD COLUMN token_group text;'); + sqlite.close(); + + process.env.DATA_DIR = dataDir; + vi.resetModules(); + + await expect(import('./migrate.js')).resolves.toMatchObject({ + runSqliteMigrations: expect.any(Function), + }); + + const verified = new Database(dbPath, { readonly: true }); + const appliedRows = verified + .prepare('SELECT created_at FROM __drizzle_migrations ORDER BY created_at ASC') + .all() as Array<{ created_at: number }>; + const disabledModelsTable = verified + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'site_disabled_models'") + .get() as { name?: string } | undefined; + + expect(disabledModelsTable?.name).toBe('site_disabled_models'); + expect(appliedRows.map((row) => Number(row.created_at))).toEqual( + journalEntries.map((entry) => entry.when), + ); + + verified.close(); + }); }); diff --git a/src/server/db/migrate.ts b/src/server/db/migrate.ts index 0359658..d177310 100644 --- a/src/server/db/migrate.ts +++ b/src/server/db/migrate.ts @@ -30,6 +30,10 @@ type RecoveryMigrationRecord = MigrationRecord & { tag: string; }; +type RecoveryMigration = RecoveryMigrationRecord & { + statements: string[]; +}; + const VERIFIED_BOOTSTRAP_TAG = '0007_account_token_group'; const VERIFIED_SCHEMA_MARKERS: SchemaMarker[] = [ { table: 'sites' }, @@ -170,6 +174,21 @@ function findMatchingSingleStatementMigration( return null; } +function readRecoveryMigrations(migrationsFolder: string): RecoveryMigration[] { + const journalPath = resolve(migrationsFolder, 'meta', '_journal.json'); + const journal = JSON.parse(readFileSync(journalPath, 'utf8')) as MigrationJournalFile; + + return (journal.entries ?? []).map((entry) => { + const migrationSql = readFileSync(resolve(migrationsFolder, `${entry.tag}.sql`), 'utf8'); + return { + tag: entry.tag, + createdAt: Number(entry.when), + hash: createHash('sha256').update(migrationSql).digest('hex'), + statements: splitMigrationStatements(migrationSql), + }; + }); +} + function ensureDrizzleMigrationsTable(sqlite: Database.Database): void { sqlite.exec(` CREATE TABLE IF NOT EXISTS "__drizzle_migrations" ( @@ -232,6 +251,61 @@ function isDuplicateColumnError(error: unknown): boolean { || lowered.includes('duplicate column name'); } +function isRecoverableSchemaConflictError(error: unknown): boolean { + const lowered = normalizeSchemaErrorMessage(error).toLowerCase(); + return lowered.includes('duplicate column') + || lowered.includes('duplicate column name') + || lowered.includes('already exists'); +} + +function getLatestRecordedMigrationCreatedAt(sqlite: Database.Database): number | null { + if (!tableExists(sqlite, '__drizzle_migrations')) return null; + const row = sqlite + .prepare('SELECT created_at FROM "__drizzle_migrations" ORDER BY created_at DESC LIMIT 1') + .get() as { created_at?: number } | undefined; + if (!row || row.created_at === undefined || row.created_at === null) { + return null; + } + return Number(row.created_at); +} + +function replayMigrationStatements(sqlite: Database.Database, statements: string[]): void { + for (const statement of statements) { + try { + sqlite.exec(statement); + } catch (error) { + if (!isRecoverableSchemaConflictError(error)) { + throw error; + } + } + } +} + +function recoverMigrationSequence( + sqlite: Database.Database, + migrationsFolder: string, + failedMigrationTag: string, +): boolean { + const migrations = readRecoveryMigrations(migrationsFolder); + const failedMigrationIndex = migrations.findIndex((migration) => migration.tag === failedMigrationTag); + if (failedMigrationIndex < 0) { + return false; + } + + let latestRecordedCreatedAt = getLatestRecordedMigrationCreatedAt(sqlite); + for (const migration of migrations.slice(0, failedMigrationIndex + 1)) { + if (latestRecordedCreatedAt !== null && latestRecordedCreatedAt >= migration.createdAt) { + continue; + } + + replayMigrationStatements(sqlite, migration.statements); + markMigrationRecordIfMissing(sqlite, migration); + latestRecordedCreatedAt = migration.createdAt; + } + + return true; +} + function tryRecoverDuplicateColumnMigrationError( sqlite: Database.Database, migrationsFolder: string, @@ -251,11 +325,11 @@ function tryRecoverDuplicateColumnMigrationError( return false; } - const inserted = markMigrationRecordIfMissing(sqlite, matchedMigration); - if (inserted) { - console.warn(`[db] Recovered duplicate-column migration by marking ${matchedMigration.tag} as applied.`); + const recovered = recoverMigrationSequence(sqlite, migrationsFolder, matchedMigration.tag); + if (recovered) { + console.warn(`[db] Recovered duplicate-column migration sequence through ${matchedMigration.tag}.`); } - return true; + return recovered; } export const __migrateTestUtils = { @@ -263,7 +337,9 @@ export const __migrateTestUtils = { normalizeSqlForMatch, extractFailedSqlFromError, findMatchingSingleStatementMigration, + readRecoveryMigrations, markMigrationRecordIfMissing, + recoverMigrationSequence, tryRecoverDuplicateColumnMigrationError, };