diff --git a/src/server/db/migrate.test.ts b/src/server/db/migrate.test.ts index 4adaa82..08695c5 100644 --- a/src/server/db/migrate.test.ts +++ b/src/server/db/migrate.test.ts @@ -384,4 +384,84 @@ describe('sqlite migrate bootstrap', () => { verified.close(); }); + + it('deduplicates legacy duplicate sites before applying the oauth site unique index', async () => { + const dataDir = mkdtempSync(join(tmpdir(), 'metapi-migrate-duplicate-sites-')); + const dbPath = join(dataDir, 'hub.db'); + const sqlite = new Database(dbPath); + const journalEntries = readMigrationJournalEntries(); + const appliedEntries = journalEntries.filter((entry) => entry.tag !== '0013_oauth_multi_provider'); + + for (const entry of appliedEntries) { + const sqlText = readFileSync(join(migrationsDir, `${entry.tag}.sql`), 'utf8'); + applyMigrationSql(sqlite, sqlText); + } + recordAppliedMigrations(sqlite, appliedEntries); + + sqlite.exec(` + INSERT INTO sites (id, name, url, platform, status, is_pinned, sort_order, global_weight) + VALUES + (101, 'Primary Codex', 'https://chatgpt.com/backend-api/codex', 'codex', 'active', 0, 0, 1), + (202, 'Duplicate Codex', 'https://chatgpt.com/backend-api/codex', 'codex', 'disabled', 1, 9, 3); + + INSERT INTO accounts (site_id, username, access_token, status, checkin_enabled) + VALUES + (101, 'first@example.com', 'token-a', 'active', 0), + (202, 'second@example.com', 'token-b', 'disabled', 0); + + INSERT INTO site_disabled_models (site_id, model_name) + VALUES + (101, 'gpt-5'), + (202, 'gpt-5'), + (202, 'gpt-5-mini'); + `); + + sqlite.close(); + + process.env.DATA_DIR = dataDir; + vi.resetModules(); + + await expect(import('./migrate.js')).resolves.toMatchObject({ + runSqliteMigrations: expect.any(Function), + }); + + const verified = new Database(dbPath, { readonly: true }); + const sites = verified + .prepare('SELECT id, name, url, platform, status, is_pinned, sort_order, global_weight FROM sites ORDER BY id ASC') + .all() as Array<{ + id: number; + name: string; + url: string; + platform: string; + status: string; + is_pinned: number; + sort_order: number; + global_weight: number; + }>; + const accounts = verified + .prepare('SELECT username, site_id FROM accounts ORDER BY username ASC') + .all() as Array<{ username: string; site_id: number }>; + const disabledModels = verified + .prepare('SELECT site_id, model_name FROM site_disabled_models ORDER BY site_id ASC, model_name ASC') + .all() as Array<{ site_id: number; model_name: string }>; + + expect(sites).toEqual([ + expect.objectContaining({ + id: 101, + name: 'Primary Codex', + url: 'https://chatgpt.com/backend-api/codex', + platform: 'codex', + }), + ]); + expect(accounts).toEqual([ + { username: 'first@example.com', site_id: 101 }, + { username: 'second@example.com', site_id: 101 }, + ]); + expect(disabledModels).toEqual([ + { site_id: 101, model_name: 'gpt-5' }, + { site_id: 101, model_name: 'gpt-5-mini' }, + ]); + + verified.close(); + }); }); diff --git a/src/server/db/migrate.ts b/src/server/db/migrate.ts index bc12b61..bfa3a91 100644 --- a/src/server/db/migrate.ts +++ b/src/server/db/migrate.ts @@ -34,6 +34,12 @@ type RecoveryMigration = RecoveryMigrationRecord & { statements: string[]; }; +type LegacySiteRow = { + id: number; + platform: string; + url: string; +}; + const VERIFIED_BOOTSTRAP_TAG = '0012_account_token_value_status'; const VERIFIED_SCHEMA_MARKERS: SchemaMarker[] = [ { table: 'sites' }, @@ -311,6 +317,21 @@ function isRecoverableSchemaConflictError(error: unknown): boolean { || lowered.includes('already exists'); } +function isSitesPlatformUrlUniqueConflictError(error: unknown): boolean { + const lowered = normalizeSchemaErrorMessage(error).toLowerCase(); + if (!lowered.includes('unique constraint failed: sites.platform, sites.url')) { + return false; + } + + const failedSqlText = extractFailedSqlFromError(error); + if (!failedSqlText) { + return true; + } + + return normalizeSqlForMatch(failedSqlText) + === normalizeSqlForMatch('CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`,`url`);'); +} + function getLatestRecordedMigrationCreatedAt(sqlite: Database.Database): number | null { if (!tableExists(sqlite, '__drizzle_migrations')) return null; const row = sqlite @@ -384,6 +405,107 @@ function tryRecoverDuplicateColumnMigrationError( return recovered; } +function rewriteDownstreamSiteWeightMultipliers( + sqlite: Database.Database, + siteIdMapping: Map, +): void { + if (siteIdMapping.size <= 0) return; + if (!tableExists(sqlite, 'downstream_api_keys')) return; + if (!columnExists(sqlite, 'downstream_api_keys', 'site_weight_multipliers')) return; + + const rows = sqlite.prepare(` + SELECT id, site_weight_multipliers + FROM downstream_api_keys + WHERE site_weight_multipliers IS NOT NULL + AND TRIM(site_weight_multipliers) <> '' + `).all() as Array<{ id: number; site_weight_multipliers: string | null }>; + + const update = sqlite.prepare('UPDATE downstream_api_keys SET site_weight_multipliers = ? WHERE id = ?'); + for (const row of rows) { + if (!row.site_weight_multipliers) continue; + + let parsed: unknown; + try { + parsed = JSON.parse(row.site_weight_multipliers); + } catch { + continue; + } + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue; + const nextValue = { ...(parsed as Record) }; + let changed = false; + + for (const [fromSiteId, toSiteId] of siteIdMapping.entries()) { + const fromKey = String(fromSiteId); + const toKey = String(toSiteId); + if (!(fromKey in nextValue)) continue; + if (!(toKey in nextValue)) { + nextValue[toKey] = nextValue[fromKey]; + } + delete nextValue[fromKey]; + changed = true; + } + + if (!changed) continue; + update.run(JSON.stringify(nextValue), row.id); + } +} + +function deduplicateLegacySitesForUniqueIndex(sqlite: Database.Database): boolean { + const duplicateGroups = sqlite.prepare(` + SELECT platform, url + FROM sites + GROUP BY platform, url + HAVING COUNT(*) > 1 + `).all() as Array<{ platform: string; url: string }>; + + if (duplicateGroups.length <= 0) { + return false; + } + + const selectSitesByIdentity = sqlite.prepare(` + SELECT id, platform, url + FROM sites + WHERE platform = ? AND url = ? + ORDER BY id ASC + `); + const rebindAccounts = sqlite.prepare('UPDATE accounts SET site_id = ? WHERE site_id = ?'); + const mergeDisabledModels = sqlite.prepare(` + INSERT OR IGNORE INTO site_disabled_models (site_id, model_name, created_at) + SELECT ?, model_name, created_at + FROM site_disabled_models + WHERE site_id = ? + `); + const deleteDisabledModels = sqlite.prepare('DELETE FROM site_disabled_models WHERE site_id = ?'); + const deleteSite = sqlite.prepare('DELETE FROM sites WHERE id = ?'); + + const siteIdMapping = new Map(); + + const transaction = sqlite.transaction(() => { + for (const group of duplicateGroups) { + const sites = selectSitesByIdentity.all(group.platform, group.url) as LegacySiteRow[]; + if (sites.length <= 1) continue; + + const canonicalSiteId = sites[0]!.id; + for (const site of sites.slice(1)) { + mergeDisabledModels.run(canonicalSiteId, site.id); + deleteDisabledModels.run(site.id); + rebindAccounts.run(canonicalSiteId, site.id); + siteIdMapping.set(site.id, canonicalSiteId); + deleteSite.run(site.id); + } + } + + rewriteDownstreamSiteWeightMultipliers(sqlite, siteIdMapping); + }); + + transaction(); + if (siteIdMapping.size > 0) { + console.warn(`[db] Deduplicated ${siteIdMapping.size} legacy site entries before applying sites_platform_url_unique.`); + } + return siteIdMapping.size > 0; +} + export const __migrateTestUtils = { splitMigrationStatements, normalizeSqlForMatch, @@ -395,6 +517,8 @@ export const __migrateTestUtils = { markMigrationRecordIfMissing, recoverMigrationSequence, tryRecoverDuplicateColumnMigrationError, + isSitesPlatformUrlUniqueConflictError, + deduplicateLegacySitesForUniqueIndex, }; function bootstrapLegacyDrizzleMigrations(sqlite: Database.Database, migrationsFolder: string): boolean { @@ -437,7 +561,13 @@ export function runSqliteMigrations(): void { try { migrate(drizzle(sqlite), { migrationsFolder }); } catch (error) { - if (!tryRecoverDuplicateColumnMigrationError(sqlite, migrationsFolder, error)) { + const recoveredDuplicateColumns = tryRecoverDuplicateColumnMigrationError(sqlite, migrationsFolder, error); + const recoveredDuplicateSites = ( + !recoveredDuplicateColumns + && isSitesPlatformUrlUniqueConflictError(error) + && deduplicateLegacySitesForUniqueIndex(sqlite) + ); + if (!recoveredDuplicateColumns && !recoveredDuplicateSites) { sqlite.close(); throw error; } diff --git a/src/server/db/runtimeSchemaBootstrap.test.ts b/src/server/db/runtimeSchemaBootstrap.test.ts index f47e790..adcf117 100644 --- a/src/server/db/runtimeSchemaBootstrap.test.ts +++ b/src/server/db/runtimeSchemaBootstrap.test.ts @@ -231,4 +231,74 @@ describe('runtime schema bootstrap', () => { 'CREATE INDEX `proxy_logs_downstream_api_key_created_at_idx` ON `proxy_logs` (`downstream_api_key_id`, `created_at`(191))', ); }); + + it('does not add mysql text prefixes when live indexed text columns are varchar-backed', async () => { + const executedSql: string[] = []; + const minimalContract: SchemaContract = { + tables: { + sites: { + columns: { + platform: makeColumn({ logicalType: 'text', notNull: true }), + url: makeColumn({ logicalType: 'text', notNull: true }), + }, + }, + }, + indexes: [], + uniques: [ + { + name: 'sites_platform_url_unique', + table: 'sites', + columns: ['platform', 'url'], + }, + ], + foreignKeys: [], + }; + + await ensureRuntimeDatabaseSchema({ + ...createStubClient('mysql', executedSql), + execute: async (sqlText: string) => { + if (sqlText.includes('FROM information_schema.columns')) { + return [[ + { + table_name: 'sites', + column_name: 'platform', + data_type: 'varchar', + column_type: 'varchar(32)', + }, + { + table_name: 'sites', + column_name: 'url', + data_type: 'varchar', + column_type: 'varchar(255)', + }, + ]]; + } + + executedSql.push(sqlText); + return []; + }, + queryScalar: async (sqlText: string, params: unknown[] = []) => { + if (sqlText.includes('information_schema.tables')) { + return params[0] === 'sites' ? 1 : 0; + } + if (sqlText.includes('information_schema.columns')) { + return 1; + } + return 0; + }, + }, { + currentContract: minimalContract, + liveContract: { + ...minimalContract, + uniques: [], + }, + }); + + expect(executedSql).toContain( + 'CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`, `url`)', + ); + expect(executedSql).not.toContain( + 'CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`(191), `url`(191))', + ); + }); }); diff --git a/src/server/db/runtimeSchemaBootstrap.ts b/src/server/db/runtimeSchemaBootstrap.ts index b664af5..673c9c4 100644 --- a/src/server/db/runtimeSchemaBootstrap.ts +++ b/src/server/db/runtimeSchemaBootstrap.ts @@ -312,12 +312,8 @@ async function resolveMySqlIndexPrefixRequirements( } const declaredType = String(row.column_type || row.data_type || ''); - if (!requiresMysqlIndexPrefixForColumnType(declaredType)) { - continue; - } - requirements[tableName] ??= {}; - requirements[tableName][columnName] = true; + requirements[tableName][columnName] = requiresMysqlIndexPrefixForColumnType(declaredType); } return requirements;