From 23f302e50fb95b76d9ce528cc33dd48c5da7aa88 Mon Sep 17 00:00:00 2001 From: tbphp Date: Wed, 29 Jul 2026 04:46:54 +0800 Subject: [PATCH] =?UTF-8?q?test(web):=20=E5=BB=BA=E7=AB=8B=E7=A1=AE?= =?UTF-8?q?=E5=AE=9A=E6=80=A7=E8=A7=86=E8=A7=89=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/e2e/harness-lifecycle.test.ts | 5 +- web/e2e/run-e2e.mjs | 3 + web/e2e/start-e2e-server.mjs | 13 +- web/e2e/visual-api.ts | 426 ++++++++++++++++++++++++++++++ web/e2e/visual-capture.ts | 143 ++++++++++ web/e2e/visual-fixtures.test.ts | 24 +- web/e2e/visual-fixtures.ts | 148 +++++++++++ web/e2e/visual-scenarios.spec.ts | 143 ++++++++++ 8 files changed, 898 insertions(+), 7 deletions(-) create mode 100644 web/e2e/visual-api.ts create mode 100644 web/e2e/visual-capture.ts create mode 100644 web/e2e/visual-scenarios.spec.ts diff --git a/web/e2e/harness-lifecycle.test.ts b/web/e2e/harness-lifecycle.test.ts index fa14e2ee..aa27aa48 100644 --- a/web/e2e/harness-lifecycle.test.ts +++ b/web/e2e/harness-lifecycle.test.ts @@ -44,13 +44,14 @@ it('fails before starting services when the harness runs on Windows', async () = } }) -it('uses ephemeral ports with explicit serial, per-run isolation', async () => { +it('uses ephemeral ports by default and an explicit fixed visual port', async () => { const launcherPath = resolve(process.cwd(), 'e2e/start-e2e-server.mjs') const runnerPath = resolve(process.cwd(), 'e2e/run-e2e.mjs') const source = `${await readFile(launcherPath, 'utf8')}\n${await readFile(runnerPath, 'utf8')}` expect(source).toContain('server.listen(0, host') - expect(source).not.toMatch(/\b3107\b|\b3108\b/) + expect(source).toContain('GPT_LOAD_E2E_APP_PORT') + expect(source).toContain("'43101'") expect(source).toContain('`${artifactStem}-ready.json`') expect(source).toContain('`${project}-serial-${scenario}`') expect(source).toContain("isolation: 'per-run'") diff --git a/web/e2e/run-e2e.mjs b/web/e2e/run-e2e.mjs index 52aa291d..307ec6c6 100644 --- a/web/e2e/run-e2e.mjs +++ b/web/e2e/run-e2e.mjs @@ -24,6 +24,8 @@ const scenario = (playwrightArgs.find((value) => value.endsWith('.spec.ts')) ?? .replace(/\.spec\.ts$/, '') .replace(/[^a-z0-9-]/gi, '-') const artifactStem = `${project}-serial-${scenario}` +const visualAppPort = + scenario === 'visual-scenarios' ? (process.env.GPT_LOAD_E2E_APP_PORT ?? '43101') : undefined const runDirectory = await mkdtemp(join(tmpdir(), 'gpt-load-e2e-run-')) const readyFile = join(runDirectory, `${artifactStem}-ready.json`) const authKey = `e2e-auth-${randomBytes(24).toString('hex')}` @@ -35,6 +37,7 @@ const server = spawn(process.execPath, [join(webRoot, 'e2e/start-e2e-server.mjs' GPT_LOAD_E2E_AUTH_KEY: authKey, GPT_LOAD_E2E_PROJECT: project, GPT_LOAD_E2E_SCENARIO: scenario, + ...(visualAppPort === undefined ? {} : { GPT_LOAD_E2E_APP_PORT: visualAppPort }), }, stdio: ['ignore', 'inherit', 'inherit'], }) diff --git a/web/e2e/start-e2e-server.mjs b/web/e2e/start-e2e-server.mjs index 67ab86f3..f522d46d 100644 --- a/web/e2e/start-e2e-server.mjs +++ b/web/e2e/start-e2e-server.mjs @@ -67,6 +67,17 @@ async function reserveEphemeralPort() { return port } +function configuredAppPort() { + const value = process.env.GPT_LOAD_E2E_APP_PORT + if (value === undefined) return undefined + if (!/^\d+$/.test(value)) throw new Error('E2E application port must be a decimal integer') + const port = Number(value) + if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { + throw new Error('E2E application port is outside the valid range') + } + return port +} + async function requestBody(request) { const chunks = [] for await (const chunk of request) chunks.push(chunk) @@ -139,7 +150,7 @@ try { await rm(dataDir, { recursive: true, force: true }) throw error } -const appPort = await reserveEphemeralPort() +const appPort = configuredAppPort() ?? (await reserveEphemeralPort()) const child = spawn(binary, [], { detached: true, env: { diff --git a/web/e2e/visual-api.ts b/web/e2e/visual-api.ts new file mode 100644 index 00000000..9e1a0f96 --- /dev/null +++ b/web/e2e/visual-api.ts @@ -0,0 +1,426 @@ +import type { Page, Route } from '@playwright/test' + +import { visualClock, visualFixtureData, type VisualScenario } from './visual-fixtures' + +export const visualAuthKey = 'visual-fixture-auth' + +const group = { + id: 7, + name: visualFixtureData.groupName, + upstream_url: 'https://visual-fixture.example/v1', + protocols: ['openai'], + models: [ + { id: visualFixtureData.modelName, alias: 'stable-public-alias' }, + { id: 'visual-fixture-secondary-model', alias: '' }, + ], + enabled: true, + key_count: 4, +} + +const accessKey = { + id: 12, + name: visualFixtureData.longIdentifier, + masked_key: `sk-gl-••••••••${visualFixtureData.accessKeySuffix}`, + status: 'active', + filters: { + groups: [group.id], + protocols: ['openai'], + models: [visualFixtureData.modelName], + }, + rpm_limit: 120, + created_at: '2026-07-28T10:00:00.000Z', + updated_at: visualClock, +} + +const requestLogHealth = { + enqueued_total: 8, + persisted_total: 6, + dropped_not_running_total: 0, + dropped_queue_full_total: 1, + dropped_stopping_total: 0, + dropped_persist_failed_total: 1, + dropped_shutdown_total: 0, + dropped_total: 2, + write_failure_total: 1, + retention_delete_failure_total: 0, + queue_depth: 2, + queue_capacity: 256, + last_write_failure_at: '2026-07-28T23:50:00.000Z', + last_retention_failure_at: null, +} + +const normalHealth = { + observed_at: visualClock, + snapshot_revision: 42, + stats_window_seconds: 300, + counts: { total: 4, available: 4, cooldown: 0, blacklisted: 0, disabled: 0 }, + groups: [ + { + id: group.id, + name: group.name, + enabled: true, + counts: { total: 4, available: 4, cooldown: 0, blacklisted: 0, disabled: 0 }, + }, + ], + cooldown_keys: [], + blacklisted_keys: [], + request_log: { + ...requestLogHealth, + dropped_queue_full_total: 0, + dropped_persist_failed_total: 0, + dropped_total: 0, + write_failure_total: 0, + last_write_failure_at: null, + }, +} + +const anomalyHealth = { + ...normalHealth, + counts: { total: 4, available: 1, cooldown: 1, blacklisted: 1, disabled: 1 }, + groups: [ + { + id: group.id, + name: group.name, + enabled: true, + counts: { total: 4, available: 1, cooldown: 1, blacklisted: 1, disabled: 1 }, + }, + ], + cooldown_keys: [ + { + key_id: 91, + group_id: group.id, + group_name: group.name, + cooldown_until: '2026-07-29T00:10:00.000Z', + failure_count: 3, + recent_success_count: 1, + recent_failure_count: 3, + consecutive_failure_count: 2, + weight_manual: null, + weight_auto: 35, + recovery: { + automatic: true, + mode: 'cooldown_expiry', + at: '2026-07-29T00:10:00.000Z', + }, + }, + ], + blacklisted_keys: [ + { + key_id: 92, + group_id: group.id, + group_name: group.name, + failure_count: 8, + recent_success_count: 0, + recent_failure_count: 5, + consecutive_failure_count: 5, + weight_manual: 50, + weight_auto: 0, + recovery: { automatic: true, mode: 'validation_probe', at: null }, + }, + ], + request_log: requestLogHealth, +} + +const aggregate = { + request_count: 6, + success_count: 4, + failure_count: 2, + uncached_input_tokens: 1200, + cache_read_tokens: 300, + cache_write_5m_tokens: 120, + cache_write_1h_tokens: 80, + output_tokens: 500, + total_tokens: 2200, + estimated_cost_usd: 0.01234, + usage_missing_count: 1, + partial_count: 1, + unpriced_request_count: 2, +} + +const usageReport = { + range: '24h', + granularity: 'hour', + timezone: 'UTC', + from: '2026-07-28T00:00:00.000Z', + to: '2026-07-29T00:00:00.000Z', + observed_at: '2026-07-28T23:59:59.000Z', + summary: aggregate, + series: [ + { + ...aggregate, + bucket_start: '2026-07-28T23:00:00.000Z', + bucket_end: '2026-07-29T00:00:00.000Z', + }, + ], + breakdown: [{ ...aggregate, group_id: group.id, model: visualFixtureData.modelName }], + breakdown_truncated: true, + collection_health: { + scope: 'current_process', + dropped_total: 2, + write_failure_total: 1, + last_write_failure_at: '2026-07-28T23:50:00.000Z', + }, +} + +const settings = { + values: { + connect_timeout: 15, + first_byte_timeout: 120, + request_timeout: 600, + stream_idle_timeout: 300, + header_rules: { set: { 'X-Visual-Fixture': 'stable-value' }, remove: ['X-Legacy'] }, + inject_usage_options: true, + request_log_retention_days: 7, + }, + overrides: [], +} + +const modelPrices = { + price_unit: 'usd_per_million_tokens', + builtin: [ + { + pattern: 'visual-model-*', + source: 'builtin', + prices: { + uncached_input: 1.25, + cache_read: 0, + cache_write_5m: null, + cache_write_1h: 2.5, + output: 7.5, + }, + source_url: 'https://visual-fixture.example/pricing', + updated_at: visualClock, + pricing_policy: { + input_threshold_tokens: 272000, + input_multiplier: 2, + output_multiplier: 1.5, + }, + }, + ], + overrides: [ + { + pattern: '*', + source: 'user', + prices: { + uncached_input: 2, + cache_read: 0, + cache_write_5m: null, + cache_write_1h: null, + output: 8, + }, + source_url: null, + updated_at: visualClock, + pricing_policy: null, + }, + ], +} + +const requestLogPage = { + items: [ + { + request_id: visualFixtureData.requestId, + completed_at: visualClock, + access_key: { id: accessKey.id, name: accessKey.name, deleted: false }, + protocol: 'openai', + client_model: 'stable-public-alias', + upstream_model: visualFixtureData.modelName, + status: 'error', + status_code: 429, + duration_ms: 842, + error_code: 'RATE_LIMITED', + error_summary: 'Fixture upstream rate limit', + affinity_hit: false, + attempts: [ + { + sequence: 1, + group_id: group.id, + group_name: group.name, + key_id: 91, + key_mask: 'sk-v****1234', + upstream_model: visualFixtureData.modelName, + status_code: 429, + duration_ms: 842, + failure_category: 'rate_limited', + action: 'cooldown_key', + will_retry: false, + error_code: 'RATE_LIMITED', + error_summary: 'Fixture upstream rate limit', + committed: true, + }, + ], + group_id: group.id, + usage_state: 'missing', + cost_state: 'unpriced', + uncached_input_tokens: 0, + cache_read_tokens: 0, + cache_write_5m_tokens: 0, + cache_write_1h_tokens: 0, + output_tokens: 0, + estimated_cost_usd: 0, + }, + ], + next_cursor: null, +} + +const routeInspection = { + observed_at: visualClock, + snapshot_revision: 42, + protocol: 'openai', + external_model: 'stable-public-alias', + access_key: { id: accessKey.id, name: accessKey.name, status: 'active' }, + routable: true, + reason_code: null, + groups: [ + { + group_id: group.id, + group_name: group.name, + upstream_model: visualFixtureData.modelName, + weight_manual: null, + included: true, + routable: true, + reason_code: null, + keys: [ + { + key_id: 91, + available: true, + reason_code: null, + weight_manual: null, + weight_auto: 72, + effective_weight: 5184, + cooldown_until: null, + }, + ], + }, + ], +} + +function success(data: unknown, headers: Record = {}) { + return { + status: 200, + headers: { 'content-type': 'application/json; charset=utf-8', ...headers }, + body: JSON.stringify({ code: 0, message: 'OK', data }), + } +} + +function failure() { + return { + status: 503, + headers: { 'content-type': 'application/json; charset=utf-8' }, + body: JSON.stringify({ + code: 'SERVICE_UNAVAILABLE', + message: 'Fixture service unavailable', + }), + } +} + +export interface VisualApiState { + healthRequests: number + unexpectedRequests: string[] +} + +export async function installVisualApi( + page: Page, + scenario: VisualScenario, +): Promise { + const state: VisualApiState = { healthRequests: 0, unexpectedRequests: [] } + await page.route('**/api/**', async (route: Route) => { + const request = route.request() + const url = new URL(request.url()) + const method = request.method() + + if (url.pathname === '/api/auth/session') { + await route.fulfill(success({ authenticated: true })) + return + } + if (url.pathname === '/api/groups' && method === 'GET') { + await route.fulfill(success(scenario === 'home-empty-error' ? [] : [group])) + return + } + if (url.pathname === '/api/access-keys/options' && method === 'GET') { + await route.fulfill( + success( + scenario === 'home-empty-error' + ? [] + : [{ id: accessKey.id, name: accessKey.name, status: accessKey.status }], + ), + ) + return + } + if (url.pathname === '/api/access-keys' && method === 'GET') { + await route.fulfill(success([accessKey])) + return + } + if ( + url.pathname === '/api/access-keys' && + method === 'POST' && + scenario === 'access-key-operation' + ) { + await route.abort('connectionreset') + return + } + if (url.pathname === '/api/health' && method === 'GET') { + state.healthRequests += 1 + if ( + scenario === 'home-empty-error' || + (scenario === 'home-anomaly' && state.healthRequests > 1) + ) { + await route.fulfill(failure()) + return + } + await route.fulfill(success(scenario === 'home-anomaly' ? anomalyHealth : normalHealth)) + return + } + if (url.pathname === '/api/usage' && method === 'GET') { + await route.fulfill(scenario === 'home-empty-error' ? failure() : success(usageReport)) + return + } + if (url.pathname === '/api/logs' && method === 'GET') { + await route.fulfill(success(requestLogPage)) + return + } + if (url.pathname === '/api/settings' && method === 'GET') { + await route.fulfill( + success(settings, { + etag: '"sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"', + }), + ) + return + } + if (url.pathname === '/api/system/info' && method === 'GET') { + await route.fulfill( + success({ + version: '2.0.0-visual', + deployment: { + instance_mode: 'single', + database: 'sqlite', + distribution: 'single_binary', + }, + data_dir: '/visual/data', + auth_key: { source: 'environment', path: null }, + encryption: { enabled: true, source: 'environment', path: null }, + }), + ) + return + } + if (url.pathname === '/api/model-prices' && method === 'GET') { + await route.fulfill(success(modelPrices)) + return + } + if (url.pathname === '/api/route/inspect' && method === 'POST') { + await route.fulfill(success(routeInspection)) + return + } + + const identity = `${method} ${url.pathname}${url.search}` + state.unexpectedRequests.push(identity) + await route.fulfill({ + status: 500, + contentType: 'application/json', + body: JSON.stringify({ + code: 'VISUAL_FIXTURE_UNEXPECTED_REQUEST', + message: identity, + }), + }) + }) + return state +} diff --git a/web/e2e/visual-capture.ts b/web/e2e/visual-capture.ts new file mode 100644 index 00000000..bfba4581 --- /dev/null +++ b/web/e2e/visual-capture.ts @@ -0,0 +1,143 @@ +import { createHash } from 'node:crypto' +import { mkdir, writeFile } from 'node:fs/promises' +import { relative, resolve, sep } from 'node:path' + +import type { Page, TestInfo } from '@playwright/test' + +import { visualClock, visualFixtureVersion, type VisualScenarioCase } from './visual-fixtures' + +interface Rect { + x: number + y: number + width: number + height: number +} + +interface VisualGeometry { + viewport: { width: number; height: number } + document: { width: number; height: number } + main: Rect + heading: Rect + horizontalOverflow: number +} + +export interface VisualCapture { + id: string + scenario: VisualScenarioCase['scenario'] + path: string + locale: VisualScenarioCase['locale'] + theme: VisualScenarioCase['theme'] + viewport: VisualScenarioCase['viewport'] + screenshot: { + path: string + sha256: string + } + geometry: VisualGeometry +} + +function normalizePath(value: string): string { + return value.split(sep).join('/') +} + +function artifactRoot(): string { + return resolve(process.cwd(), process.env.GPT_LOAD_E2E_ARTIFACT_DIR ?? 'test-results/unscoped') +} + +export async function captureVisualScenario( + page: Page, + testInfo: TestInfo, + scenarioCase: VisualScenarioCase, +): Promise { + const geometry = await page.evaluate(() => { + const main = document.querySelector('main') + const heading = main?.querySelector('h1') + if (!(main instanceof HTMLElement) || !(heading instanceof HTMLElement)) { + throw new Error('Visual scenario requires one rendered main heading') + } + const mainRect = main.getBoundingClientRect() + const headingRect = heading.getBoundingClientRect() + const round = (value: number) => Math.round(value * 1_000) / 1_000 + const rect = (value: DOMRect): Rect => ({ + x: round(value.x), + y: round(value.y), + width: round(value.width), + height: round(value.height), + }) + return { + viewport: { + width: document.documentElement.clientWidth, + height: document.documentElement.clientHeight, + }, + document: { + width: document.documentElement.scrollWidth, + height: document.documentElement.scrollHeight, + }, + main: rect(mainRect), + heading: rect(headingRect), + horizontalOverflow: Math.max( + 0, + document.documentElement.scrollWidth - document.documentElement.clientWidth, + ), + } + }) + + if ( + geometry.viewport.width !== scenarioCase.viewport.width || + geometry.viewport.height !== scenarioCase.viewport.height + ) { + throw new Error(`Visual viewport drifted for ${scenarioCase.id}`) + } + if (geometry.horizontalOverflow !== 0) { + throw new Error(`Body overflowed horizontally for ${scenarioCase.id}`) + } + if ( + geometry.heading.width <= 0 || + geometry.heading.height <= 0 || + geometry.heading.x < 0 || + geometry.heading.x + geometry.heading.width > geometry.viewport.width + ) { + throw new Error(`Main heading geometry is invalid for ${scenarioCase.id}`) + } + + const screenshotPath = testInfo.outputPath('candidate.png') + const screenshot = await page.screenshot({ + path: screenshotPath, + fullPage: true, + animations: 'disabled', + caret: 'hide', + }) + return { + id: scenarioCase.id, + scenario: scenarioCase.scenario, + path: scenarioCase.path, + locale: scenarioCase.locale, + theme: scenarioCase.theme, + viewport: scenarioCase.viewport, + screenshot: { + path: normalizePath(relative(artifactRoot(), screenshotPath)), + sha256: createHash('sha256').update(screenshot).digest('hex'), + }, + geometry, + } +} + +export async function writeVisualScenarioManifest(captures: VisualCapture[]): Promise { + const sortedCaptures = [...captures].sort((left, right) => left.id.localeCompare(right.id)) + const payload = { + schema_version: 1, + fixture_version: visualFixtureVersion, + clock: visualClock, + captures: sortedCaptures, + } + const manifest = { + ...payload, + manifest_sha256: createHash('sha256').update(JSON.stringify(payload)).digest('hex'), + } + const root = artifactRoot() + await mkdir(root, { recursive: true }) + await writeFile( + resolve(root, 'scenario-manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + 'utf8', + ) +} diff --git a/web/e2e/visual-fixtures.test.ts b/web/e2e/visual-fixtures.test.ts index 75c2698e..4e6430fd 100644 --- a/web/e2e/visual-fixtures.test.ts +++ b/web/e2e/visual-fixtures.test.ts @@ -7,23 +7,28 @@ import { visualClock, visualFixtureData, visualLocales, + visualScenarioCases, visualScenarios, visualThemes, visualViewports, } from './visual-fixtures' -const businessFlowSource = readFileSync(resolve('e2e/business-flows.spec.ts'), 'utf8') +const visualScenarioSource = readFileSync(resolve('e2e/visual-scenarios.spec.ts'), 'utf8') describe('deterministic visual fixtures', () => { it('freezes the approved scenario, viewport, theme, and locale matrix', () => { expect(visualScenarios).toEqual([ 'home-normal', 'home-anomaly', + 'home-empty-error', 'access-keys-long', + 'access-key-operation', 'model-prices-mixed', 'settings-dirty', + 'settings-validation', 'usage-quality', 'logs-signal-path', + 'inspector-routing', ]) expect(visualViewports).toEqual([ { width: 375, height: 812 }, @@ -48,10 +53,21 @@ describe('deterministic visual fixtures', () => { expect(serialized).not.toContain('Bearer ') }) - it('binds every named scenario and locale to the executable browser journey', () => { + it('binds every scenario and each approved render dimension to executable cases', () => { for (const scenario of visualScenarios) { - expect(businessFlowSource).toContain(`visualScenarioLabel('${scenario}')`) + expect(visualScenarioCases.some((candidate) => candidate.scenario === scenario)).toBe(true) } - expect(businessFlowSource).toContain('for (const locale of visualLocales)') + for (const viewport of visualViewports) { + expect(visualScenarioCases.some((candidate) => candidate.viewport === viewport)).toBe(true) + } + for (const locale of visualLocales) { + expect(visualScenarioCases.some((candidate) => candidate.locale === locale)).toBe(true) + } + for (const theme of visualThemes) { + expect(visualScenarioCases.some((candidate) => candidate.theme === theme)).toBe(true) + } + + expect(visualScenarioSource).toContain('installVisualApi') + expect(visualScenarioSource).toContain('captureVisualScenario') }) }) diff --git a/web/e2e/visual-fixtures.ts b/web/e2e/visual-fixtures.ts index 0b5258b8..977802f7 100644 --- a/web/e2e/visual-fixtures.ts +++ b/web/e2e/visual-fixtures.ts @@ -4,11 +4,15 @@ export const visualClock = '2026-07-29T00:00:00.000Z' export const visualScenarios = [ 'home-normal', 'home-anomaly', + 'home-empty-error', 'access-keys-long', + 'access-key-operation', 'model-prices-mixed', 'settings-dirty', + 'settings-validation', 'usage-quality', 'logs-signal-path', + 'inspector-routing', ] as const export type VisualScenario = (typeof visualScenarios)[number] @@ -16,11 +20,15 @@ export type VisualScenario = (typeof visualScenarios)[number] const visualScenarioDescriptions: Record = { 'home-normal': 'Home normal operation', 'home-anomaly': 'Home usage and health anomaly', + 'home-empty-error': 'Home empty and unavailable resources', 'access-keys-long': 'AccessKey long record', + 'access-key-operation': 'AccessKey indeterminate operation notice', 'model-prices-mixed': 'Model prices mixed values', 'settings-dirty': 'Settings dirty state', + 'settings-validation': 'Settings validation summary', 'usage-quality': 'Usage quality signals', 'logs-signal-path': 'Logs signal path', + 'inspector-routing': 'Route inspector result', } export function visualScenarioLabel(scenario: VisualScenario): string { @@ -37,6 +45,146 @@ export const visualViewports = [ export const visualThemes = ['light', 'dark'] as const export const visualLocales = ['en-US', 'zh-CN'] as const +export interface VisualScenarioCase { + id: string + scenario: VisualScenario + path: string + viewport: (typeof visualViewports)[number] + theme: (typeof visualThemes)[number] + locale: (typeof visualLocales)[number] +} + +export const visualScenarioCases: readonly VisualScenarioCase[] = [ + { + id: 'home-normal-desktop-en-light', + scenario: 'home-normal', + path: '/', + viewport: visualViewports[3], + theme: 'light', + locale: 'en-US', + }, + { + id: 'home-normal-mobile-zh-dark', + scenario: 'home-normal', + path: '/', + viewport: visualViewports[0], + theme: 'dark', + locale: 'zh-CN', + }, + { + id: 'home-anomaly-tablet-en-dark', + scenario: 'home-anomaly', + path: '/', + viewport: visualViewports[2], + theme: 'dark', + locale: 'en-US', + }, + { + id: 'home-empty-error-tablet-zh-light', + scenario: 'home-empty-error', + path: '/', + viewport: visualViewports[1], + theme: 'light', + locale: 'zh-CN', + }, + { + id: 'access-keys-long-desktop-en-light', + scenario: 'access-keys-long', + path: '/access-keys', + viewport: visualViewports[3], + theme: 'light', + locale: 'en-US', + }, + { + id: 'access-keys-long-mobile-zh-dark', + scenario: 'access-keys-long', + path: '/access-keys', + viewport: visualViewports[0], + theme: 'dark', + locale: 'zh-CN', + }, + { + id: 'access-key-operation-tablet-en-light', + scenario: 'access-key-operation', + path: '/access-keys', + viewport: visualViewports[1], + theme: 'light', + locale: 'en-US', + }, + { + id: 'model-prices-tablet-zh-dark', + scenario: 'model-prices-mixed', + path: '/settings/model-prices', + viewport: visualViewports[2], + theme: 'dark', + locale: 'zh-CN', + }, + { + id: 'settings-dirty-desktop-en-light', + scenario: 'settings-dirty', + path: '/settings', + viewport: visualViewports[3], + theme: 'light', + locale: 'en-US', + }, + { + id: 'settings-validation-mobile-zh-light', + scenario: 'settings-validation', + path: '/settings', + viewport: visualViewports[0], + theme: 'light', + locale: 'zh-CN', + }, + { + id: 'usage-quality-desktop-en-dark', + scenario: 'usage-quality', + path: '/monitor?tab=usage&range=24h', + viewport: visualViewports[3], + theme: 'dark', + locale: 'en-US', + }, + { + id: 'usage-quality-tablet-zh-light', + scenario: 'usage-quality', + path: '/monitor?tab=usage&range=24h', + viewport: visualViewports[1], + theme: 'light', + locale: 'zh-CN', + }, + { + id: 'logs-signal-tablet-en-light', + scenario: 'logs-signal-path', + path: '/monitor?tab=logs', + viewport: visualViewports[2], + theme: 'light', + locale: 'en-US', + }, + { + id: 'logs-signal-mobile-zh-dark', + scenario: 'logs-signal-path', + path: '/monitor?tab=logs', + viewport: visualViewports[0], + theme: 'dark', + locale: 'zh-CN', + }, + { + id: 'inspector-routing-desktop-en-light', + scenario: 'inspector-routing', + path: '/monitor?tab=inspector&protocol=openai&external_model=stable-public-alias&access_key_id=12', + viewport: visualViewports[3], + theme: 'light', + locale: 'en-US', + }, + { + id: 'inspector-routing-tablet-zh-dark', + scenario: 'inspector-routing', + path: '/monitor?tab=inspector&protocol=openai&external_model=stable-public-alias&access_key_id=12', + viewport: visualViewports[1], + theme: 'dark', + locale: 'zh-CN', + }, +] + export const visualFixtureData = Object.freeze({ requestId: '5297f18d-3ca7-42d9-8a5b-2ce08ccd1b01', accessKeySuffix: 'a11e', diff --git a/web/e2e/visual-scenarios.spec.ts b/web/e2e/visual-scenarios.spec.ts new file mode 100644 index 00000000..abd35e90 --- /dev/null +++ b/web/e2e/visual-scenarios.spec.ts @@ -0,0 +1,143 @@ +import { expect, test } from './fixtures' +import { installVisualApi, visualAuthKey } from './visual-api' +import { + captureVisualScenario, + writeVisualScenarioManifest, + type VisualCapture, +} from './visual-capture' +import { + visualFixtureData, + visualScenarioCases, + visualScenarioLabel, + type VisualScenarioCase, +} from './visual-fixtures' + +const captures: VisualCapture[] = [] + +test.setTimeout(60_000) +test.afterAll(async () => { + await writeVisualScenarioManifest(captures) +}) + +async function prepareScenario( + page: Parameters[0], + item: VisualScenarioCase, +) { + if (item.scenario === 'home-normal') { + await expect(page.locator('[data-test="home-health-total"]')).toBeVisible() + await expect(page.locator('[data-test="home-usage-requests"]')).toBeVisible() + await expect(page.getByText(visualFixtureData.groupName, { exact: true })).toBeVisible() + return + } + + if (item.scenario === 'home-anomaly') { + await expect(page.locator('[data-test="home-health-cooldown"]')).toHaveAttribute( + 'data-state', + 'anomaly', + ) + const desktopSettings = page.locator('.desktop-nav a[href="/settings"]') + if (await desktopSettings.isVisible()) { + await desktopSettings.click() + } else { + await page.locator('.mobile-menu-trigger').click() + await page.locator('.mobile-nav a[href="/settings"]').click() + } + await expect(page).toHaveURL('/settings') + await page.locator('a.brand').click() + await expect(page.locator('[data-test="home-operational-overview"]')).toBeVisible() + await expect(page.getByText(/stale|旧数据/).first()).toBeVisible() + return + } + + if (item.scenario === 'home-empty-error') { + await expect(page.locator('[data-test="home-usage-error"]')).toBeVisible() + await expect(page.locator('[data-test="home-groups"]')).toBeVisible() + return + } + + if (item.scenario === 'access-keys-long') { + await expect(page.getByText(visualFixtureData.longIdentifier, { exact: true })).toBeVisible() + return + } + + if (item.scenario === 'access-key-operation') { + await page.locator('[data-test="access-key-create"]').click() + const drawer = page.getByRole('dialog') + await expect(drawer).toBeVisible() + await drawer.locator('[data-test="access-key-name"]').fill('Visual pending AccessKey') + await drawer.locator('[data-test="access-key-save"]').click() + await expect(drawer.getByText(/unknown|未确认/).first()).toBeVisible() + await drawer.locator('.app-drawer__close').click() + await expect(page.locator('[data-test="access-key-operation-notice"]')).toBeVisible() + return + } + + if (item.scenario === 'model-prices-mixed') { + await expect(page.locator('[data-test="builtin-price-row-0"]')).toBeVisible() + await expect(page.locator('[data-test="override-price-row-0"]')).toBeVisible() + return + } + + if (item.scenario === 'settings-dirty' || item.scenario === 'settings-validation') { + await page.locator('[data-test="override-request_timeout"]').check() + await page + .locator('[data-test="value-request_timeout"]') + .fill(item.scenario === 'settings-dirty' ? '901' : '0') + await expect( + page.locator( + item.scenario === 'settings-dirty' + ? '[data-test="settings-dirty-summary"]' + : '[data-test="settings-validation-summary"]', + ), + ).toBeVisible() + return + } + + if (item.scenario === 'usage-quality') { + await expect(page.locator('[data-test="usage-window-quality"]')).toBeVisible() + await expect(page.locator('[data-test="usage-process-health"]')).toBeVisible() + return + } + + if (item.scenario === 'logs-signal-path') { + await expect(page.locator(`[data-test="log-row-${visualFixtureData.requestId}"]`)).toBeVisible() + return + } + + if (item.scenario === 'inspector-routing') { + await page.locator('[data-test="inspector-submit"]').click() + await expect(page.locator('[data-test="inspector-result"]')).toBeVisible() + } +} + +for (const item of visualScenarioCases) { + test(`${visualScenarioLabel(item.scenario)} · ${item.id}`, async ({ page }, testInfo) => { + await page.setViewportSize(item.viewport) + await page.emulateMedia({ + colorScheme: item.theme, + reducedMotion: 'reduce', + }) + await page.addInitScript( + ({ authKey, locale, theme }) => { + window.sessionStorage.setItem('gpt-load.auth-key', authKey) + window.localStorage.setItem('gpt-load.locale', locale) + window.localStorage.setItem('gpt-load.theme', theme) + }, + { authKey: visualAuthKey, locale: item.locale, theme: item.theme }, + ) + const api = await installVisualApi(page, item.scenario) + + await page.goto(item.path) + await expect(page.locator('main h1')).toHaveCount(1) + await prepareScenario(page, item) + await expect(page.locator('.query-feedback--loading')).toHaveCount(0) + await page.evaluate(async () => { + await document.fonts.ready + await new Promise((resolveFrame) => requestAnimationFrame(() => resolveFrame())) + await new Promise((resolveFrame) => requestAnimationFrame(() => resolveFrame())) + }) + expect(api.unexpectedRequests).toEqual([]) + + captures.push(await captureVisualScenario(page, testInfo, item)) + }) +}