test(web): 建立确定性视觉场景
This commit is contained in:
tbphp committed
1 parent
0d645b169b
commit
23f302e50f
8 files changed
+898
-7
No files matched your search
@@ -44,13 +44,14 @@ it('fails before starting services when the harness runs on Windows', async () =
|
||||
}
|
||||
})
|
||||
|
||||
it('uses ephemeral ports with explicit serial, per-run isolation', async () => {
|
||||
it('uses ephemeral ports by default and an explicit fixed visual port', async () => {
|
||||
const launcherPath = resolve(process.cwd(), 'e2e/start-e2e-server.mjs')
|
||||
const runnerPath = resolve(process.cwd(), 'e2e/run-e2e.mjs')
|
||||
const source = `${await readFile(launcherPath, 'utf8')}\n${await readFile(runnerPath, 'utf8')}`
|
||||
|
||||
expect(source).toContain('server.listen(0, host')
|
||||
expect(source).not.toMatch(/\b3107\b|\b3108\b/)
|
||||
expect(source).toContain('GPT_LOAD_E2E_APP_PORT')
|
||||
expect(source).toContain("'43101'")
|
||||
expect(source).toContain('`${artifactStem}-ready.json`')
|
||||
expect(source).toContain('`${project}-serial-${scenario}`')
|
||||
expect(source).toContain("isolation: 'per-run'")
|
||||
|
||||
@@ -24,6 +24,8 @@ const scenario = (playwrightArgs.find((value) => value.endsWith('.spec.ts')) ??
|
||||
.replace(/\.spec\.ts$/, '')
|
||||
.replace(/[^a-z0-9-]/gi, '-')
|
||||
const artifactStem = `${project}-serial-${scenario}`
|
||||
const visualAppPort =
|
||||
scenario === 'visual-scenarios' ? (process.env.GPT_LOAD_E2E_APP_PORT ?? '43101') : undefined
|
||||
const runDirectory = await mkdtemp(join(tmpdir(), 'gpt-load-e2e-run-'))
|
||||
const readyFile = join(runDirectory, `${artifactStem}-ready.json`)
|
||||
const authKey = `e2e-auth-${randomBytes(24).toString('hex')}`
|
||||
@@ -35,6 +37,7 @@ const server = spawn(process.execPath, [join(webRoot, 'e2e/start-e2e-server.mjs'
|
||||
GPT_LOAD_E2E_AUTH_KEY: authKey,
|
||||
GPT_LOAD_E2E_PROJECT: project,
|
||||
GPT_LOAD_E2E_SCENARIO: scenario,
|
||||
...(visualAppPort === undefined ? {} : { GPT_LOAD_E2E_APP_PORT: visualAppPort }),
|
||||
},
|
||||
stdio: ['ignore', 'inherit', 'inherit'],
|
||||
})
|
||||
|
||||
@@ -67,6 +67,17 @@ async function reserveEphemeralPort() {
|
||||
return port
|
||||
}
|
||||
|
||||
function configuredAppPort() {
|
||||
const value = process.env.GPT_LOAD_E2E_APP_PORT
|
||||
if (value === undefined) return undefined
|
||||
if (!/^\d+$/.test(value)) throw new Error('E2E application port must be a decimal integer')
|
||||
const port = Number(value)
|
||||
if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) {
|
||||
throw new Error('E2E application port is outside the valid range')
|
||||
}
|
||||
return port
|
||||
}
|
||||
|
||||
async function requestBody(request) {
|
||||
const chunks = []
|
||||
for await (const chunk of request) chunks.push(chunk)
|
||||
@@ -139,7 +150,7 @@ try {
|
||||
await rm(dataDir, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
const appPort = await reserveEphemeralPort()
|
||||
const appPort = configuredAppPort() ?? (await reserveEphemeralPort())
|
||||
const child = spawn(binary, [], {
|
||||
detached: true,
|
||||
env: {
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
import type { Page, Route } from '@playwright/test'
|
||||
|
||||
import { visualClock, visualFixtureData, type VisualScenario } from './visual-fixtures'
|
||||
|
||||
export const visualAuthKey = 'visual-fixture-auth'
|
||||
|
||||
const group = {
|
||||
id: 7,
|
||||
name: visualFixtureData.groupName,
|
||||
upstream_url: 'https://visual-fixture.example/v1',
|
||||
protocols: ['openai'],
|
||||
models: [
|
||||
{ id: visualFixtureData.modelName, alias: 'stable-public-alias' },
|
||||
{ id: 'visual-fixture-secondary-model', alias: '' },
|
||||
],
|
||||
enabled: true,
|
||||
key_count: 4,
|
||||
}
|
||||
|
||||
const accessKey = {
|
||||
id: 12,
|
||||
name: visualFixtureData.longIdentifier,
|
||||
masked_key: `sk-gl-••••••••${visualFixtureData.accessKeySuffix}`,
|
||||
status: 'active',
|
||||
filters: {
|
||||
groups: [group.id],
|
||||
protocols: ['openai'],
|
||||
models: [visualFixtureData.modelName],
|
||||
},
|
||||
rpm_limit: 120,
|
||||
created_at: '2026-07-28T10:00:00.000Z',
|
||||
updated_at: visualClock,
|
||||
}
|
||||
|
||||
const requestLogHealth = {
|
||||
enqueued_total: 8,
|
||||
persisted_total: 6,
|
||||
dropped_not_running_total: 0,
|
||||
dropped_queue_full_total: 1,
|
||||
dropped_stopping_total: 0,
|
||||
dropped_persist_failed_total: 1,
|
||||
dropped_shutdown_total: 0,
|
||||
dropped_total: 2,
|
||||
write_failure_total: 1,
|
||||
retention_delete_failure_total: 0,
|
||||
queue_depth: 2,
|
||||
queue_capacity: 256,
|
||||
last_write_failure_at: '2026-07-28T23:50:00.000Z',
|
||||
last_retention_failure_at: null,
|
||||
}
|
||||
|
||||
const normalHealth = {
|
||||
observed_at: visualClock,
|
||||
snapshot_revision: 42,
|
||||
stats_window_seconds: 300,
|
||||
counts: { total: 4, available: 4, cooldown: 0, blacklisted: 0, disabled: 0 },
|
||||
groups: [
|
||||
{
|
||||
id: group.id,
|
||||
name: group.name,
|
||||
enabled: true,
|
||||
counts: { total: 4, available: 4, cooldown: 0, blacklisted: 0, disabled: 0 },
|
||||
},
|
||||
],
|
||||
cooldown_keys: [],
|
||||
blacklisted_keys: [],
|
||||
request_log: {
|
||||
...requestLogHealth,
|
||||
dropped_queue_full_total: 0,
|
||||
dropped_persist_failed_total: 0,
|
||||
dropped_total: 0,
|
||||
write_failure_total: 0,
|
||||
last_write_failure_at: null,
|
||||
},
|
||||
}
|
||||
|
||||
const anomalyHealth = {
|
||||
...normalHealth,
|
||||
counts: { total: 4, available: 1, cooldown: 1, blacklisted: 1, disabled: 1 },
|
||||
groups: [
|
||||
{
|
||||
id: group.id,
|
||||
name: group.name,
|
||||
enabled: true,
|
||||
counts: { total: 4, available: 1, cooldown: 1, blacklisted: 1, disabled: 1 },
|
||||
},
|
||||
],
|
||||
cooldown_keys: [
|
||||
{
|
||||
key_id: 91,
|
||||
group_id: group.id,
|
||||
group_name: group.name,
|
||||
cooldown_until: '2026-07-29T00:10:00.000Z',
|
||||
failure_count: 3,
|
||||
recent_success_count: 1,
|
||||
recent_failure_count: 3,
|
||||
consecutive_failure_count: 2,
|
||||
weight_manual: null,
|
||||
weight_auto: 35,
|
||||
recovery: {
|
||||
automatic: true,
|
||||
mode: 'cooldown_expiry',
|
||||
at: '2026-07-29T00:10:00.000Z',
|
||||
},
|
||||
},
|
||||
],
|
||||
blacklisted_keys: [
|
||||
{
|
||||
key_id: 92,
|
||||
group_id: group.id,
|
||||
group_name: group.name,
|
||||
failure_count: 8,
|
||||
recent_success_count: 0,
|
||||
recent_failure_count: 5,
|
||||
consecutive_failure_count: 5,
|
||||
weight_manual: 50,
|
||||
weight_auto: 0,
|
||||
recovery: { automatic: true, mode: 'validation_probe', at: null },
|
||||
},
|
||||
],
|
||||
request_log: requestLogHealth,
|
||||
}
|
||||
|
||||
const aggregate = {
|
||||
request_count: 6,
|
||||
success_count: 4,
|
||||
failure_count: 2,
|
||||
uncached_input_tokens: 1200,
|
||||
cache_read_tokens: 300,
|
||||
cache_write_5m_tokens: 120,
|
||||
cache_write_1h_tokens: 80,
|
||||
output_tokens: 500,
|
||||
total_tokens: 2200,
|
||||
estimated_cost_usd: 0.01234,
|
||||
usage_missing_count: 1,
|
||||
partial_count: 1,
|
||||
unpriced_request_count: 2,
|
||||
}
|
||||
|
||||
const usageReport = {
|
||||
range: '24h',
|
||||
granularity: 'hour',
|
||||
timezone: 'UTC',
|
||||
from: '2026-07-28T00:00:00.000Z',
|
||||
to: '2026-07-29T00:00:00.000Z',
|
||||
observed_at: '2026-07-28T23:59:59.000Z',
|
||||
summary: aggregate,
|
||||
series: [
|
||||
{
|
||||
...aggregate,
|
||||
bucket_start: '2026-07-28T23:00:00.000Z',
|
||||
bucket_end: '2026-07-29T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
breakdown: [{ ...aggregate, group_id: group.id, model: visualFixtureData.modelName }],
|
||||
breakdown_truncated: true,
|
||||
collection_health: {
|
||||
scope: 'current_process',
|
||||
dropped_total: 2,
|
||||
write_failure_total: 1,
|
||||
last_write_failure_at: '2026-07-28T23:50:00.000Z',
|
||||
},
|
||||
}
|
||||
|
||||
const settings = {
|
||||
values: {
|
||||
connect_timeout: 15,
|
||||
first_byte_timeout: 120,
|
||||
request_timeout: 600,
|
||||
stream_idle_timeout: 300,
|
||||
header_rules: { set: { 'X-Visual-Fixture': 'stable-value' }, remove: ['X-Legacy'] },
|
||||
inject_usage_options: true,
|
||||
request_log_retention_days: 7,
|
||||
},
|
||||
overrides: [],
|
||||
}
|
||||
|
||||
const modelPrices = {
|
||||
price_unit: 'usd_per_million_tokens',
|
||||
builtin: [
|
||||
{
|
||||
pattern: 'visual-model-*',
|
||||
source: 'builtin',
|
||||
prices: {
|
||||
uncached_input: 1.25,
|
||||
cache_read: 0,
|
||||
cache_write_5m: null,
|
||||
cache_write_1h: 2.5,
|
||||
output: 7.5,
|
||||
},
|
||||
source_url: 'https://visual-fixture.example/pricing',
|
||||
updated_at: visualClock,
|
||||
pricing_policy: {
|
||||
input_threshold_tokens: 272000,
|
||||
input_multiplier: 2,
|
||||
output_multiplier: 1.5,
|
||||
},
|
||||
},
|
||||
],
|
||||
overrides: [
|
||||
{
|
||||
pattern: '*',
|
||||
source: 'user',
|
||||
prices: {
|
||||
uncached_input: 2,
|
||||
cache_read: 0,
|
||||
cache_write_5m: null,
|
||||
cache_write_1h: null,
|
||||
output: 8,
|
||||
},
|
||||
source_url: null,
|
||||
updated_at: visualClock,
|
||||
pricing_policy: null,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const requestLogPage = {
|
||||
items: [
|
||||
{
|
||||
request_id: visualFixtureData.requestId,
|
||||
completed_at: visualClock,
|
||||
access_key: { id: accessKey.id, name: accessKey.name, deleted: false },
|
||||
protocol: 'openai',
|
||||
client_model: 'stable-public-alias',
|
||||
upstream_model: visualFixtureData.modelName,
|
||||
status: 'error',
|
||||
status_code: 429,
|
||||
duration_ms: 842,
|
||||
error_code: 'RATE_LIMITED',
|
||||
error_summary: 'Fixture upstream rate limit',
|
||||
affinity_hit: false,
|
||||
attempts: [
|
||||
{
|
||||
sequence: 1,
|
||||
group_id: group.id,
|
||||
group_name: group.name,
|
||||
key_id: 91,
|
||||
key_mask: 'sk-v****1234',
|
||||
upstream_model: visualFixtureData.modelName,
|
||||
status_code: 429,
|
||||
duration_ms: 842,
|
||||
failure_category: 'rate_limited',
|
||||
action: 'cooldown_key',
|
||||
will_retry: false,
|
||||
error_code: 'RATE_LIMITED',
|
||||
error_summary: 'Fixture upstream rate limit',
|
||||
committed: true,
|
||||
},
|
||||
],
|
||||
group_id: group.id,
|
||||
usage_state: 'missing',
|
||||
cost_state: 'unpriced',
|
||||
uncached_input_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_5m_tokens: 0,
|
||||
cache_write_1h_tokens: 0,
|
||||
output_tokens: 0,
|
||||
estimated_cost_usd: 0,
|
||||
},
|
||||
],
|
||||
next_cursor: null,
|
||||
}
|
||||
|
||||
const routeInspection = {
|
||||
observed_at: visualClock,
|
||||
snapshot_revision: 42,
|
||||
protocol: 'openai',
|
||||
external_model: 'stable-public-alias',
|
||||
access_key: { id: accessKey.id, name: accessKey.name, status: 'active' },
|
||||
routable: true,
|
||||
reason_code: null,
|
||||
groups: [
|
||||
{
|
||||
group_id: group.id,
|
||||
group_name: group.name,
|
||||
upstream_model: visualFixtureData.modelName,
|
||||
weight_manual: null,
|
||||
included: true,
|
||||
routable: true,
|
||||
reason_code: null,
|
||||
keys: [
|
||||
{
|
||||
key_id: 91,
|
||||
available: true,
|
||||
reason_code: null,
|
||||
weight_manual: null,
|
||||
weight_auto: 72,
|
||||
effective_weight: 5184,
|
||||
cooldown_until: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
function success(data: unknown, headers: Record<string, string> = {}) {
|
||||
return {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json; charset=utf-8', ...headers },
|
||||
body: JSON.stringify({ code: 0, message: 'OK', data }),
|
||||
}
|
||||
}
|
||||
|
||||
function failure() {
|
||||
return {
|
||||
status: 503,
|
||||
headers: { 'content-type': 'application/json; charset=utf-8' },
|
||||
body: JSON.stringify({
|
||||
code: 'SERVICE_UNAVAILABLE',
|
||||
message: 'Fixture service unavailable',
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export interface VisualApiState {
|
||||
healthRequests: number
|
||||
unexpectedRequests: string[]
|
||||
}
|
||||
|
||||
export async function installVisualApi(
|
||||
page: Page,
|
||||
scenario: VisualScenario,
|
||||
): Promise<VisualApiState> {
|
||||
const state: VisualApiState = { healthRequests: 0, unexpectedRequests: [] }
|
||||
await page.route('**/api/**', async (route: Route) => {
|
||||
const request = route.request()
|
||||
const url = new URL(request.url())
|
||||
const method = request.method()
|
||||
|
||||
if (url.pathname === '/api/auth/session') {
|
||||
await route.fulfill(success({ authenticated: true }))
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/groups' && method === 'GET') {
|
||||
await route.fulfill(success(scenario === 'home-empty-error' ? [] : [group]))
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/access-keys/options' && method === 'GET') {
|
||||
await route.fulfill(
|
||||
success(
|
||||
scenario === 'home-empty-error'
|
||||
? []
|
||||
: [{ id: accessKey.id, name: accessKey.name, status: accessKey.status }],
|
||||
),
|
||||
)
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/access-keys' && method === 'GET') {
|
||||
await route.fulfill(success([accessKey]))
|
||||
return
|
||||
}
|
||||
if (
|
||||
url.pathname === '/api/access-keys' &&
|
||||
method === 'POST' &&
|
||||
scenario === 'access-key-operation'
|
||||
) {
|
||||
await route.abort('connectionreset')
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/health' && method === 'GET') {
|
||||
state.healthRequests += 1
|
||||
if (
|
||||
scenario === 'home-empty-error' ||
|
||||
(scenario === 'home-anomaly' && state.healthRequests > 1)
|
||||
) {
|
||||
await route.fulfill(failure())
|
||||
return
|
||||
}
|
||||
await route.fulfill(success(scenario === 'home-anomaly' ? anomalyHealth : normalHealth))
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/usage' && method === 'GET') {
|
||||
await route.fulfill(scenario === 'home-empty-error' ? failure() : success(usageReport))
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/logs' && method === 'GET') {
|
||||
await route.fulfill(success(requestLogPage))
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/settings' && method === 'GET') {
|
||||
await route.fulfill(
|
||||
success(settings, {
|
||||
etag: '"sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"',
|
||||
}),
|
||||
)
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/system/info' && method === 'GET') {
|
||||
await route.fulfill(
|
||||
success({
|
||||
version: '2.0.0-visual',
|
||||
deployment: {
|
||||
instance_mode: 'single',
|
||||
database: 'sqlite',
|
||||
distribution: 'single_binary',
|
||||
},
|
||||
data_dir: '/visual/data',
|
||||
auth_key: { source: 'environment', path: null },
|
||||
encryption: { enabled: true, source: 'environment', path: null },
|
||||
}),
|
||||
)
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/model-prices' && method === 'GET') {
|
||||
await route.fulfill(success(modelPrices))
|
||||
return
|
||||
}
|
||||
if (url.pathname === '/api/route/inspect' && method === 'POST') {
|
||||
await route.fulfill(success(routeInspection))
|
||||
return
|
||||
}
|
||||
|
||||
const identity = `${method} ${url.pathname}${url.search}`
|
||||
state.unexpectedRequests.push(identity)
|
||||
await route.fulfill({
|
||||
status: 500,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
code: 'VISUAL_FIXTURE_UNEXPECTED_REQUEST',
|
||||
message: identity,
|
||||
}),
|
||||
})
|
||||
})
|
||||
return state
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdir, writeFile } from 'node:fs/promises'
|
||||
import { relative, resolve, sep } from 'node:path'
|
||||
|
||||
import type { Page, TestInfo } from '@playwright/test'
|
||||
|
||||
import { visualClock, visualFixtureVersion, type VisualScenarioCase } from './visual-fixtures'
|
||||
|
||||
interface Rect {
|
||||
x: number
|
||||
y: number
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
interface VisualGeometry {
|
||||
viewport: { width: number; height: number }
|
||||
document: { width: number; height: number }
|
||||
main: Rect
|
||||
heading: Rect
|
||||
horizontalOverflow: number
|
||||
}
|
||||
|
||||
export interface VisualCapture {
|
||||
id: string
|
||||
scenario: VisualScenarioCase['scenario']
|
||||
path: string
|
||||
locale: VisualScenarioCase['locale']
|
||||
theme: VisualScenarioCase['theme']
|
||||
viewport: VisualScenarioCase['viewport']
|
||||
screenshot: {
|
||||
path: string
|
||||
sha256: string
|
||||
}
|
||||
geometry: VisualGeometry
|
||||
}
|
||||
|
||||
function normalizePath(value: string): string {
|
||||
return value.split(sep).join('/')
|
||||
}
|
||||
|
||||
function artifactRoot(): string {
|
||||
return resolve(process.cwd(), process.env.GPT_LOAD_E2E_ARTIFACT_DIR ?? 'test-results/unscoped')
|
||||
}
|
||||
|
||||
export async function captureVisualScenario(
|
||||
page: Page,
|
||||
testInfo: TestInfo,
|
||||
scenarioCase: VisualScenarioCase,
|
||||
): Promise<VisualCapture> {
|
||||
const geometry = await page.evaluate(() => {
|
||||
const main = document.querySelector('main')
|
||||
const heading = main?.querySelector('h1')
|
||||
if (!(main instanceof HTMLElement) || !(heading instanceof HTMLElement)) {
|
||||
throw new Error('Visual scenario requires one rendered main heading')
|
||||
}
|
||||
const mainRect = main.getBoundingClientRect()
|
||||
const headingRect = heading.getBoundingClientRect()
|
||||
const round = (value: number) => Math.round(value * 1_000) / 1_000
|
||||
const rect = (value: DOMRect): Rect => ({
|
||||
x: round(value.x),
|
||||
y: round(value.y),
|
||||
width: round(value.width),
|
||||
height: round(value.height),
|
||||
})
|
||||
return {
|
||||
viewport: {
|
||||
width: document.documentElement.clientWidth,
|
||||
height: document.documentElement.clientHeight,
|
||||
},
|
||||
document: {
|
||||
width: document.documentElement.scrollWidth,
|
||||
height: document.documentElement.scrollHeight,
|
||||
},
|
||||
main: rect(mainRect),
|
||||
heading: rect(headingRect),
|
||||
horizontalOverflow: Math.max(
|
||||
0,
|
||||
document.documentElement.scrollWidth - document.documentElement.clientWidth,
|
||||
),
|
||||
}
|
||||
})
|
||||
|
||||
if (
|
||||
geometry.viewport.width !== scenarioCase.viewport.width ||
|
||||
geometry.viewport.height !== scenarioCase.viewport.height
|
||||
) {
|
||||
throw new Error(`Visual viewport drifted for ${scenarioCase.id}`)
|
||||
}
|
||||
if (geometry.horizontalOverflow !== 0) {
|
||||
throw new Error(`Body overflowed horizontally for ${scenarioCase.id}`)
|
||||
}
|
||||
if (
|
||||
geometry.heading.width <= 0 ||
|
||||
geometry.heading.height <= 0 ||
|
||||
geometry.heading.x < 0 ||
|
||||
geometry.heading.x + geometry.heading.width > geometry.viewport.width
|
||||
) {
|
||||
throw new Error(`Main heading geometry is invalid for ${scenarioCase.id}`)
|
||||
}
|
||||
|
||||
const screenshotPath = testInfo.outputPath('candidate.png')
|
||||
const screenshot = await page.screenshot({
|
||||
path: screenshotPath,
|
||||
fullPage: true,
|
||||
animations: 'disabled',
|
||||
caret: 'hide',
|
||||
})
|
||||
return {
|
||||
id: scenarioCase.id,
|
||||
scenario: scenarioCase.scenario,
|
||||
path: scenarioCase.path,
|
||||
locale: scenarioCase.locale,
|
||||
theme: scenarioCase.theme,
|
||||
viewport: scenarioCase.viewport,
|
||||
screenshot: {
|
||||
path: normalizePath(relative(artifactRoot(), screenshotPath)),
|
||||
sha256: createHash('sha256').update(screenshot).digest('hex'),
|
||||
},
|
||||
geometry,
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeVisualScenarioManifest(captures: VisualCapture[]): Promise<void> {
|
||||
const sortedCaptures = [...captures].sort((left, right) => left.id.localeCompare(right.id))
|
||||
const payload = {
|
||||
schema_version: 1,
|
||||
fixture_version: visualFixtureVersion,
|
||||
clock: visualClock,
|
||||
captures: sortedCaptures,
|
||||
}
|
||||
const manifest = {
|
||||
...payload,
|
||||
manifest_sha256: createHash('sha256').update(JSON.stringify(payload)).digest('hex'),
|
||||
}
|
||||
const root = artifactRoot()
|
||||
await mkdir(root, { recursive: true })
|
||||
await writeFile(
|
||||
resolve(root, 'scenario-manifest.json'),
|
||||
`${JSON.stringify(manifest, null, 2)}\n`,
|
||||
'utf8',
|
||||
)
|
||||
}
|
||||
@@ -7,23 +7,28 @@ import {
|
||||
visualClock,
|
||||
visualFixtureData,
|
||||
visualLocales,
|
||||
visualScenarioCases,
|
||||
visualScenarios,
|
||||
visualThemes,
|
||||
visualViewports,
|
||||
} from './visual-fixtures'
|
||||
|
||||
const businessFlowSource = readFileSync(resolve('e2e/business-flows.spec.ts'), 'utf8')
|
||||
const visualScenarioSource = readFileSync(resolve('e2e/visual-scenarios.spec.ts'), 'utf8')
|
||||
|
||||
describe('deterministic visual fixtures', () => {
|
||||
it('freezes the approved scenario, viewport, theme, and locale matrix', () => {
|
||||
expect(visualScenarios).toEqual([
|
||||
'home-normal',
|
||||
'home-anomaly',
|
||||
'home-empty-error',
|
||||
'access-keys-long',
|
||||
'access-key-operation',
|
||||
'model-prices-mixed',
|
||||
'settings-dirty',
|
||||
'settings-validation',
|
||||
'usage-quality',
|
||||
'logs-signal-path',
|
||||
'inspector-routing',
|
||||
])
|
||||
expect(visualViewports).toEqual([
|
||||
{ width: 375, height: 812 },
|
||||
@@ -48,10 +53,21 @@ describe('deterministic visual fixtures', () => {
|
||||
expect(serialized).not.toContain('Bearer ')
|
||||
})
|
||||
|
||||
it('binds every named scenario and locale to the executable browser journey', () => {
|
||||
it('binds every scenario and each approved render dimension to executable cases', () => {
|
||||
for (const scenario of visualScenarios) {
|
||||
expect(businessFlowSource).toContain(`visualScenarioLabel('${scenario}')`)
|
||||
expect(visualScenarioCases.some((candidate) => candidate.scenario === scenario)).toBe(true)
|
||||
}
|
||||
expect(businessFlowSource).toContain('for (const locale of visualLocales)')
|
||||
for (const viewport of visualViewports) {
|
||||
expect(visualScenarioCases.some((candidate) => candidate.viewport === viewport)).toBe(true)
|
||||
}
|
||||
for (const locale of visualLocales) {
|
||||
expect(visualScenarioCases.some((candidate) => candidate.locale === locale)).toBe(true)
|
||||
}
|
||||
for (const theme of visualThemes) {
|
||||
expect(visualScenarioCases.some((candidate) => candidate.theme === theme)).toBe(true)
|
||||
}
|
||||
|
||||
expect(visualScenarioSource).toContain('installVisualApi')
|
||||
expect(visualScenarioSource).toContain('captureVisualScenario')
|
||||
})
|
||||
})
|
||||
@@ -4,11 +4,15 @@ export const visualClock = '2026-07-29T00:00:00.000Z'
|
||||
export const visualScenarios = [
|
||||
'home-normal',
|
||||
'home-anomaly',
|
||||
'home-empty-error',
|
||||
'access-keys-long',
|
||||
'access-key-operation',
|
||||
'model-prices-mixed',
|
||||
'settings-dirty',
|
||||
'settings-validation',
|
||||
'usage-quality',
|
||||
'logs-signal-path',
|
||||
'inspector-routing',
|
||||
] as const
|
||||
|
||||
export type VisualScenario = (typeof visualScenarios)[number]
|
||||
@@ -16,11 +20,15 @@ export type VisualScenario = (typeof visualScenarios)[number]
|
||||
const visualScenarioDescriptions: Record<VisualScenario, string> = {
|
||||
'home-normal': 'Home normal operation',
|
||||
'home-anomaly': 'Home usage and health anomaly',
|
||||
'home-empty-error': 'Home empty and unavailable resources',
|
||||
'access-keys-long': 'AccessKey long record',
|
||||
'access-key-operation': 'AccessKey indeterminate operation notice',
|
||||
'model-prices-mixed': 'Model prices mixed values',
|
||||
'settings-dirty': 'Settings dirty state',
|
||||
'settings-validation': 'Settings validation summary',
|
||||
'usage-quality': 'Usage quality signals',
|
||||
'logs-signal-path': 'Logs signal path',
|
||||
'inspector-routing': 'Route inspector result',
|
||||
}
|
||||
|
||||
export function visualScenarioLabel(scenario: VisualScenario): string {
|
||||
@@ -37,6 +45,146 @@ export const visualViewports = [
|
||||
export const visualThemes = ['light', 'dark'] as const
|
||||
export const visualLocales = ['en-US', 'zh-CN'] as const
|
||||
|
||||
export interface VisualScenarioCase {
|
||||
id: string
|
||||
scenario: VisualScenario
|
||||
path: string
|
||||
viewport: (typeof visualViewports)[number]
|
||||
theme: (typeof visualThemes)[number]
|
||||
locale: (typeof visualLocales)[number]
|
||||
}
|
||||
|
||||
export const visualScenarioCases: readonly VisualScenarioCase[] = [
|
||||
{
|
||||
id: 'home-normal-desktop-en-light',
|
||||
scenario: 'home-normal',
|
||||
path: '/',
|
||||
viewport: visualViewports[3],
|
||||
theme: 'light',
|
||||
locale: 'en-US',
|
||||
},
|
||||
{
|
||||
id: 'home-normal-mobile-zh-dark',
|
||||
scenario: 'home-normal',
|
||||
path: '/',
|
||||
viewport: visualViewports[0],
|
||||
theme: 'dark',
|
||||
locale: 'zh-CN',
|
||||
},
|
||||
{
|
||||
id: 'home-anomaly-tablet-en-dark',
|
||||
scenario: 'home-anomaly',
|
||||
path: '/',
|
||||
viewport: visualViewports[2],
|
||||
theme: 'dark',
|
||||
locale: 'en-US',
|
||||
},
|
||||
{
|
||||
id: 'home-empty-error-tablet-zh-light',
|
||||
scenario: 'home-empty-error',
|
||||
path: '/',
|
||||
viewport: visualViewports[1],
|
||||
theme: 'light',
|
||||
locale: 'zh-CN',
|
||||
},
|
||||
{
|
||||
id: 'access-keys-long-desktop-en-light',
|
||||
scenario: 'access-keys-long',
|
||||
path: '/access-keys',
|
||||
viewport: visualViewports[3],
|
||||
theme: 'light',
|
||||
locale: 'en-US',
|
||||
},
|
||||
{
|
||||
id: 'access-keys-long-mobile-zh-dark',
|
||||
scenario: 'access-keys-long',
|
||||
path: '/access-keys',
|
||||
viewport: visualViewports[0],
|
||||
theme: 'dark',
|
||||
locale: 'zh-CN',
|
||||
},
|
||||
{
|
||||
id: 'access-key-operation-tablet-en-light',
|
||||
scenario: 'access-key-operation',
|
||||
path: '/access-keys',
|
||||
viewport: visualViewports[1],
|
||||
theme: 'light',
|
||||
locale: 'en-US',
|
||||
},
|
||||
{
|
||||
id: 'model-prices-tablet-zh-dark',
|
||||
scenario: 'model-prices-mixed',
|
||||
path: '/settings/model-prices',
|
||||
viewport: visualViewports[2],
|
||||
theme: 'dark',
|
||||
locale: 'zh-CN',
|
||||
},
|
||||
{
|
||||
id: 'settings-dirty-desktop-en-light',
|
||||
scenario: 'settings-dirty',
|
||||
path: '/settings',
|
||||
viewport: visualViewports[3],
|
||||
theme: 'light',
|
||||
locale: 'en-US',
|
||||
},
|
||||
{
|
||||
id: 'settings-validation-mobile-zh-light',
|
||||
scenario: 'settings-validation',
|
||||
path: '/settings',
|
||||
viewport: visualViewports[0],
|
||||
theme: 'light',
|
||||
locale: 'zh-CN',
|
||||
},
|
||||
{
|
||||
id: 'usage-quality-desktop-en-dark',
|
||||
scenario: 'usage-quality',
|
||||
path: '/monitor?tab=usage&range=24h',
|
||||
viewport: visualViewports[3],
|
||||
theme: 'dark',
|
||||
locale: 'en-US',
|
||||
},
|
||||
{
|
||||
id: 'usage-quality-tablet-zh-light',
|
||||
scenario: 'usage-quality',
|
||||
path: '/monitor?tab=usage&range=24h',
|
||||
viewport: visualViewports[1],
|
||||
theme: 'light',
|
||||
locale: 'zh-CN',
|
||||
},
|
||||
{
|
||||
id: 'logs-signal-tablet-en-light',
|
||||
scenario: 'logs-signal-path',
|
||||
path: '/monitor?tab=logs',
|
||||
viewport: visualViewports[2],
|
||||
theme: 'light',
|
||||
locale: 'en-US',
|
||||
},
|
||||
{
|
||||
id: 'logs-signal-mobile-zh-dark',
|
||||
scenario: 'logs-signal-path',
|
||||
path: '/monitor?tab=logs',
|
||||
viewport: visualViewports[0],
|
||||
theme: 'dark',
|
||||
locale: 'zh-CN',
|
||||
},
|
||||
{
|
||||
id: 'inspector-routing-desktop-en-light',
|
||||
scenario: 'inspector-routing',
|
||||
path: '/monitor?tab=inspector&protocol=openai&external_model=stable-public-alias&access_key_id=12',
|
||||
viewport: visualViewports[3],
|
||||
theme: 'light',
|
||||
locale: 'en-US',
|
||||
},
|
||||
{
|
||||
id: 'inspector-routing-tablet-zh-dark',
|
||||
scenario: 'inspector-routing',
|
||||
path: '/monitor?tab=inspector&protocol=openai&external_model=stable-public-alias&access_key_id=12',
|
||||
viewport: visualViewports[1],
|
||||
theme: 'dark',
|
||||
locale: 'zh-CN',
|
||||
},
|
||||
]
|
||||
|
||||
export const visualFixtureData = Object.freeze({
|
||||
requestId: '5297f18d-3ca7-42d9-8a5b-2ce08ccd1b01',
|
||||
accessKeySuffix: 'a11e',
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import { expect, test } from './fixtures'
|
||||
import { installVisualApi, visualAuthKey } from './visual-api'
|
||||
import {
|
||||
captureVisualScenario,
|
||||
writeVisualScenarioManifest,
|
||||
type VisualCapture,
|
||||
} from './visual-capture'
|
||||
import {
|
||||
visualFixtureData,
|
||||
visualScenarioCases,
|
||||
visualScenarioLabel,
|
||||
type VisualScenarioCase,
|
||||
} from './visual-fixtures'
|
||||
|
||||
const captures: VisualCapture[] = []
|
||||
|
||||
test.setTimeout(60_000)
|
||||
test.afterAll(async () => {
|
||||
await writeVisualScenarioManifest(captures)
|
||||
})
|
||||
|
||||
async function prepareScenario(
|
||||
page: Parameters<typeof installVisualApi>[0],
|
||||
item: VisualScenarioCase,
|
||||
) {
|
||||
if (item.scenario === 'home-normal') {
|
||||
await expect(page.locator('[data-test="home-health-total"]')).toBeVisible()
|
||||
await expect(page.locator('[data-test="home-usage-requests"]')).toBeVisible()
|
||||
await expect(page.getByText(visualFixtureData.groupName, { exact: true })).toBeVisible()
|
||||
return
|
||||
}
|
||||
|
||||
if (item.scenario === 'home-anomaly') {
|
||||
await expect(page.locator('[data-test="home-health-cooldown"]')).toHaveAttribute(
|
||||
'data-state',
|
||||
'anomaly',
|
||||
)
|
||||
const desktopSettings = page.locator('.desktop-nav a[href="/settings"]')
|
||||
if (await desktopSettings.isVisible()) {
|
||||
await desktopSettings.click()
|
||||
} else {
|
||||
await page.locator('.mobile-menu-trigger').click()
|
||||
await page.locator('.mobile-nav a[href="/settings"]').click()
|
||||
}
|
||||
await expect(page).toHaveURL('/settings')
|
||||
await page.locator('a.brand').click()
|
||||
await expect(page.locator('[data-test="home-operational-overview"]')).toBeVisible()
|
||||
await expect(page.getByText(/stale|旧数据/).first()).toBeVisible()
|
||||
return
|
||||
}
|
||||
|
||||
if (item.scenario === 'home-empty-error') {
|
||||
await expect(page.locator('[data-test="home-usage-error"]')).toBeVisible()
|
||||
await expect(page.locator('[data-test="home-groups"]')).toBeVisible()
|
||||
return
|
||||
}
|
||||
|
||||
if (item.scenario === 'access-keys-long') {
|
||||
await expect(page.getByText(visualFixtureData.longIdentifier, { exact: true })).toBeVisible()
|
||||
return
|
||||
}
|
||||
|
||||
if (item.scenario === 'access-key-operation') {
|
||||
await page.locator('[data-test="access-key-create"]').click()
|
||||
const drawer = page.getByRole('dialog')
|
||||
await expect(drawer).toBeVisible()
|
||||
await drawer.locator('[data-test="access-key-name"]').fill('Visual pending AccessKey')
|
||||
await drawer.locator('[data-test="access-key-save"]').click()
|
||||
await expect(drawer.getByText(/unknown|未确认/).first()).toBeVisible()
|
||||
await drawer.locator('.app-drawer__close').click()
|
||||
await expect(page.locator('[data-test="access-key-operation-notice"]')).toBeVisible()
|
||||
return
|
||||
}
|
||||
|
||||
if (item.scenario === 'model-prices-mixed') {
|
||||
await expect(page.locator('[data-test="builtin-price-row-0"]')).toBeVisible()
|
||||
await expect(page.locator('[data-test="override-price-row-0"]')).toBeVisible()
|
||||
return
|
||||
}
|
||||
|
||||
if (item.scenario === 'settings-dirty' || item.scenario === 'settings-validation') {
|
||||
await page.locator('[data-test="override-request_timeout"]').check()
|
||||
await page
|
||||
.locator('[data-test="value-request_timeout"]')
|
||||
.fill(item.scenario === 'settings-dirty' ? '901' : '0')
|
||||
await expect(
|
||||
page.locator(
|
||||
item.scenario === 'settings-dirty'
|
||||
? '[data-test="settings-dirty-summary"]'
|
||||
: '[data-test="settings-validation-summary"]',
|
||||
),
|
||||
).toBeVisible()
|
||||
return
|
||||
}
|
||||
|
||||
if (item.scenario === 'usage-quality') {
|
||||
await expect(page.locator('[data-test="usage-window-quality"]')).toBeVisible()
|
||||
await expect(page.locator('[data-test="usage-process-health"]')).toBeVisible()
|
||||
return
|
||||
}
|
||||
|
||||
if (item.scenario === 'logs-signal-path') {
|
||||
await expect(page.locator(`[data-test="log-row-${visualFixtureData.requestId}"]`)).toBeVisible()
|
||||
return
|
||||
}
|
||||
|
||||
if (item.scenario === 'inspector-routing') {
|
||||
await page.locator('[data-test="inspector-submit"]').click()
|
||||
await expect(page.locator('[data-test="inspector-result"]')).toBeVisible()
|
||||
}
|
||||
}
|
||||
|
||||
for (const item of visualScenarioCases) {
|
||||
test(`${visualScenarioLabel(item.scenario)} · ${item.id}`, async ({ page }, testInfo) => {
|
||||
await page.setViewportSize(item.viewport)
|
||||
await page.emulateMedia({
|
||||
colorScheme: item.theme,
|
||||
reducedMotion: 'reduce',
|
||||
})
|
||||
await page.addInitScript(
|
||||
({ authKey, locale, theme }) => {
|
||||
window.sessionStorage.setItem('gpt-load.auth-key', authKey)
|
||||
window.localStorage.setItem('gpt-load.locale', locale)
|
||||
window.localStorage.setItem('gpt-load.theme', theme)
|
||||
},
|
||||
{ authKey: visualAuthKey, locale: item.locale, theme: item.theme },
|
||||
)
|
||||
const api = await installVisualApi(page, item.scenario)
|
||||
|
||||
await page.goto(item.path)
|
||||
await expect(page.locator('main h1')).toHaveCount(1)
|
||||
await prepareScenario(page, item)
|
||||
await expect(page.locator('.query-feedback--loading')).toHaveCount(0)
|
||||
await page.evaluate(async () => {
|
||||
await document.fonts.ready
|
||||
await new Promise<void>((resolveFrame) => requestAnimationFrame(() => resolveFrame()))
|
||||
await new Promise<void>((resolveFrame) => requestAnimationFrame(() => resolveFrame()))
|
||||
})
|
||||
expect(api.unexpectedRequests).toEqual([])
|
||||
|
||||
captures.push(await captureVisualScenario(page, testInfo, item))
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user