import { createHash } from 'node:crypto' import { execFile } from 'node:child_process' import { access, cp, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { basename, dirname, join, resolve } from 'node:path' import { promisify } from 'node:util' import { fileURLToPath } from 'node:url' import { loadVisualRunnerContract } from './check-visual-runner-lock.mjs' const executeFile = promisify(execFile) const webRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') const repositoryRoot = resolve(webRoot, '..') const runnerTemporaryPrefix = 'gpt-load-visual-runner-' export const candidateArtifactRetention = { local_only: ['artifacts/'], versioned: ['candidate.json', 'inspection.md'], } function fail(message) { throw new Error(`visual runner: ${message}`) } function selectedPlatform() { if (process.arch === 'arm64') return 'linux/arm64' if (process.arch === 'x64') return 'linux/amd64' fail(`unsupported host architecture ${process.arch}`) } export function parseVisualRunnerInvocation(arguments_) { const normalizedArguments = arguments_.filter((argument) => argument !== '--') const supportedModes = ['dry-run', 'verify', 'candidate', 'functional'] const modes = normalizedArguments .filter((argument) => argument.startsWith('--') && !argument.startsWith('--browser=')) .map((argument) => argument.slice(2)) .filter((argument) => supportedModes.includes(argument)) const unknown = normalizedArguments.filter( (argument) => !supportedModes.some((mode) => argument === `--${mode}`) && !argument.startsWith('--browser='), ) if (unknown.length > 0) fail(`unknown option ${unknown[0]}`) if (modes.length !== 1) { fail('choose exactly one of --dry-run, --verify, --candidate, or --functional') } const browserArguments = normalizedArguments .filter((argument) => argument.startsWith('--browser=')) .map((argument) => argument.slice('--browser='.length)) if (browserArguments.length > 1) fail('choose at most one browser') const browser = browserArguments[0] if (browser !== undefined && browser !== 'chromium' && browser !== 'webkit') { fail(`unsupported browser ${browser}`) } if (modes[0] === 'candidate' && browser !== 'chromium') { fail('candidate capture requires Chromium') } if (modes[0] === 'functional' && browser === undefined) { fail('functional execution requires an explicit browser') } if (modes[0] === 'verify' && browser !== undefined) { fail('verify does not accept a browser') } return { mode: modes[0], ...(browser === undefined ? {} : { browser }), } } function fingerprintScript(lock) { return String.raw` const { createHash } = require('node:crypto') const { execFileSync } = require('node:child_process') const { existsSync } = require('node:fs') const fonts = execFileSync('bash', ['-lc', 'fc-list --format "%{file}|%{family}|%{style}\\n" | LC_ALL=C sort -u' ]) const fontLines = fonts.toString('utf8').trimEnd().split('\n').filter(Boolean) const result = { runtime: { container_node: process.versions.node, zlib: process.versions.zlib, brotli: process.versions.brotli, platform: process.platform, architecture: process.arch, }, browsers: { chromium: { revision: ${JSON.stringify(lock.browsers.chromium.revision)}, present: existsSync(${JSON.stringify( `/ms-playwright/chromium-${lock.browsers.chromium.revision}`, )}), }, webkit: { revision: ${JSON.stringify(lock.browsers.webkit.revision)}, present: existsSync(${JSON.stringify(`/ms-playwright/webkit-${lock.browsers.webkit.revision}`)}), }, }, fonts: { count: fontLines.length, sha256: createHash('sha256').update(fonts).digest('hex'), }, } process.stdout.write(JSON.stringify(result)) ` } function findPlatformDigest(rawManifest, platform) { const [os, architecture] = platform.split('/') const manifest = JSON.parse(rawManifest) const match = manifest.manifests?.find( (candidate) => candidate.platform?.os === os && candidate.platform?.architecture === architecture, ) if (!match?.digest) fail(`image manifest is missing ${platform}`) return match.digest } async function runFile(command, arguments_, options = {}) { return executeFile(command, arguments_, { maxBuffer: 64 * 1024 * 1024, ...options, }) } async function verifyContainer(contract, platform) { const imageManifest = await runFile( 'docker', ['buildx', 'imagetools', 'inspect', '--raw', contract.lock.container.image], { encoding: 'utf8' }, ) const actualPlatformDigest = findPlatformDigest(imageManifest.stdout, platform) const expectedPlatformDigest = contract.lock.container.platform_digests[platform] if (actualPlatformDigest !== expectedPlatformDigest) { fail( `${platform} digest mismatch: expected ${expectedPlatformDigest}, received ${actualPlatformDigest}`, ) } const command = [ 'docker', 'run', '--rm', '--platform', platform, contract.lock.container.image, 'node', '-e', fingerprintScript(contract.lock), ] const fingerprintResult = await runFile(command[0], command.slice(1), { encoding: 'utf8' }) const fingerprint = JSON.parse(fingerprintResult.stdout) const expectedArchitecture = platform.endsWith('/arm64') ? 'arm64' : 'x64' const expectedRuntime = { container_node: contract.lock.runtime.container_node, zlib: contract.lock.runtime.zlib, brotli: contract.lock.runtime.brotli, platform: 'linux', architecture: expectedArchitecture, } if (JSON.stringify(fingerprint.runtime) !== JSON.stringify(expectedRuntime)) { fail(`container runtime mismatch: ${JSON.stringify(fingerprint.runtime)}`) } for (const browser of ['chromium', 'webkit']) { if ( fingerprint.browsers[browser].revision !== contract.lock.browsers[browser].revision || fingerprint.browsers[browser].present !== true ) { fail(`${browser} binary fingerprint mismatch`) } } if (JSON.stringify(fingerprint.fonts) !== JSON.stringify(contract.lock.fonts)) { fail(`font inventory mismatch: ${JSON.stringify(fingerprint.fonts)}`) } return { command, fingerprint, actualPlatformDigest } } async function gitValue(arguments_) { const result = await runFile('git', arguments_, { cwd: repositoryRoot, encoding: 'utf8' }) return result.stdout.trim() } async function writeRunnerEvidence(evidence, name = 'visual-runner') { const path = join(webRoot, 'test-results', `${name}.json`) await mkdir(dirname(path), { recursive: true }) await writeFile(path, `${JSON.stringify(evidence, null, 2)}\n`) return path } function sha256(value) { return createHash('sha256').update(value).digest('hex') } async function createSourceStage() { const temporaryRoot = await mkdtemp(join(tmpdir(), runnerTemporaryPrefix)) const archivePath = join(temporaryRoot, 'source.tar') const workspace = join(temporaryRoot, 'workspace') await mkdir(workspace) const archive = await runFile('git', ['archive', '--format=tar', 'HEAD'], { cwd: repositoryRoot, encoding: 'buffer', }) await writeFile(archivePath, archive.stdout) await runFile('tar', ['-xf', archivePath, '-C', workspace]) return { temporaryRoot, workspace } } async function buildLinuxApplication(workspace, platform) { await runFile('corepack', ['pnpm', '--dir', webRoot, 'run', 'build'], { cwd: repositoryRoot, encoding: 'utf8', }) const architecture = platform.endsWith('/arm64') ? 'arm64' : 'amd64' await runFile('go', ['build', '-trimpath', '-o', join(workspace, 'gpt-load'), '.'], { cwd: repositoryRoot, encoding: 'utf8', env: { ...process.env, CGO_ENABLED: '0', GOOS: 'linux', GOARCH: architecture, }, }) } async function runContainerSuite(contract, platform, workspace, browser, specs, sourceSHA) { const command = [ 'docker', 'run', '--rm', '--platform', platform, '--env', 'CI=1', '--env', 'PLAYWRIGHT_BROWSERS_PATH=/ms-playwright', '--env', 'GPT_LOAD_E2E_APP_PORT=43101', ...(sourceSHA ? ['--env', `GPT_LOAD_E2E_SOURCE_SHA=${sourceSHA}`] : []), '--volume', `${workspace}:/workspace`, '--workdir', '/workspace', contract.lock.container.image, 'bash', '-lc', [ `corepack install --global ${contract.lock.runtime.package_manager}`, 'corepack pnpm --dir /workspace/web install --frozen-lockfile', `corepack pnpm --dir /workspace/web run test:e2e ${specs.join(' ')} --project=${browser}`, ].join(' && '), ] const result = await runFile(command[0], command.slice(1), { encoding: 'utf8' }) return { command, stdout: result.stdout, stderr: result.stderr } } async function validateScenarioArtifacts(artifactRoot) { const manifestPath = join(artifactRoot, 'scenario-manifest.json') const manifestSource = await readFile(manifestPath, 'utf8') const manifest = JSON.parse(manifestSource) const { manifest_sha256: declaredHash, ...payload } = manifest if (!/^[0-9a-f]{64}$/.test(declaredHash) || sha256(JSON.stringify(payload)) !== declaredHash) { fail('scenario manifest hash is invalid') } if (!Array.isArray(manifest.captures) || manifest.captures.length === 0) { fail('scenario manifest has no captures') } const ids = new Set() for (const capture of manifest.captures) { if (typeof capture.id !== 'string' || ids.has(capture.id)) { fail('scenario manifest contains a duplicate or invalid ID') } ids.add(capture.id) const screenshotPath = resolve(artifactRoot, capture.screenshot?.path ?? '') if (!screenshotPath.startsWith(`${artifactRoot}/`)) fail('scenario artifact escaped its root') const screenshot = await readFile(screenshotPath) if (sha256(screenshot) !== capture.screenshot.sha256) { fail(`screenshot hash mismatch for ${capture.id}`) } } return { manifest, manifestPath, manifestFileSha256: sha256(manifestSource), } } async function publishCandidate({ contract, platform, verification, sourceSHA, artifactRoot, scenarioArtifacts, }) { const candidatesRoot = join(webRoot, 'visual-baselines', 'candidates') const target = join(candidatesRoot, sourceSHA) await access(target).then( () => fail(`candidate ${sourceSHA} already exists; automatic replacement is forbidden`), () => undefined, ) await mkdir(candidatesRoot, { recursive: true }) const pending = join(candidatesRoot, `.pending-${sourceSHA}`) await mkdir(pending) try { await cp(artifactRoot, join(pending, 'artifacts'), { recursive: true }) await rm(join(pending, 'artifacts', '.last-run.json'), { force: true }) const payload = { schema_version: 1, status: 'CANDIDATE', source_sha: sourceSHA, source_dirty: false, lock_sha256: contract.lockSha256, image: contract.lock.container.image, platform, platform_digest: verification.actualPlatformDigest, runtime: verification.fingerprint.runtime, browsers: contract.lock.browsers, fonts: verification.fingerprint.fonts, render_fingerprint: contract.lock.render_fingerprint, scenario_manifest: { path: 'artifacts/scenario-manifest.json', sha256: scenarioArtifacts.manifestFileSha256, declared_sha256: scenarioArtifacts.manifest.manifest_sha256, capture_count: scenarioArtifacts.manifest.captures.length, }, artifacts: scenarioArtifacts.manifest.captures.map((capture) => ({ id: capture.id, path: `artifacts/${capture.screenshot.path}`, sha256: capture.screenshot.sha256, })), human_approval: { status: 'NOT RUN', activated: false, required: true, }, } const metadata = { ...payload, candidate_sha256: sha256(JSON.stringify(payload)), } await writeFile(join(pending, 'candidate.json'), `${JSON.stringify(metadata, null, 2)}\n`) await rename(pending, target) } catch (error) { await rm(pending, { recursive: true, force: true }) throw error } return target } async function runCandidate(contract, platform, verification) { const sourceDirty = (await gitValue(['status', '--porcelain', '--untracked-files=all'])).length > 0 if (sourceDirty) fail('candidate capture requires a clean source tree') const sourceSHA = await gitValue(['rev-parse', 'HEAD']) const stage = await createSourceStage() try { await buildLinuxApplication(stage.workspace, platform) const execution = await runContainerSuite(contract, platform, stage.workspace, 'chromium', [ 'e2e/visual-scenarios.spec.ts', ]) const artifactRoot = join( stage.workspace, 'web', 'test-results', 'chromium-serial-visual-scenarios', ) const scenarioArtifacts = await validateScenarioArtifacts(artifactRoot) const target = await publishCandidate({ contract, platform, verification, sourceSHA, artifactRoot, scenarioArtifacts, }) return { mode: 'candidate', browser: 'chromium', runner_execution_verified: true, source_sha: sourceSHA, source_dirty: false, platform, platform_digest: verification.actualPlatformDigest, scenario_manifest_sha256: scenarioArtifacts.manifestFileSha256, candidate: target, artifact_retention: candidateArtifactRetention, stdout: execution.stdout.trim(), stderr: execution.stderr.trim(), } } finally { if ( dirname(stage.temporaryRoot) !== resolve(tmpdir()) || !basename(stage.temporaryRoot).startsWith(runnerTemporaryPrefix) ) { fail('refusing to remove an unsafe visual runner directory') } await rm(stage.temporaryRoot, { recursive: true, force: true }) } } async function validateFunctionalArtifacts(workspace, browser, sourceSHA) { const artifactRoot = join(workspace, 'web', 'test-results', `${browser}-serial-business-flows`) const resultPath = join(artifactRoot, '.last-run.json') const result = JSON.parse(await readFile(resultPath, 'utf8')) if (result.status !== 'passed' || !Array.isArray(result.failedTests)) { fail(`${browser} functional result is invalid`) } if (result.failedTests.length > 0) { fail(`${browser} functional result contains failed tests`) } const accessibilityJSONPath = join(artifactRoot, 'accessibility-evidence.json') const accessibilityMarkdownPath = join(artifactRoot, 'accessibility-evidence.md') const accessibilityJSONSource = await readFile(accessibilityJSONPath, 'utf8') const accessibilityMarkdownSource = await readFile(accessibilityMarkdownPath, 'utf8') const accessibility = JSON.parse(accessibilityJSONSource) if ( accessibility.status !== 'PASS' || accessibility.source_sha !== sourceSHA || accessibility.source_dirty !== false || accessibility.project !== browser || !Array.isArray(accessibility.results) || accessibility.results.length === 0 || accessibility.results.some((entry) => entry.status !== 'PASS') ) { fail(`${browser} accessibility evidence is invalid`) } return { result, accessibility: { status: accessibility.status, result_count: accessibility.results.length, json_sha256: sha256(accessibilityJSONSource), markdown_sha256: sha256(accessibilityMarkdownSource), }, } } async function runFunctional(contract, platform, verification, browser) { const sourceDirty = (await gitValue(['status', '--porcelain', '--untracked-files=all'])).length > 0 if (sourceDirty) fail('functional execution requires a clean source tree') const sourceSHA = await gitValue(['rev-parse', 'HEAD']) const stage = await createSourceStage() try { await buildLinuxApplication(stage.workspace, platform) const execution = await runContainerSuite( contract, platform, stage.workspace, browser, ['e2e/business-flows.spec.ts', 'e2e/accessibility.spec.ts'], sourceSHA, ) const functionalArtifacts = await validateFunctionalArtifacts( stage.workspace, browser, sourceSHA, ) const payload = { schema_version: 1, mode: 'functional', status: 'PASS', source_sha: sourceSHA, source_dirty: false, lock_sha256: contract.lockSha256, image: contract.lock.container.image, platform, platform_digest: verification.actualPlatformDigest, host_fallback: false, runtime: verification.fingerprint.runtime, browser: { name: browser, ...contract.lock.browsers[browser], }, fonts: verification.fingerprint.fonts, specs: ['e2e/business-flows.spec.ts', 'e2e/accessibility.spec.ts'], result: functionalArtifacts.result, accessibility: functionalArtifacts.accessibility, } const evidence = { ...payload, evidence_sha256: sha256(JSON.stringify(payload)), } const evidencePath = await writeRunnerEvidence(evidence, `visual-runner-functional-${browser}`) return { mode: 'functional', browser, runner_execution_verified: true, source_sha: sourceSHA, source_dirty: false, platform, platform_digest: verification.actualPlatformDigest, evidence: evidencePath, stdout: execution.stdout.trim(), stderr: execution.stderr.trim(), } } finally { if ( dirname(stage.temporaryRoot) !== resolve(tmpdir()) || !basename(stage.temporaryRoot).startsWith(runnerTemporaryPrefix) ) { fail('refusing to remove an unsafe visual runner directory') } await rm(stage.temporaryRoot, { recursive: true, force: true }) } } async function main() { const contract = await loadVisualRunnerContract() const platform = selectedPlatform() const invocation = parseVisualRunnerInvocation(process.argv.slice(2)) const platformDigest = contract.lock.container.platform_digests[platform] const containerCommand = [ 'docker', 'run', '--rm', '--platform', platform, contract.lock.container.image, ] if (invocation.mode === 'dry-run') { process.stdout.write( `${JSON.stringify({ ...invocation, host_fallback: false, container: { image: contract.lock.container.image, platform, platform_digest: platformDigest, }, command: containerCommand, })}\n`, ) return } const verification = await verifyContainer(contract, platform) if (invocation.mode === 'candidate') { process.stdout.write( `${JSON.stringify(await runCandidate(contract, platform, verification))}\n`, ) return } if (invocation.mode === 'functional') { process.stdout.write( `${JSON.stringify( await runFunctional(contract, platform, verification, invocation.browser), )}\n`, ) return } const evidence = { schema_version: 1, mode: invocation.mode, source_sha: await gitValue(['rev-parse', 'HEAD']), source_dirty: (await gitValue(['status', '--porcelain'])).length > 0, lock_sha256: contract.lockSha256, image: contract.lock.container.image, platform, platform_digest: verification.actualPlatformDigest, host_fallback: false, runtime: verification.fingerprint.runtime, browsers: contract.lock.browsers, fonts: verification.fingerprint.fonts, render_fingerprint: contract.lock.render_fingerprint, evidence_sha256: '', } evidence.evidence_sha256 = sha256(JSON.stringify(evidence)) const evidencePath = await writeRunnerEvidence(evidence) process.stdout.write( `${JSON.stringify({ mode: invocation.mode, runner_execution_verified: true, evidence: evidencePath, source_sha: evidence.source_sha, source_dirty: evidence.source_dirty, platform, platform_digest: verification.actualPlatformDigest, })}\n`, ) } if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { await main() }