diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e0508e86..75c82f3d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -57,6 +57,20 @@ jobs: libwayland-dev libxkbcommon-dev libasound2-dev \ mesa-vulkan-drivers libvulkan-dev vulkan-tools glslang-tools + - name: Prepare C++ dependencies outside CodeQL tracing + if: matrix.language == 'cpp' + timeout-minutes: 40 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Third-party sources are pinned and scanned in their own upstreams. + # Build them before CodeQL initialization so the database contains + # EVEngine production code rather than 1000+ external translation units. + cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_PLATFORM=linux \ + -DBUILD_TESTING=OFF \ + -B build/codeql -S . + cmake --build build/codeql --target deps -j "$(nproc)" + - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: @@ -69,13 +83,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # Third-party deps ship prebuilt SDK headers/libs; the engine itself - # must be built under CodeQL tracing. BUILD_TESTING=OFF keeps the - # scan focused on engine sources and avoids the test-only TUs. - cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_PLATFORM=linux \ - -DBUILD_TESTING=OFF \ - -B build/codeql -S . - cmake --build build/codeql --target deps -j "$(nproc)" + # The already-prepared build tree contains pinned dependency outputs. + # Only EVEngine production translation units are traced here. cmake --build build/codeql -j "$(nproc)" - name: Perform CodeQL analysis diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index a4bc2317..b4be0ddd 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -25,6 +25,12 @@ on: permissions: contents: write +# gh-pages is a single mutable publication target. Only the newest source +# revision may promote docs; superseded runs must never race a force-push. +concurrency: + group: publish-docs-gh-pages + cancel-in-progress: true + jobs: publish: name: Build and publish docs diff --git a/scripts/compare_render_backends.py b/scripts/compare_render_backends.py index a09cb3f4..43c5ecaf 100644 --- a/scripts/compare_render_backends.py +++ b/scripts/compare_render_backends.py @@ -7,7 +7,9 @@ import argparse import json import math import sys +from dataclasses import dataclass from pathlib import Path +from typing import Any from PIL import Image @@ -16,6 +18,59 @@ class ArtifactContractError(ValueError): """Raised when render-parity producers emitted incomplete artifact sets.""" +@dataclass(frozen=True) +class RenderContract: + """Versioned, backend-neutral comparison semantics for one rendered scene.""" + + color_space: str + alpha_coverage_delta_max: float + mean_rgb_error_max: float + p99_rgb_error_max: float + + +def parse_manifest(path: Path) -> dict[str, Any]: + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ArtifactContractError(f"manifest {path.name} is invalid: {error}") from error + for field in ("schema", "version", "scene", "backend", "contract"): + if field not in manifest: + raise ArtifactContractError( + f"manifest {path.name} missing required field: {field}" + ) + if manifest["schema"] != "evengine.render-parity" or manifest["version"] != 1: + raise ArtifactContractError( + f"manifest {path.name} has unsupported schema/version: " + f"{manifest['schema']!r}/{manifest['version']!r}" + ) + return manifest + + +def parse_render_contract(manifest: dict[str, Any], path: Path) -> RenderContract: + payload = manifest["contract"] + required = ( + "color_space", + "alpha_coverage_delta_max", + "mean_rgb_error_max", + "p99_rgb_error_max", + ) + if not isinstance(payload, dict): + raise ArtifactContractError(f"manifest {path.name} contract must be an object") + missing = [field for field in required if field not in payload] + if missing: + raise ArtifactContractError( + f"manifest {path.name} contract missing: {', '.join(missing)}" + ) + if payload["color_space"] != "srgb-linearized": + raise ArtifactContractError( + f"manifest {path.name} has unsupported color space: {payload['color_space']!r}" + ) + limits = [float(payload[field]) for field in required[1:]] + if any(not math.isfinite(value) or value < 0.0 for value in limits): + raise ArtifactContractError(f"manifest {path.name} has invalid tolerance") + return RenderContract(str(payload["color_space"]), *limits) + + def srgb_to_linear(value: int) -> float: channel = value / 255.0 if channel <= 0.04045: @@ -26,9 +81,9 @@ def srgb_to_linear(value: int) -> float: def compare_scene( reference_dir: Path, candidate_dir: Path, manifest_path: Path ) -> list[str]: - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest = parse_manifest(manifest_path) + contract = parse_render_contract(manifest, manifest_path) scene = manifest["scene"] - profile = manifest.get("profile", "lit3d") reference = Image.open(reference_dir / f"{scene}.png").convert("RGBA") candidate = Image.open(candidate_dir / f"{scene}.png").convert("RGBA") failures: list[str] = [] @@ -50,21 +105,24 @@ def compare_scene( mean_error = sum(errors) / len(errors) percentile_99 = errors[min(len(errors) - 1, math.ceil(len(errors) * 0.99) - 1)] - mean_limit = 2.0 / 255.0 if profile == "flat2d" else 6.0 / 255.0 - p99_limit = 8.0 / 255.0 if profile == "flat2d" else 20.0 / 255.0 - if alpha_ratio > 0.005: - failures.append(f"{scene}: alpha coverage delta {alpha_ratio:.4%} > 0.5%") - if mean_error > mean_limit: + if alpha_ratio > contract.alpha_coverage_delta_max: failures.append( - f"{scene}: mean linear RGB error {mean_error:.6f} > {mean_limit:.6f}" + f"{scene}: alpha coverage delta {alpha_ratio:.4%} > " + f"{contract.alpha_coverage_delta_max:.4%}" ) - if percentile_99 > p99_limit: + if mean_error > contract.mean_rgb_error_max: failures.append( - f"{scene}: p99 linear RGB error {percentile_99:.6f} > {p99_limit:.6f}" + f"{scene}: mean linear RGB error {mean_error:.6f} > " + f"{contract.mean_rgb_error_max:.6f}" + ) + if percentile_99 > contract.p99_rgb_error_max: + failures.append( + f"{scene}: p99 linear RGB error {percentile_99:.6f} > " + f"{contract.p99_rgb_error_max:.6f}" ) print( f"{scene}: alpha={alpha_ratio:.4%} mean={mean_error:.6f} " - f"p99={percentile_99:.6f} profile={profile}" + f"p99={percentile_99:.6f} color_space={contract.color_space}" ) return failures @@ -94,9 +152,10 @@ def validate_artifact_contract( ): for name, manifest_path in sorted(manifests.items()): try: - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest = parse_manifest(manifest_path) + parse_render_contract(manifest, manifest_path) scene = manifest["scene"] - except (OSError, json.JSONDecodeError, KeyError) as error: + except (ArtifactContractError, KeyError, TypeError, ValueError) as error: errors.append(f"{backend} manifest {name} is invalid: {error}") continue image_path = directory / f"{scene}.png" @@ -105,6 +164,18 @@ def validate_artifact_contract( f"{backend} manifest {name} is missing image {image_path.name}" ) + for name in sorted(reference_manifests.keys() & candidate_manifests.keys()): + try: + reference_manifest = parse_manifest(reference_manifests[name]) + candidate_manifest = parse_manifest(candidate_manifests[name]) + if reference_manifest["scene"] != candidate_manifest["scene"]: + errors.append(f"manifest {name} scene differs between backends") + if reference_manifest["contract"] != candidate_manifest["contract"]: + errors.append(f"manifest {name} contract differs between backends") + except ArtifactContractError: + # The backend-specific validation above already reports malformed input. + pass + if errors: raise ArtifactContractError("\n".join(errors)) return [reference_manifests[name] for name in sorted(reference_manifests)] diff --git a/scripts/tests/test_compare_render_backends.py b/scripts/tests/test_compare_render_backends.py index 4f186008..0b316e9d 100644 --- a/scripts/tests/test_compare_render_backends.py +++ b/scripts/tests/test_compare_render_backends.py @@ -13,12 +13,32 @@ from scripts.compare_render_backends import ( class CompareRenderBackendsTest(unittest.TestCase): - def write_scene(self, root: Path, rgba: tuple[int, int, int, int]) -> Path: + def write_scene( + self, + root: Path, + rgba: tuple[int, int, int, int], + *, + contract: dict | None = None, + ) -> Path: root.mkdir(parents=True) Image.new("RGBA", (4, 4), rgba).save(root / "flat.png") manifest = root / "flat.json" manifest.write_text( - json.dumps({"scene": "flat", "backend": root.name, "profile": "flat2d"}), + json.dumps( + { + "schema": "evengine.render-parity", + "version": 1, + "scene": "flat", + "backend": root.name, + "contract": contract + or { + "color_space": "srgb-linearized", + "alpha_coverage_delta_max": 0.005, + "mean_rgb_error_max": 2.0 / 255.0, + "p99_rgb_error_max": 8.0 / 255.0, + }, + } + ), encoding="utf-8", ) return manifest @@ -40,6 +60,52 @@ class CompareRenderBackendsTest(unittest.TestCase): failures = compare_scene(root / "vulkan", root / "webgpu", manifest) self.assertTrue(any("mean linear RGB" in failure for failure in failures)) + def test_scene_contract_controls_tolerance(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + permissive = { + "color_space": "srgb-linearized", + "alpha_coverage_delta_max": 0.005, + "mean_rgb_error_max": 1.0, + "p99_rgb_error_max": 1.0, + } + manifest = self.write_scene( + root / "vulkan", (0, 0, 0, 255), contract=permissive + ) + self.write_scene( + root / "webgpu", (255, 255, 255, 255), contract=permissive + ) + self.assertEqual( + compare_scene(root / "vulkan", root / "webgpu", manifest), [] + ) + + def test_contract_rejects_unversioned_manifest(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.write_scene(root / "vulkan", (10, 20, 30, 255)) + self.write_scene(root / "webgpu", (10, 20, 30, 255)) + manifest = root / "webgpu" / "flat.json" + payload = json.loads(manifest.read_text(encoding="utf-8")) + del payload["version"] + manifest.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaisesRegex( + ArtifactContractError, "missing required field: version" + ): + validate_artifact_contract(root / "vulkan", root / "webgpu") + + def test_contract_rejects_backend_tolerance_disagreement(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.write_scene(root / "vulkan", (10, 20, 30, 255)) + candidate = self.write_scene(root / "webgpu", (10, 20, 30, 255)) + payload = json.loads(candidate.read_text(encoding="utf-8")) + payload["contract"]["mean_rgb_error_max"] = 1.0 + candidate.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaisesRegex( + ArtifactContractError, "contract differs between backends" + ): + validate_artifact_contract(root / "vulkan", root / "webgpu") + def test_contract_rejects_missing_candidate_manifest(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) diff --git a/src/engine/common/ScriptCompiler.cpp b/src/engine/common/ScriptCompiler.cpp index ab7a8c4f..1b50b95f 100644 --- a/src/engine/common/ScriptCompiler.cpp +++ b/src/engine/common/ScriptCompiler.cpp @@ -404,9 +404,10 @@ ssq::Script ScriptCompiler::compileFile(std::string_view path) { } } -const ScriptMetadata* ScriptCompiler::metadata(std::string_view canonicalUri) const noexcept { +eve::OptionalRef ScriptCompiler::metadata(std::string_view canonicalUri) const noexcept { const auto found = metadata_.find(std::string(canonicalUri)); - return found == metadata_.end() ? nullptr : &found->second; + return found == metadata_.end() ? eve::OptionalRef{} + : eve::OptionalRef{std::cref(found->second)}; } std::vector ScriptCompiler::metadataSnapshot() const { @@ -443,11 +444,11 @@ std::vector ScriptCompiler::completions(std::string_view canon std::string_view prefix) const { std::vector result; const auto matches = [prefix](std::string_view value) { return value.rfind(prefix, 0) == 0; }; - if (const ScriptMetadata* unit = metadata(canonicalUri)) { - for (const ScriptSymbolMetadata& symbol : unit->symbols) { + if (const auto unit = metadata(canonicalUri)) { + for (const ScriptSymbolMetadata& symbol : unit->get().symbols) { if (matches(symbol.name)) result.push_back({symbol.name, symbol.kind, symbol.erasedType, symbol.name}); } - for (const std::string& name : unit->exports) { + for (const std::string& name : unit->get().exports) { if (matches(name)) result.push_back({name, "export", "module export", name}); } } @@ -480,8 +481,8 @@ std::vector ScriptCompiler::completions(std::string_view canon } std::optional ScriptCompiler::hover(std::string_view canonicalUri, std::string_view symbol) const { - if (const ScriptMetadata* unit = metadata(canonicalUri)) { - for (const ScriptSymbolMetadata& candidate : unit->symbols) { + if (const auto unit = metadata(canonicalUri)) { + for (const ScriptSymbolMetadata& candidate : unit->get().symbols) { if (candidate.name == symbol) return ScriptHover{candidate.name, "`" + candidate.kind + " " + candidate.name + ": " + candidate.erasedType + "`", @@ -502,8 +503,8 @@ std::optional ScriptCompiler::hover(std::string_view canonicalUri, } std::vector ScriptCompiler::diagnostics(std::string_view canonicalUri) const { - const ScriptMetadata* unit = metadata(canonicalUri); - return unit == nullptr ? std::vector{} : unit->diagnostics; + const auto unit = metadata(canonicalUri); + return unit ? unit->get().diagnostics : std::vector{}; } bool ScriptCompiler::setSourceMap(std::string_view canonicalUri, ScriptSourceMap sourceMap) { diff --git a/src/engine/common/ScriptCompiler.h b/src/engine/common/ScriptCompiler.h index 5d75d3c2..879dc46d 100644 --- a/src/engine/common/ScriptCompiler.h +++ b/src/engine/common/ScriptCompiler.h @@ -1,5 +1,7 @@ #pragma once +#include "common/BorrowedRef.h" + #include "common/Export.h" #include @@ -173,8 +175,14 @@ public: ssq::Script compileSource(std::string_view source, std::string_view sourceName); /** @brief Compiles a file through the VM and records its canonical identity. */ ssq::Script compileFile(std::string_view path); - /** @brief Returns metadata for the most recent successful compilation. */ - const ScriptMetadata* metadata(std::string_view canonicalUri) const noexcept; + /** + * @brief Borrows metadata for the most recent successful compilation. + * @return Empty when no compiled unit exists for the URI. + * @note The compiler owns the metadata. The reference is invalidated by the + * next compile or source-map update and by destruction of this compiler. + * It must not be retained across those operations or across threads. + */ + [[nodiscard]] eve::OptionalRef metadata(std::string_view canonicalUri) const noexcept; /** @brief Returns all successful compilation metadata sorted by URI. */ std::vector metadataSnapshot() const; /** @brief Native binding contracts used by semantic checks and tools. */ diff --git a/test/graphics_backend_parity.cpp b/test/graphics_backend_parity.cpp index 8c0f7868..65ba67b2 100644 --- a/test/graphics_backend_parity.cpp +++ b/test/graphics_backend_parity.cpp @@ -91,12 +91,21 @@ void writeParityArtifact(const eve::image::ImageData &image, const std::string & const bool lit3d = scene.starts_with("pbr_") || scene.starts_with("surface_") || scene.starts_with("masked_") || scene.starts_with("gbuffer_") || scene.starts_with("decal_") || scene == "dither" || scene == "coverage"; + const double meanLimit = (lit3d ? 6.0 : 2.0) / 255.0; + const double p99Limit = (lit3d ? 20.0 : 8.0) / 255.0; manifest << "{\n" + << " \"schema\": \"evengine.render-parity\",\n" + << " \"version\": 1,\n" << " \"scene\": \"" << scene << "\",\n" << " \"backend\": \"" << backend << "\",\n" << " \"width\": " << image.getWidth() << ",\n" << " \"height\": " << image.getHeight() << ",\n" - << " \"profile\": \"" << (lit3d ? "lit3d" : "flat2d") << "\"\n" + << " \"contract\": {\n" + << " \"color_space\": \"srgb-linearized\",\n" + << " \"alpha_coverage_delta_max\": 0.005,\n" + << " \"mean_rgb_error_max\": " << meanLimit << ",\n" + << " \"p99_rgb_error_max\": " << p99Limit << "\n" + << " }\n" << "}\n"; REQUIRE(manifest.good()); } diff --git a/test/mcp.cpp b/test/mcp.cpp index 08787c0b..96975edd 100644 --- a/test/mcp.cpp +++ b/test/mcp.cpp @@ -788,8 +788,8 @@ TEST_CASE("devtools.mcp.evalAndPause") { // One-shot MCP source must remain in memory: no user-space script and no // persistent compiler identity that can collide with project hot reload. - CHECK(runtime.scriptCompiler().metadata("mcp_snippet.nut") == nullptr); - CHECK(runtime.scriptCompiler().metadata("eval") == nullptr); + CHECK(!runtime.scriptCompiler().metadata("mcp_snippet.nut").has_value()); + CHECK(!runtime.scriptCompiler().metadata("eval").has_value()); bool foundScript = false; for (const auto& entry : std::filesystem::recursive_directory_iterator(projectRoot)) { if (entry.is_regular_file() && entry.path().extension() == ".nut") foundScript = true; diff --git a/test/script_compiler.cpp b/test/script_compiler.cpp index 426f0d30..e6597d58 100644 --- a/test/script_compiler.cpp +++ b/test/script_compiler.cpp @@ -19,14 +19,14 @@ TEST_CASE("scriptCompiler.recordsErasedLanguageMetadata") { "export function answer(value: int) -> int { return value + score }\n", "game:/metadata.nut"); - const script::ScriptMetadata* metadata = runtime.scriptCompiler().metadata("game:/metadata.nut"); - CHECK(metadata != nullptr); - CHECK(!metadata->sourceHash.empty()); - CHECK_EQ(metadata->persistRoots.size(), size_t(1)); - CHECK_EQ(metadata->persistRoots[0], std::string("score")); - CHECK_EQ(metadata->exports.size(), size_t(1)); - CHECK_EQ(metadata->exports[0], std::string("answer")); - CHECK_EQ(metadata->sourceMap.originalPosition({2, 9}).line, uint32_t(2)); + const auto metadata = runtime.scriptCompiler().metadata("game:/metadata.nut"); + REQUIRE(metadata.has_value()); + CHECK(!metadata->get().sourceHash.empty()); + CHECK_EQ(metadata->get().persistRoots.size(), size_t(1)); + CHECK_EQ(metadata->get().persistRoots[0], std::string("score")); + CHECK_EQ(metadata->get().exports.size(), size_t(1)); + CHECK_EQ(metadata->get().exports[0], std::string("answer")); + CHECK_EQ(metadata->get().sourceMap.originalPosition({2, 9}).line, uint32_t(2)); } TEST_CASE("scriptCompiler.metadataIgnoresCommentedLanguageForms") { @@ -232,12 +232,12 @@ TEST_CASE("scriptCompiler.retainsStructuredDiagnosticsAfterFailure") { rejected = true; } CHECK(rejected); - const script::ScriptMetadata* metadata = runtime.scriptCompiler().metadata("game:/invalid.nut"); - CHECK(metadata != nullptr); - CHECK_EQ(metadata->diagnostics.size(), size_t(1)); - CHECK_EQ(metadata->diagnostics[0].code, std::string("EVE2601")); - CHECK_EQ(metadata->diagnostics[0].position.line, uint32_t(1)); - CHECK(!metadata->diagnostics[0].fix.empty()); + const auto metadata = runtime.scriptCompiler().metadata("game:/invalid.nut"); + REQUIRE(metadata.has_value()); + CHECK_EQ(metadata->get().diagnostics.size(), size_t(1)); + CHECK_EQ(metadata->get().diagnostics[0].code, std::string("EVE2601")); + CHECK_EQ(metadata->get().diagnostics[0].position.line, uint32_t(1)); + CHECK(!metadata->get().diagnostics[0].fix.empty()); } TEST_CASE("scriptCompiler.requiresPluginAnnotationsToBeRegistered") { @@ -287,10 +287,10 @@ TEST_CASE("scriptCompiler.rawVmBridgeRecordsToolingMetadata") { runtime.handle(), source, static_cast(std::strlen(source)), "console_repl.nut", SQTrue); CHECK(SQ_SUCCEEDED(result)); sq_settop(runtime.handle(), top); - const script::ScriptMetadata* metadata = runtime.scriptCompiler().metadata("console_repl.nut"); - CHECK(metadata != nullptr); - CHECK_EQ(metadata->symbols.size(), size_t(1)); - CHECK_EQ(metadata->symbols[0].name, std::string("replValue")); + const auto metadata = runtime.scriptCompiler().metadata("console_repl.nut"); + REQUIRE(metadata.has_value()); + CHECK_EQ(metadata->get().symbols.size(), size_t(1)); + CHECK_EQ(metadata->get().symbols[0].name, std::string("replValue")); } TEST_CASE("scriptCompiler.bindingContractChecksLiteralTypes") { diff --git a/test/script_module.cpp b/test/script_module.cpp index 6387d497..e387933c 100644 --- a/test/script_module.cpp +++ b/test/script_module.cpp @@ -103,9 +103,9 @@ TEST_CASE("scriptModule.resolvesCachesAndInstantiatesDependencyGraph") { const auto reverse = runtime.scriptModules().reverseDependencies("mem:/answer.nut"); CHECK_EQ(reverse.size(), size_t(1)); CHECK_EQ(reverse[0], std::string("game:/main.nut")); - const script::ScriptMetadata* metadata = runtime.scriptCompiler().metadata("game:/main.nut"); - CHECK(metadata != nullptr); - CHECK_EQ(metadata->dependencies, dependencies); + const auto metadata = runtime.scriptCompiler().metadata("game:/main.nut"); + REQUIRE(metadata.has_value()); + CHECK_EQ(metadata->get().dependencies, dependencies); } TEST_CASE("scriptModule.rejectsCyclesBeforeExecution") {