193 lines
9.6 KiB
Python
193 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Inventory resource tests; optionally run exact process-isolated CTest cases.
|
|
|
|
Source discovery is evidence of test existence, never evidence of format support
|
|
or successful execution. Missing CTest registrations fail execution closed.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import xml.etree.ElementTree as ET
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
FAMILIES = {
|
|
"image": ["resource_format_image.cpp", "image.cpp", "image_script.cpp", "asset_graphics_loader.cpp"],
|
|
"sound": ["resource_decoder_lifetime.cpp", "resource_format_sound.cpp", "sound.cpp", "audio.cpp", "editor_audio_import_diagnostics.cpp"],
|
|
"model": ["resource_format_model.cpp", "model3d.cpp", "model3d_normals.cpp", "medialoader_model_link.cpp", "avatar_vrm_vulkan.cpp"],
|
|
"font": ["font.cpp", "graphics_font.cpp"],
|
|
"map": ["map.cpp", "resource_format_map.cpp"],
|
|
"asset": [p.name for p in sorted((ROOT / "test").glob("asset_*.cpp"))],
|
|
"resource": ["filesystem_cpp.cpp", "resource.cpp", "resource_uri.cpp", "resource_lifetime.cpp"],
|
|
}
|
|
|
|
|
|
def inventory(root=ROOT):
|
|
result = {}
|
|
for family, files in FAMILIES.items():
|
|
cases = {}
|
|
for name in files:
|
|
path = root / "test" / name
|
|
for case in re.findall(r'TEST_CASE\(\s*"([^"\n]+)"', path.read_text(encoding="utf-8-sig")):
|
|
if case in cases:
|
|
raise ValueError("duplicate test: " + case)
|
|
cases[case] = "test/" + name
|
|
extensions = []
|
|
loader = {"image": "image/ImageLoader.cpp", "sound": "sound/SoundLoader.cpp", "model": "model3d/ModelLoader.cpp"}.get(family)
|
|
if loader:
|
|
extensions = sorted(set(re.findall(r'ext == "(\.[^"]+)"', (root / "src/modules" / loader).read_text())))
|
|
result[family] = {"advertised_resource_extensions": extensions, "tests": cases}
|
|
return result
|
|
|
|
|
|
def fixture_errors(root=ROOT):
|
|
directory = root / "test/fixtures/resource_formats"
|
|
manifest = json.loads((directory / "manifest.json").read_text())
|
|
if manifest.get("schema") != "evengine.resource-format-fixtures" or manifest.get("version") != 1:
|
|
raise ValueError("unsupported fixture manifest schema/version")
|
|
errors = []
|
|
for name, expected in manifest["sha256"].items():
|
|
if Path(name).name != name:
|
|
raise ValueError("fixture path must be a basename")
|
|
path = directory / name
|
|
if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected:
|
|
errors.append(name)
|
|
return errors
|
|
|
|
|
|
def format_coverage(groups, results, root=ROOT):
|
|
contract = json.loads((root / "scripts/resource_format_contracts.json").read_text())
|
|
if (set(contract) != {"schema", "version", "unknownFields", "formats"}
|
|
or contract["schema"] != "evengine.resource-format-contracts"
|
|
or contract["version"] != 1 or contract["unknownFields"] != "reject"):
|
|
raise ValueError("unsupported format contract schema/version or unknown fields")
|
|
known_tests = {name for group in inventory(root).values() for name in group["tests"]}
|
|
statuses = {test["name"]: test["status"] for test in results}
|
|
rows = []
|
|
seen = set()
|
|
for row in contract["formats"]:
|
|
if set(row) != {"id", "family", "extensions", "importTests", "exportTests", "exportMode", "scope"}:
|
|
raise ValueError("invalid format contract fields")
|
|
if row["id"] in seen:
|
|
raise ValueError("duplicate format contract: " + row["id"])
|
|
seen.add(row["id"])
|
|
for name in row["importTests"] + row["exportTests"]:
|
|
if name not in known_tests:
|
|
raise ValueError("stale format test reference: " + name)
|
|
if row["family"] not in groups:
|
|
continue
|
|
entry = dict(row)
|
|
for operation in ["import", "export"]:
|
|
names = row[operation + "Tests"]
|
|
entry[operation + "Status"] = ("coverage-missing" if not names else
|
|
"passed" if all(statuses.get(name) == "passed" for name in names) else
|
|
"failed" if any(name in statuses and statuses[name] != "passed" for name in names) else "not-run")
|
|
rows.append(entry)
|
|
for family, group in groups.items():
|
|
for extension in group["advertised_resource_extensions"]:
|
|
if not any(row["family"] == family and extension in row["extensions"] for row in rows):
|
|
raise ValueError("resource extension missing from contract: " + family + extension)
|
|
return rows
|
|
|
|
|
|
def run_cases(build_dir, selected):
|
|
listing = subprocess.run(["ctest", "--test-dir", str(build_dir), "--show-only=json-v1"],
|
|
check=True, capture_output=True, text=True, errors="replace")
|
|
registered = {t["name"]: t for t in json.loads(listing.stdout)["tests"]}
|
|
missing = sorted(set(selected) - registered.keys())
|
|
disabled = sorted(name for name in selected if name in registered and any(
|
|
p["name"] == "DISABLED" and p["value"] for p in registered[name].get("properties", [])))
|
|
if missing or disabled or not selected:
|
|
return 1, {"status": "failed", "missing_registrations": missing, "disabled_tests": disabled}
|
|
results = []
|
|
outputs = []
|
|
code = 0
|
|
# Keep the command under Windows' process command-line limit as suites grow.
|
|
with tempfile.TemporaryDirectory(prefix="eve-resource-ctest-") as directory:
|
|
for offset in range(0, len(selected), 40):
|
|
batch = selected[offset:offset + 40]
|
|
junit = Path(directory) / "results.xml"
|
|
junit.unlink(missing_ok=True)
|
|
pattern = "^(" + "|".join(re.escape(name) for name in batch) + ")$"
|
|
run = subprocess.run(["ctest", "--test-dir", str(build_dir), "--output-on-failure",
|
|
"--no-tests=error", "--output-junit", str(junit), "-R", pattern],
|
|
capture_output=True, text=True, errors="replace")
|
|
outputs.append(run.stdout + run.stderr)
|
|
code = code or run.returncode
|
|
if not junit.is_file():
|
|
code = code or 1
|
|
results.extend({"name": name, "status": "missing-result"} for name in batch)
|
|
continue
|
|
observed = set()
|
|
for case in ET.parse(junit).getroot().iter("testcase"):
|
|
name = case.attrib["name"]
|
|
status = "passed"
|
|
if (case.find("failure") is not None or case.find("error") is not None
|
|
or "Assertion Failed" in case.findtext("system-out", "")):
|
|
status = "failed"
|
|
elif case.find("skipped") is not None or case.get("status") in ("notrun", "disabled"):
|
|
status = "skipped"
|
|
if name not in batch or name in observed:
|
|
status = "unexpected-result"
|
|
observed.add(name)
|
|
result = {"name": name, "status": status, "seconds": case.get("time")}
|
|
if status != "passed":
|
|
result["output"] = case.findtext("system-out", "")[:16000]
|
|
results.append(result)
|
|
if status != "passed":
|
|
code = code or 1
|
|
for name in sorted(set(batch) - observed):
|
|
results.append({"name": name, "status": "missing-result"})
|
|
code = code or 1
|
|
return code, {"status": "passed" if code == 0 else "failed", "results": results,
|
|
"output": "\n".join(outputs)}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--build-dir", type=Path, help="Run tests from this configured CTest build")
|
|
parser.add_argument("--family", choices=sorted(FAMILIES), action="append")
|
|
parser.add_argument("--new-contracts-only", action="store_true", help="Select resourceFormats.* cases only")
|
|
parser.add_argument("--report", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
groups = {k: v for k, v in inventory().items() if not args.family or k in args.family}
|
|
if args.new_contracts_only:
|
|
for group in groups.values():
|
|
group["tests"] = {k: v for k, v in group["tests"].items() if k.startswith("resourceFormats.")}
|
|
report = {"schema": "evengine.resource-test-report", "version": 1,
|
|
"status": "not-run", "families": groups,
|
|
"limitation": "Extension routing and test names do not prove per-format coverage or export support."}
|
|
code = 0
|
|
try:
|
|
report["fixture_errors"] = fixture_errors()
|
|
if report["fixture_errors"]:
|
|
code, report["status"] = 1, "failed"
|
|
elif args.build_dir:
|
|
selected = sorted({name for group in groups.values() for name in group["tests"]})
|
|
code, result = run_cases(args.build_dir, selected)
|
|
report.update(result)
|
|
except (subprocess.SubprocessError, OSError, ValueError, KeyError, ET.ParseError) as error:
|
|
report.update(status="failed", error=str(error))
|
|
code = 1
|
|
try:
|
|
report["format_coverage"] = format_coverage(groups, report.get("results", []))
|
|
except (OSError, ValueError, KeyError) as error:
|
|
report.update(status="failed", error=str(error))
|
|
code = 1
|
|
args.report.parent.mkdir(parents=True, exist_ok=True)
|
|
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
print(f"{report['status']}: {sum(len(g['tests']) for g in groups.values())} source tests; {args.report}")
|
|
if report.get("error"):
|
|
print(report["error"], file=sys.stderr)
|
|
if report.get("fixture_errors"):
|
|
print("Fixture hash mismatch: " + ", ".join(report["fixture_errors"]), file=sys.stderr)
|
|
return code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|