ci: budget full cold release matrix

.github/workflows/ci.yml

@@ -288,9 +288,9 @@ jobs:
     if: ${{ needs.source-quality.result == 'success' && needs.change-scope.outputs.windows == 'true' && (github.event_name != 'workflow_call' || inputs.build_type == 'debug' || inputs.build_type == 'both') }}
     runs-on: windows-latest
     # This lane builds and tests both Vulkan and native Dawn configurations.
-    # A cold Windows runner needs more than 90 minutes for the two independent
-    # unit_test builds; keep the per-step limits below as the hang guards.
-    timeout-minutes: 180
+    # Release run 35436382201 exhausted the former 180-minute job budget while
+    # the bounded native Dawn build step was still making progress.
+    timeout-minutes: 300
     env:
       SCCACHE_GHA_ENABLED: "true"
       SCCACHE_GHA_VERSION: evengine-cpp-v1-windows-x64
@@ -530,7 +530,7 @@ jobs:
     # A cold Release build took 55 minutes and the process-isolated 3510-case
     # suite reached 3060/3510 at the former 30-minute test guard. Preserve
     # bounded hang detection while budgeting setup, build, and the full suite.
-    timeout-minutes: 180
+    timeout-minutes: 240
     env:
       SCCACHE_GHA_ENABLED: "true"
       SCCACHE_GHA_VERSION: evengine-cpp-v1-windows-x64-release
@@ -576,9 +576,9 @@ jobs:
         run: bash .github/scripts/setup-software-vulkan.sh
 
       - name: Build win32 release
-        # A cold release build reached the former 90-minute guard without a
-        # compiler failure in release run 35419568563.
-        timeout-minutes: 120
+        # Cold release builds reached the former 90- and 120-minute guards
+        # without a compiler failure in runs 35419568563 and 35436382201.
+        timeout-minutes: 160
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: >-
@@ -601,10 +601,9 @@ jobs:
     needs: [source-quality, change-scope]
     if: ${{ needs.source-quality.result == 'success' && needs.change-scope.outputs.android == 'true' && (github.event_name != 'workflow_call' || inputs.build_type == 'debug' || inputs.build_type == 'both') }}
     runs-on: windows-latest
-    # A cold NDK build can consume almost the entire 45-minute build-step budget
-    # before Gradle packages the finished native library. Keep the step timeout
-    # below as the hang guard while leaving enough job budget for setup/cleanup.
-    timeout-minutes: 90
+    # Release run 35436382201 reached 1363/1408 with no compiler failures at
+    # the former 60-minute build guard. Leave setup/cleanup headroom as well.
+    timeout-minutes: 120
     env:
       SCCACHE_GHA_ENABLED: "true"
       SCCACHE_GHA_VERSION: evengine-cpp-v1-android-arm64-debug
@@ -666,7 +665,7 @@ jobs:
           test -f "$ANDROID_NDK/build/cmake/android.toolchain.cmake"
 
       - name: Build android debug
-        timeout-minutes: 60
+        timeout-minutes: 85
         run: |
           test -f "$ANDROID_NDK/build/cmake/android.toolchain.cmake"
           make build/android-debug \
@@ -768,7 +767,7 @@ jobs:
     needs: [source-quality, change-scope]
     if: ${{ needs.source-quality.result == 'success' && needs.change-scope.outputs.macos == 'true' && (github.event_name != 'workflow_call' || inputs.build_type == 'debug' || inputs.build_type == 'both') }}
     runs-on: macos-latest
-    timeout-minutes: 120
+    timeout-minutes: 180
     env:
       SCCACHE_GHA_ENABLED: "true"
       SCCACHE_GHA_VERSION: evengine-cpp-v1-macos-arm64-debug
@@ -875,7 +874,9 @@ jobs:
           fi
 
       - name: Build native Dawn Metal lane
-        timeout-minutes: 40
+        # A cold Dawn/Tint build was still progressing with zero compiler
+        # failures at the former 40-minute guard in release run 35436382201.
+        timeout-minutes: 75
         run: cmake --build build/webgpu-macos --target unit_test -j "$(sysctl -n hw.ncpu)"
 
       - name: Report C/C++ compilation cache
@@ -1475,7 +1476,9 @@ jobs:
     needs: [source-quality]
     if: ${{ needs.source-quality.result == 'success' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
     runs-on: ubuntu-24.04
-    timeout-minutes: 90
+    # The build plus half of the coverage test suite consumed the former
+    # 90-minute job budget in release run 35436382201.
+    timeout-minutes: 150
     env:
       SCCACHE_GHA_ENABLED: "true"
       SCCACHE_GHA_VERSION: evengine-cpp-v1-linux-x64-coverage

scripts/tests/test_ci_release_timeouts.py

@@ -37,7 +37,7 @@ class ReleaseJobTimeoutTests(unittest.TestCase):
 
     def test_release_job_budgets_cover_cold_builds_and_tests(self) -> None:
         minimums = {
-            "windows-release": 180,
+            "windows-release": 240,
             "android-release": 150,
             "macos-release": 150,
             "linux-release": 150,
@@ -52,13 +52,29 @@ class ReleaseJobTimeoutTests(unittest.TestCase):
     def test_cold_native_build_steps_have_observed_headroom(self) -> None:
         minimums = {
             ("windows", "Build win32 debug"): 100,
-            ("windows-release", "Build win32 release"): 120,
+            ("windows-release", "Build win32 release"): 160,
+            ("android", "Build android debug"): 85,
             ("macos", "Build macosx debug"): 75,
+            ("macos", "Build native Dawn Metal lane"): 75,
         }
         for (job, step), minimum in minimums.items():
             with self.subTest(job=job, step=step):
                 self.assertGreaterEqual(step_timeout(job, step, self.workflow), minimum)
 
+    def test_multi_phase_job_budgets_cover_observed_cold_runs(self) -> None:
+        minimums = {
+            "windows": 300,
+            "android": 120,
+            "macos": 180,
+            "linux-coverage": 150,
+        }
+        for job, minimum in minimums.items():
+            with self.subTest(job=job):
+                block = job_block(self.workflow, job)
+                match = re.search(r"^    timeout-minutes: (\d+)$", block, re.MULTILINE)
+                self.assertIsNotNone(match)
+                self.assertGreaterEqual(int(match.group(1)), minimum)
+
     def test_android_jobs_start_sccache_before_the_build(self) -> None:
         for job in ("android", "android-release"):
             with self.subTest(job=job):