diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 920a3ead..d190b1ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -288,9 +288,9 @@ jobs: if: ${{ needs.source-quality.result == 'success' && needs.change-scope.outputs.windows == 'true' && (github.event_name != 'workflow_call' || inputs.build_type == 'debug' || inputs.build_type == 'both') }} runs-on: windows-latest # This lane builds and tests both Vulkan and native Dawn configurations. - # A cold Windows runner needs more than 90 minutes for the two independent - # unit_test builds; keep the per-step limits below as the hang guards. - timeout-minutes: 180 + # Release run 35436382201 exhausted the former 180-minute job budget while + # the bounded native Dawn build step was still making progress. + timeout-minutes: 300 env: SCCACHE_GHA_ENABLED: "true" SCCACHE_GHA_VERSION: evengine-cpp-v1-windows-x64 @@ -530,7 +530,7 @@ jobs: # A cold Release build took 55 minutes and the process-isolated 3510-case # suite reached 3060/3510 at the former 30-minute test guard. Preserve # bounded hang detection while budgeting setup, build, and the full suite. - timeout-minutes: 180 + timeout-minutes: 240 env: SCCACHE_GHA_ENABLED: "true" SCCACHE_GHA_VERSION: evengine-cpp-v1-windows-x64-release @@ -576,9 +576,9 @@ jobs: run: bash .github/scripts/setup-software-vulkan.sh - name: Build win32 release - # A cold release build reached the former 90-minute guard without a - # compiler failure in release run 35419568563. - timeout-minutes: 120 + # Cold release builds reached the former 90- and 120-minute guards + # without a compiler failure in runs 35419568563 and 35436382201. + timeout-minutes: 160 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: >- @@ -601,10 +601,9 @@ jobs: needs: [source-quality, change-scope] if: ${{ needs.source-quality.result == 'success' && needs.change-scope.outputs.android == 'true' && (github.event_name != 'workflow_call' || inputs.build_type == 'debug' || inputs.build_type == 'both') }} runs-on: windows-latest - # A cold NDK build can consume almost the entire 45-minute build-step budget - # before Gradle packages the finished native library. Keep the step timeout - # below as the hang guard while leaving enough job budget for setup/cleanup. - timeout-minutes: 90 + # Release run 35436382201 reached 1363/1408 with no compiler failures at + # the former 60-minute build guard. Leave setup/cleanup headroom as well. + timeout-minutes: 120 env: SCCACHE_GHA_ENABLED: "true" SCCACHE_GHA_VERSION: evengine-cpp-v1-android-arm64-debug @@ -666,7 +665,7 @@ jobs: test -f "$ANDROID_NDK/build/cmake/android.toolchain.cmake" - name: Build android debug - timeout-minutes: 60 + timeout-minutes: 85 run: | test -f "$ANDROID_NDK/build/cmake/android.toolchain.cmake" make build/android-debug \ @@ -768,7 +767,7 @@ jobs: needs: [source-quality, change-scope] if: ${{ needs.source-quality.result == 'success' && needs.change-scope.outputs.macos == 'true' && (github.event_name != 'workflow_call' || inputs.build_type == 'debug' || inputs.build_type == 'both') }} runs-on: macos-latest - timeout-minutes: 120 + timeout-minutes: 180 env: SCCACHE_GHA_ENABLED: "true" SCCACHE_GHA_VERSION: evengine-cpp-v1-macos-arm64-debug @@ -875,7 +874,9 @@ jobs: fi - name: Build native Dawn Metal lane - timeout-minutes: 40 + # A cold Dawn/Tint build was still progressing with zero compiler + # failures at the former 40-minute guard in release run 35436382201. + timeout-minutes: 75 run: cmake --build build/webgpu-macos --target unit_test -j "$(sysctl -n hw.ncpu)" - name: Report C/C++ compilation cache @@ -1475,7 +1476,9 @@ jobs: needs: [source-quality] if: ${{ needs.source-quality.result == 'success' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} runs-on: ubuntu-24.04 - timeout-minutes: 90 + # The build plus half of the coverage test suite consumed the former + # 90-minute job budget in release run 35436382201. + timeout-minutes: 150 env: SCCACHE_GHA_ENABLED: "true" SCCACHE_GHA_VERSION: evengine-cpp-v1-linux-x64-coverage diff --git a/scripts/tests/test_ci_release_timeouts.py b/scripts/tests/test_ci_release_timeouts.py index 6580ab43..c6a536b0 100644 --- a/scripts/tests/test_ci_release_timeouts.py +++ b/scripts/tests/test_ci_release_timeouts.py @@ -37,7 +37,7 @@ class ReleaseJobTimeoutTests(unittest.TestCase): def test_release_job_budgets_cover_cold_builds_and_tests(self) -> None: minimums = { - "windows-release": 180, + "windows-release": 240, "android-release": 150, "macos-release": 150, "linux-release": 150, @@ -52,13 +52,29 @@ class ReleaseJobTimeoutTests(unittest.TestCase): def test_cold_native_build_steps_have_observed_headroom(self) -> None: minimums = { ("windows", "Build win32 debug"): 100, - ("windows-release", "Build win32 release"): 120, + ("windows-release", "Build win32 release"): 160, + ("android", "Build android debug"): 85, ("macos", "Build macosx debug"): 75, + ("macos", "Build native Dawn Metal lane"): 75, } for (job, step), minimum in minimums.items(): with self.subTest(job=job, step=step): self.assertGreaterEqual(step_timeout(job, step, self.workflow), minimum) + def test_multi_phase_job_budgets_cover_observed_cold_runs(self) -> None: + minimums = { + "windows": 300, + "android": 120, + "macos": 180, + "linux-coverage": 150, + } + for job, minimum in minimums.items(): + with self.subTest(job=job): + block = job_block(self.workflow, job) + match = re.search(r"^ timeout-minutes: (\d+)$", block, re.MULTILINE) + self.assertIsNotNone(match) + self.assertGreaterEqual(int(match.group(1)), minimum) + def test_android_jobs_start_sccache_before_the_build(self) -> None: for job in ("android", "android-release"): with self.subTest(job=job):