From 69f400fa19410c9fa48f804bc89e36601531912e Mon Sep 17 00:00:00 2001 From: kuddev Date: Mon, 14 Sep 2026 20:52:33 +0800 Subject: [PATCH] ci: separate compiled caches and reuse complete native validation --- .github/actions/rust-cache/action.yml | 105 ++++++++++++++++++++++++ .github/workflows/linux-lua.yml | 33 +++++++- .github/workflows/release.yml | 112 +++++++++++--------------- docs/architecture-decisions.md | 13 ++- scripts/tests/test_ci_cache.py | 68 ++++++++++++++++ scripts/tests/test_stable_release.py | 12 ++- 6 files changed, 267 insertions(+), 76 deletions(-) create mode 100644 .github/actions/rust-cache/action.yml create mode 100644 scripts/tests/test_ci_cache.py diff --git a/.github/actions/rust-cache/action.yml b/.github/actions/rust-cache/action.yml new file mode 100644 index 0000000..368f159 --- /dev/null +++ b/.github/actions/rust-cache/action.yml @@ -0,0 +1,105 @@ +name: Cargo downloads and compiled targets +description: Share downloads across jobs without duplicating them in every target cache + +inputs: + key: + description: Compiled workload (core, product, release or legacy) + required: true + target: + description: Compiled profile directory to retain, relative to the repository + required: true + legacy-key: + description: Previous combined cache key, restored only during migration + default: "" + extra-key: + description: Additional inputs for native build tools stored with this target + default: "" + +outputs: + cache-key: + description: Key to use when saving compiled targets + value: ${{ steps.identity.outputs.key }} + cache-hit: + description: Whether the compiled target cache matched exactly + value: ${{ steps.target.outputs.cache-hit }} + +runs: + using: composite + steps: + - name: Identify compiler and build environment + id: identity + shell: bash + env: + CACHE_OS: ${{ runner.os }} + CACHE_ARCH: ${{ runner.arch }} + CACHE_WORKLOAD: ${{ inputs.key }} + CACHE_MANIFESTS: ${{ hashFiles('Cargo.lock', '**/Cargo.toml', 'rust-toolchain.toml', '**/.cargo/config.toml', '.github/ci-profile.toml') }}${{ inputs.extra-key }} + run: | + python - <<'PY' + import hashlib + import json + import os + from pathlib import Path + import subprocess + facts = {"rustc": subprocess.check_output(["rustc", "-vV"], text=True)} + prefixes = ("CARGO_PROFILE_", "CFLAGS", "CXXFLAGS", "RUSTFLAGS") + names = {"CARGO_BUILD_TARGET", "CARGO_ENCODED_RUSTFLAGS", "RUSTC_WRAPPER", + "CC", "CXX", "ImageOS", "MACOSX_DEPLOYMENT_TARGET", "SDKROOT", "DEVELOPER_DIR"} + facts["environment"] = {key: value for key, value in os.environ.items() + if key in names or key.startswith(prefixes)} + if os.environ["CACHE_OS"] == "macOS": + facts["sdk"] = subprocess.check_output(["xcrun", "--sdk", "macosx", "--show-sdk-version"], text=True) + fingerprint = hashlib.sha256(json.dumps(facts, sort_keys=True).encode()).hexdigest()[:16] + prefix = "cargo-targets-v1-" + "-".join(os.environ[name] for name in + ("CACHE_OS", "CACHE_ARCH", "CACHE_WORKLOAD")) + prefix += "-" + fingerprint + "-" + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write("key=" + prefix + os.environ["CACHE_MANIFESTS"] + "\n") + output.write("restore-key=" + prefix + "\n") + output.write("cargo-home=" + os.environ.get("CARGO_HOME", str(Path.home() / ".cargo")) + "\n") + PY + - name: Restore shared Cargo downloads + id: downloads + uses: actions/cache/restore@v4 + with: + # Keep the immutable archives/Git objects once per native platform. + # Cargo recreates checkouts and extracted registry sources from these. + path: | + ${{ steps.identity.outputs.cargo-home }}/registry/index + ${{ steps.identity.outputs.cargo-home }}/registry/cache + ${{ steps.identity.outputs.cargo-home }}/git/db + key: cargo-downloads-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml') }} + restore-keys: cargo-downloads-v1-${{ runner.os }}-${{ runner.arch }}- + - name: Restore compiled targets + id: target + uses: actions/cache/restore@v4 + with: + path: ${{ inputs.target }} + key: ${{ steps.identity.outputs.key }} + restore-keys: ${{ steps.identity.outputs.restore-key }} + - name: Recover an existing combined cache during migration + if: steps.target.outputs.cache-matched-key == '' && inputs.legacy-key != '' + uses: Swatinem/rust-cache@v2 + with: + shared-key: ${{ inputs.legacy-key }} + workspaces: . -> target + env-vars: MACOSX_DEPLOYMENT_TARGET + cache-bin: false + cache-on-failure: true + save-if: false + - name: Populate complete downloads before sharing the cache + if: steps.downloads.outputs.cache-hit != 'true' + shell: bash + run: | + host_target="$(rustc -vV | sed -n 's/^host: //p')" + test -n "$host_target" + cargo fetch --locked --target "$host_target" + - name: Save complete shared Cargo downloads + if: steps.downloads.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + ${{ steps.identity.outputs.cargo-home }}/registry/index + ${{ steps.identity.outputs.cargo-home }}/registry/cache + ${{ steps.identity.outputs.cargo-home }}/git/db + key: ${{ steps.downloads.outputs.cache-primary-key }} diff --git a/.github/workflows/linux-lua.yml b/.github/workflows/linux-lua.yml index b006226..2223825 100644 --- a/.github/workflows/linux-lua.yml +++ b/.github/workflows/linux-lua.yml @@ -2,7 +2,10 @@ name: Full native tests on: workflow_dispatch: + workflow_call: push: + # Releases call this entire workflow instead of duplicating branch-push jobs. + branches-ignore: ["release/**"] paths: - "**/*.rs" - "scripts/**/*.py" @@ -11,6 +14,7 @@ on: - "rust-toolchain.toml" - ".github/workflows/linux-lua.yml" - ".github/ci-profile.toml" + - ".github/actions/rust-cache/action.yml" pull_request: paths: - "**/*.rs" @@ -20,6 +24,7 @@ on: - "rust-toolchain.toml" - ".github/workflows/linux-lua.yml" - ".github/ci-profile.toml" + - ".github/actions/rust-cache/action.yml" permissions: contents: read @@ -64,13 +69,24 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: toolchain: "1.97.1" - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/rust-cache + id: rust-cache with: - shared-key: ${{ runner.os == 'Windows' && 'windows' || runner.os == 'Linux' && 'linux' || 'macos' }} - cache-workspace-crates: true - cache-on-failure: true + key: native + target: target/ci + legacy-key: ${{ runner.os == 'Windows' && 'windows' || runner.os == 'Linux' && 'linux' || 'macos' }} + - name: Prepare pinned Windows console runtime + if: runner.os == 'Windows' + shell: pwsh + run: ./scripts/prepare-windows-runtime.ps1 -Destination assets/windows/conhost - name: Test complete workspace and native harness run: python scripts/ci_native_tests.py + - name: Save compiled workload + if: ${{ always() && steps.rust-cache.outcome == 'success' && steps.rust-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: target/ci + key: ${{ steps.rust-cache.outputs.cache-key }} - name: Check source architecture if: ${{ !cancelled() && runner.os == 'Linux' }} run: python scripts/check_architecture.py @@ -80,3 +96,12 @@ jobs: - name: Check the release workspace if: ${{ !cancelled() }} run: cargo check --locked --workspace --release + - name: Upload native compilation timings + if: always() + uses: actions/upload-artifact@v4 + with: + name: native-timings-${{ matrix.os }} + path: target/cargo-timings + if-no-files-found: warn + compression-level: 0 + retention-days: 14 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5a0c8eb..19b2f6a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,65 +120,10 @@ jobs: python3 -m py_compile scripts/stable_release.py scripts/preview_release.py native-tests: - name: Native tests (${{ matrix.platform }}) + name: Native tests if: ${{ !inputs.windows_only }} needs: prepare - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - include: - - runner: ubuntu-22.04 - platform: linux-x64 - cache: linux - - runner: windows-2022 - platform: windows-x64 - cache: windows - - runner: macos-26 - platform: macos-arm64 - cache: macos - - runner: macos-26-intel - platform: macos-x64 - cache: macos - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - name: Install native test dependencies - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - build-essential cmake pkg-config zsh libasound2-dev libdbus-1-dev libudev-dev \ - libfontconfig1-dev libfreetype6-dev libegl1-mesa-dev libgl1-mesa-dev \ - libwayland-dev libxkbcommon-dev libxkbcommon-x11-dev libx11-xcb-dev \ - libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev \ - libxcb-xinput-dev libxcb-xkb-dev libxi-dev - - uses: dtolnay/rust-toolchain@stable - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - - uses: Swatinem/rust-cache@v2 - with: - shared-key: ${{ matrix.cache }} - cache-workspace-crates: true - cache-on-failure: true - - name: Prepare pinned Windows console runtime - if: runner.os == 'Windows' - shell: pwsh - run: ./scripts/prepare-windows-runtime.ps1 -Destination assets/windows/conhost - - name: Test complete workspace and native harness - run: python scripts/ci_native_tests.py - - name: Upload native compilation timings - if: always() - uses: actions/upload-artifact@v4 - with: - compression-level: 0 - name: native-timings-${{ matrix.platform }} - path: target/cargo-timings - if-no-files-found: warn - retention-days: 14 + uses: ./.github/workflows/linux-lua.yml linux: name: Linux x86_64 stable package @@ -210,12 +155,20 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: toolchain: ${{ env.RUST_TOOLCHAIN }} - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/rust-cache + id: rust-cache with: - cache-on-failure: true - cache-workspace-crates: true + key: release + target: target/release + legacy-key: linux - name: Build GPUI release binary run: cargo build --locked --release -p nebula --bin pebrel --features gpui-shell --timings + - name: Save compiled workload + if: ${{ always() && steps.rust-cache.outcome == 'success' && steps.rust-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: target/release + key: ${{ steps.rust-cache.outputs.cache-key }} - name: Download pinned linuxdeploy env: LINUXDEPLOY: ${{ runner.temp }}/linuxdeploy-x86_64.AppImage @@ -339,12 +292,20 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: toolchain: ${{ env.RUST_TOOLCHAIN }} - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/rust-cache + id: rust-cache with: - cache-on-failure: true - cache-workspace-crates: true + key: release + target: target/release + legacy-key: macos - name: Build GPUI release binary run: cargo build --locked --release -p nebula --bin pebrel --features gpui-shell --timings + - name: Save compiled workload + if: ${{ always() && steps.rust-cache.outcome == 'success' && steps.rust-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: target/release + key: ${{ steps.rust-cache.outputs.cache-key }} - name: Import Developer ID signing credentials if: ${{ env.MACOS_SIGNING == 'developer-id' }} env: @@ -476,7 +437,7 @@ jobs: $run = Invoke-RestMethod -Uri "https://api.github.com/repos/$env:GITHUB_REPOSITORY/actions/runs/$env:VALIDATED_RUN" -Headers $headers -ErrorAction Stop if ($run.head_sha -ne $env:SELECTED_COMMIT) { throw 'Validated run source differs.' } $jobs = Invoke-RestMethod -Uri "https://api.github.com/repos/$env:GITHUB_REPOSITORY/actions/runs/$env:VALIDATED_RUN/jobs?per_page=100" -Headers $headers -ErrorAction Stop - $windows = @($jobs.jobs | Where-Object { $_.name -in @('Windows x64 stable package', 'Native tests (windows-x64)') }) + $windows = @($jobs.jobs | Where-Object { $_.name -in @('Windows x64 stable package', 'Native tests (windows-x64)') -or $_.name -match 'Tests \(windows-2022\)$' }) $tests = @($windows.steps | Where-Object { $_.name -in @('Test workspace and native harness', 'Test complete workspace and native harness') -and $_.conclusion -eq 'success' }) if ($tests.Count -ne 1) { throw 'Prior Windows native tests did not pass.' } - uses: actions/checkout@v4 @@ -489,10 +450,17 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: toolchain: ${{ env.RUST_TOOLCHAIN }} + - uses: ./.github/actions/rust-cache + id: rust-cache + with: + key: release + target: target/release + legacy-key: windows + if: ${{ inputs.source_commit == '' }} - uses: Swatinem/rust-cache@v2 + if: ${{ inputs.source_commit != '' }} with: cache-on-failure: true - cache-workspace-crates: true - name: Prepare pinned Windows console runtime shell: pwsh run: | @@ -503,7 +471,19 @@ jobs: run: python scripts/ci_native_tests.py - name: Build GPUI release binary shell: pwsh - run: ./scripts/build-windows-product.ps1 -Configuration release -TargetDirectory target + run: | + if (Test-Path scripts/build-windows-product.ps1) { + ./scripts/build-windows-product.ps1 -Configuration release -TargetDirectory target + } else { + cargo build --locked --release -p nebula --bin pebrel -p nebula_hook --bin pebrel-hook --features nebula/gpui-shell + if ($LASTEXITCODE -ne 0) { throw 'Windows product build failed' } + } + - name: Save compiled workload + if: ${{ always() && steps.rust-cache.outcome == 'success' && steps.rust-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: target/release + key: ${{ steps.rust-cache.outputs.cache-key }} - name: Retain built Windows runtime before packaging uses: actions/upload-artifact@v4 with: diff --git a/docs/architecture-decisions.md b/docs/architecture-decisions.md index 3d1cef7..bcc3746 100644 --- a/docs/architecture-decisions.md +++ b/docs/architecture-decisions.md @@ -431,9 +431,16 @@ settings files. - **Decision:** Run the complete Rust workspace with the product interaction feature enabled in one unfiltered invocation. Run native tests independently from package construction; asset aggregation depends on both. Each native - platform still runs the complete Python helper and harness suites. Cache keys - preserve the existing platform namespace and retain workspace crates. Cargo - still validates source, profile and feature fingerprints before reuse. + platform still runs the complete Python helper and harness suites. Separate + test and release cache keys prevent concurrent jobs from replacing one another's + compiled workload. Dependency archives and Git objects are shared within each + platform; only the relevant compiled profile is saved per workload. A fallback + reads existing combined caches during migration. Cargo still validates source, + profile and feature fingerprints before reuse. +- **Scheduling:** Stable releases call the same complete four-platform native + workflow used for contributions, including architecture, translation allocation + contracts and the release-workspace check. Release branch pushes omit a duplicate + automatic invocation; the release aggregation still requires the called suite. - **Test profile:** An explicit CI-only profile removes developer-preview optimization and debug information from test compilation, including named dependency overrides. It retains debug assertions and overflow checks. The diff --git a/scripts/tests/test_ci_cache.py b/scripts/tests/test_ci_cache.py new file mode 100644 index 0000000..3526327 --- /dev/null +++ b/scripts/tests/test_ci_cache.py @@ -0,0 +1,68 @@ +"""Exercise the cache identity code used by the composite action itself.""" + +import os +from pathlib import Path +import re +import tempfile +import textwrap +import unittest +from unittest.mock import patch + + +ACTION = Path(__file__).resolve().parents[2] / ".github/actions/rust-cache/action.yml" + + +class CacheIdentityTests(unittest.TestCase): + def identity(self, *, compiler="rustc 1.97.1", sdk="15.0", **changes): + source = ACTION.read_text(encoding="utf-8") + body = re.search(r"python - <<'PY'\n(.*?)\n PY", source, re.S) + self.assertIsNotNone(body, "cache identity must remain executable from this action") + code = textwrap.dedent(body.group(1)) + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "outputs" + environment = { + "CACHE_OS": "Linux", "CACHE_ARCH": "X64", "CACHE_WORKLOAD": "ci-product", + "CACHE_MANIFESTS": "pinned-manifests", "GITHUB_OUTPUT": str(output), + "CARGO_HOME": str(Path(temporary) / "cargo"), **changes, + } + + def command(arguments, **kwargs): + if arguments == ["rustc", "-vV"]: + return compiler + self.assertEqual(arguments, ["xcrun", "--sdk", "macosx", "--show-sdk-version"]) + return sdk + + with patch.dict(os.environ, environment), patch("subprocess.check_output", side_effect=command): + exec(compile(code, str(ACTION), "exec"), {}) + return dict(line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines()) + + def test_compiler_flags_and_workload_isolate_compiled_targets(self): + baseline = self.identity()["key"] + for change in ( + {"compiler": "rustc 1.98.0"}, {"RUSTFLAGS": "-C opt-level=1"}, + {"CARGO_PROFILE_RELEASE_LTO": "false"}, {"CACHE_WORKLOAD": "release"}, + {"CACHE_ARCH": "ARM64"}, {"ImageOS": "a-different-runner-image"}, + ): + with self.subTest(change=change): + self.assertNotEqual(baseline, self.identity(**change)["key"]) + + def test_macos_sdk_and_deployment_floor_are_part_of_identity(self): + baseline = self.identity(CACHE_OS="macOS", sdk="15.0")["key"] + self.assertNotEqual(baseline, self.identity(CACHE_OS="macOS", sdk="16.0")["key"]) + self.assertNotEqual(baseline, self.identity(CACHE_OS="macOS", MACOSX_DEPLOYMENT_TARGET="13.0")["key"]) + + def test_dependency_change_can_restore_compatible_previous_targets(self): + before = self.identity(CACHE_MANIFESTS="before") + after = self.identity(CACHE_MANIFESTS="after") + self.assertNotEqual(before["key"], after["key"]) + self.assertEqual(before["restore-key"], after["restore-key"]) + self.assertTrue(after["key"].startswith(after["restore-key"])) + + def test_source_only_revision_and_package_labels_do_not_bust_dependencies(self): + before = self.identity(GITHUB_SHA="one", PREVIEW_ID="first") + after = self.identity(GITHUB_SHA="two", PREVIEW_ID="second") + self.assertEqual(before["key"], after["key"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_stable_release.py b/scripts/tests/test_stable_release.py index 7dd1fd6..0163fcf 100644 --- a/scripts/tests/test_stable_release.py +++ b/scripts/tests/test_stable_release.py @@ -72,9 +72,15 @@ class StableReleaseTests(unittest.TestCase): root = Path(__file__).resolve().parents[2] workflow = (root / ".github/workflows/release.yml").read_text(encoding="utf-8") native = workflow.split(" native-tests:\n", 1)[1].split("\n linux:\n", 1)[0] - for platform in ("linux-x64", "windows-x64", "macos-arm64", "macos-x64"): - self.assertIn("platform: " + platform, native) - self.assertIn("run: python scripts/ci_native_tests.py", native) + self.assertIn("uses: ./.github/workflows/linux-lua.yml", native) + shared = (root / ".github/workflows/linux-lua.yml").read_text(encoding="utf-8") + for platform in ("ubuntu-24.04", "windows-2022", "macos-26", "macos-26-intel"): + self.assertIn(platform, shared) + self.assertIn("workflow_call:", shared) + self.assertIn("run: python scripts/ci_native_tests.py", shared) + self.assertIn("cargo check --locked --workspace --release", shared) + self.assertIn("tools/i18n-contract/Cargo.toml", shared) + self.assertNotIn("continue-on-error", shared) self.assertNotIn("continue-on-error", native) aggregate = workflow.split("\n aggregate:\n", 1)[1].split("\n publish:\n", 1)[0] self.assertIn("needs: [prepare, native-tests, linux, macos, windows]", aggregate)