diff --git a/.ci/compute_projects.py b/.ci/compute_projects.py index fd97773e329e..42d1524d638e 100644 --- a/.ci/compute_projects.py +++ b/.ci/compute_projects.py @@ -132,11 +132,12 @@ EXCLUDE_DEPENDENTS_WINDOWS = { EXCLUDE_MAC = { "bolt", - "compiler-rt", + "CIR", # Depends on mlir, which is excluded below. "cross-project-tests", "flang", + "flang-rt", "libc", - "lldb", + "mlir", "openmp", "polly", "libcxx", @@ -145,6 +146,13 @@ EXCLUDE_MAC = { "offload", } +# These projects are still built on the self-hosted macOS runners, but their +# tests are temporarily skipped there. +EXCLUDE_CHECK_TARGETS_MAC = { + "lldb", + "libclc", +} + PROJECT_CHECK_TARGETS = { "clang-tools-extra": "check-clang-tools", "compiler-rt": "check-compiler-rt", @@ -262,6 +270,8 @@ def _compute_project_check_targets( ) -> Set[str]: check_targets = set() for project_to_test in projects_to_test: + if platform == "Darwin" and project_to_test in EXCLUDE_CHECK_TARGETS_MAC: + continue if project_to_test in PROJECT_CHECK_TARGETS: check_targets.add(PROJECT_CHECK_TARGETS[project_to_test]) return check_targets diff --git a/.ci/compute_projects_test.py b/.ci/compute_projects_test.py index 3989c2d92026..c0c9cc100d91 100644 --- a/.ci/compute_projects_test.py +++ b/.ci/compute_projects_test.py @@ -62,11 +62,11 @@ class TestComputeProjects(unittest.TestCase): ) self.assertEqual( env_variables["projects_to_build"], - "clang;clang-tools-extra;lld;llvm;mlir", + "clang;clang-tools-extra;lld;lldb;llvm", ) self.assertEqual( env_variables["project_check_targets"], - "check-clang check-clang-python check-clang-tools check-lld check-llvm check-mlir", + "check-clang check-clang-python check-clang-tools check-lld check-llvm", ) self.assertEqual(env_variables["runtimes_to_build"], "") self.assertEqual( @@ -78,6 +78,54 @@ class TestComputeProjects(unittest.TestCase): "", ) + def test_mlir_mac(self): + env_variables = compute_projects.get_env_variables( + ["mlir/CMakeLists.txt"], "Darwin" + ) + self.assertEqual(env_variables["projects_to_build"], "") + self.assertEqual(env_variables["project_check_targets"], "") + self.assertEqual(env_variables["runtimes_to_build"], "") + self.assertEqual(env_variables["runtimes_check_targets"], "") + self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "") + self.assertEqual(env_variables["enable_cir"], "OFF") + + def test_cir_mac(self): + env_variables = compute_projects.get_env_variables( + ["clang/lib/CIR/CMakeLists.txt"], "Darwin" + ) + self.assertEqual( + env_variables["projects_to_build"], + "clang;clang-tools-extra;lld;lldb;llvm", + ) + self.assertEqual( + env_variables["project_check_targets"], + "check-clang check-clang-python check-clang-tools", + ) + self.assertEqual(env_variables["runtimes_to_build"], "compiler-rt") + self.assertEqual(env_variables["runtimes_check_targets"], "check-compiler-rt") + self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "") + self.assertEqual(env_variables["enable_cir"], "OFF") + + def test_lldb_mac(self): + env_variables = compute_projects.get_env_variables( + ["lldb/CMakeLists.txt"], "Darwin" + ) + self.assertEqual(env_variables["projects_to_build"], "clang;lld;lldb;llvm") + self.assertEqual(env_variables["project_check_targets"], "") + self.assertEqual(env_variables["runtimes_to_build"], "compiler-rt") + self.assertEqual(env_variables["runtimes_check_targets"], "") + self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "") + + def test_libclc_mac(self): + env_variables = compute_projects.get_env_variables( + ["libclc/CMakeLists.txt"], "Darwin" + ) + self.assertEqual(env_variables["projects_to_build"], "clang;llvm") + self.assertEqual(env_variables["project_check_targets"], "") + self.assertEqual(env_variables["runtimes_to_build"], "") + self.assertEqual(env_variables["runtimes_check_targets"], "") + self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "") + def test_clang(self): env_variables = compute_projects.get_env_variables( ["clang/CMakeLists.txt"], "Linux" diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml index d506e631e2f2..2203084ae36d 100644 --- a/.github/workflows/premerge.yaml +++ b/.github/workflows/premerge.yaml @@ -199,52 +199,130 @@ jobs: path: | comments-Windows-AMD64 - premerge-check-macos: - name: MacOS Premerge Checks - runs-on: macos-14 + # Computes which projects/check-targets to build and test on the + # self-hosted macOS runners. This job intentionally runs on a + # GitHub-hosted runner rather than a self-hosted one, since self-hosted + # macOS capacity is limited and shouldn't be spent just to compute what + # to build. Its outputs feed premerge-check-macos below, which does the + # actual build/test on the self-hosted runners, so changes here directly + # affect what runs there. + premerge-compute-macos: + name: Compute macOS Projects if: >- - github.repository_owner == 'llvm' && - (startswith(github.ref_name, 'release/') || - startswith(github.base_ref, 'release/')) && - (github.event_name != 'pull_request' || github.event.action != 'closed') + github.repository_owner == 'llvm' && + (github.event_name != 'pull_request' || github.event.action != 'closed') + runs-on: ubuntu-22.04 + permissions: + contents: read + pull-requests: read + outputs: + macos-projects: ${{ steps.vars.outputs.macos-projects }} + macos-check-targets: ${{ steps.vars.outputs.macos-check-targets }} + steps: + # For pull requests we get the changed files from the GitHub API, so we + # only need a sparse checkout of .ci/ to run compute_projects.py. For + # pushes (e.g. release branches) there is no PR to query, so we fall + # back to a git diff, which needs real history. + - name: Checkout LLVM (sparse) + if: github.event_name == 'pull_request' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: .ci + sparse-checkout-cone-mode: false + - name: Checkout LLVM + if: github.event_name != 'pull_request' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 2 + - name: Get Changed Files + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename' > /tmp/changed_files.txt + else + git diff --name-only HEAD~1...HEAD > /tmp/changed_files.txt + fi + - name: Compute Projects + id: vars + run: | + python3 .ci/compute_projects.py Darwin < /tmp/changed_files.txt > /tmp/env_vars.sh + source /tmp/env_vars.sh + + echo "Building projects: ${projects_to_build}" + echo "Running project checks targets: ${project_check_targets}" + echo "Building runtimes: ${runtimes_to_build}" + echo "Running runtimes checks targets: ${runtimes_check_targets}" + echo "Running runtimes checks requiring reconfiguring targets: ${runtimes_check_targets_needs_reconfig}" + + echo "macos-projects=${projects_to_build}" >> $GITHUB_OUTPUT + echo "macos-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT + + premerge-check-macos: + name: Build and Test macOS arm64 + needs: premerge-compute-macos + runs-on: [self-hosted, macos, apple-runners] + # Skip the self-hosted runner entirely if there is nothing to build, + # rather than tying up a runner slot for a no-op run. + if: >- + github.repository_owner == 'llvm' && + (github.event_name != 'pull_request' || github.event.action != 'closed') && + needs.premerge-compute-macos.outputs.macos-projects != '' steps: - name: Checkout LLVM uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false fetch-depth: 2 - - name: Setup ccache - uses: hendrikmuhs/ccache-action@33522472633dbd32578e909b315f5ee43ba878ce # v1.2.22 - with: - max-size: "2000M" - - name: Install Ninja + - name: Install Ninja, CMake, and sccache run: | - brew install ninja + brew install ninja cmake + command -v sccache >/dev/null || brew install sccache - name: Build and Test + timeout-minutes: 240 + env: + SCCACHE_BUCKET: sccache-bucket + SCCACHE_REGION: us-west-2 + PROJECTS: ${{ needs.premerge-compute-macos.outputs.macos-projects }} + CHECK_TARGETS: ${{ needs.premerge-compute-macos.outputs.macos-check-targets }} + CLANG_TOOLCHAIN_PROGRAM_TIMEOUT: 120 run: | - source <(git diff --name-only HEAD~1...HEAD | python3 .ci/compute_projects.py) + export SCCACHE_IDLE_TIMEOUT=0 + mkdir -p artifacts + SCCACHE_LOG=info SCCACHE_ERROR_LOG=$(pwd)/artifacts/sccache.log sccache --start-server - if [[ "${projects_to_build}" == "" ]]; then - echo "No projects to build" - exit 0 - fi - - echo "Building projects: ${projects_to_build}" - echo "Running project checks targets: ${project_check_targets}" - - # -DLLVM_DISABLE_ASSEMBLY_FILES=ON is for - # https://github.com/llvm/llvm-project/issues/81967 - # Disable sharding in lit so that the LIT_XFAIL environment var works. - cmake -G Ninja \ + xcrun cmake -G Ninja \ -B build \ -S llvm \ - -DLLVM_ENABLE_PROJECTS="${projects_to_build}" \ + -DLLVM_ENABLE_PROJECTS="${PROJECTS}" \ -DLLVM_DISABLE_ASSEMBLY_FILES=ON \ -DCMAKE_BUILD_TYPE=Release \ -DLLDB_INCLUDE_TESTS=OFF \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - # The libcxx tests fail, so we are skipping the runtime targets. - ninja -C build ${project_check_targets} + ninja -C build ${CHECK_TARGETS} + - name: Show sccache stats + if: always() + run: | + mkdir -p artifacts + sccache --show-stats + sccache --show-stats >> artifacts/sccache_stats.txt + - name: Upload Artifacts + # In some cases, Github will fail to upload the artifact. We want to + # continue anyways as a failed artifact upload is an infra failure, not + # a checks failure. + # https://github.com/actions/upload-artifact/issues/569 + continue-on-error: true + if: '!cancelled()' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: Premerge Artifacts (macOS arm64) + path: artifacts/ + retention-days: 5 + include-hidden-files: 'true'