mirror-llvm-project/.github/workflows/libcxx-pr-benchmark.yml

# This file defines a workflow that runs the libc++ benchmarks when a comment is added to the PR.
#
# The comment is of the form:
#
#   /libcxx-bot benchmark <path-to-benchmarks-to-run>
#
# That will cause the specified benchmarks to be run on the PR and on the pull-request target, and
# their results to be compared.

name: "[libc++] Benchmark PR"

permissions:
  contents: read

on:
  issue_comment:
    types:
      - created
      - edited

jobs:
  permission-check:
    if: >-
      github.event.issue.pull_request &&
      contains(github.event.comment.body, '/libcxx-bot benchmark')      
    runs-on: ubuntu-24.04
    environment:
      name: main-branch-only
      deployment: false
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
          sparse-checkout: |
            .github/workflows/            

      - name: Check Permissions
        uses: ./.github/workflows/require-team-membership
        with:
          team-slug: llvm-committers
          LLVM_TOKEN_GENERATOR_CLIENT_ID: ${{ secrets.LLVM_TOKEN_GENERATOR_CLIENT_ID }}
          LLVM_TOKEN_GENERATOR_PRIVATE_KEY: ${{ secrets.LLVM_TOKEN_GENERATOR_PRIVATE_KEY }}

  extract-info:
    runs-on: ubuntu-24.04
    needs:
      - permission-check
    permissions:
      pull-requests: write
    steps:
      - name: Extract information from the PR
        id: vars
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const pr = await github.rest.pulls.get({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: context.issue.number,
            });
            core.setOutput('pr_base', pr.data.base.sha);
            core.setOutput('pr_head', pr.data.head.sha);
            const match = context.payload.comment.body.match(/\/libcxx-bot benchmark (.+)/);
            core.setOutput('benchmarks', match ? match[1] : '');            

      - name: Update comment with link to the run
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const { data: comment } = await github.rest.issues.getComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: context.payload.comment.id,
            });
            const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const note = `> _Running benchmarks in ${run_url}_`;
            await github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: context.payload.comment.id,
              body: [comment.body, note].join('\n\n'),
            });            
    outputs:
      pr_base: ${{ steps.vars.outputs.pr_base }}
      pr_head: ${{ steps.vars.outputs.pr_head }}
      benchmarks: ${{ steps.vars.outputs.benchmarks }}

  run-benchmarks:
    strategy:
      matrix:
        include:
          - platform: macOS 26.5 arm64
            runner: ["self-hosted", "macOS", "ARM64", "26", "26.5"]
            cxx: clang++
            running-on: macos
            xcode-version: '26.5'
          - platform: Linux x86_64
            runner: llvm-premerge-libcxx-runners
            cxx: clang++-22
            running-on: linux
      fail-fast: false
    permissions:
      pull-requests: write
    runs-on: ${{ matrix.runner }}
    needs:
      - extract-info
    env:
      BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }}
      COMPILER: ${{ matrix.cxx }}
      PLATFORM: ${{ matrix.platform }}
      PR_HEAD: ${{ needs.extract-info.outputs.pr_head }}
      PR_BASE: ${{ needs.extract-info.outputs.pr_base }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
          ref: ${{ needs.extract-info.outputs.pr_head }}
          fetch-depth: 0
          fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main

      - name: Install Python
        if: ${{ matrix.running-on == 'linux' }} # installed via Homebrew on macOS
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
          python-version: '3.14'

      - name: Select Xcode
        if: ${{ matrix.running-on == 'macos' }}
        run: echo "DEVELOPER_DIR=/Applications/Xcode_${{ matrix.xcode-version }}.app/Contents/Developer" >> $GITHUB_ENV

      - name: Install dependencies via Homebrew
        if: ${{ matrix.running-on == 'macos' }}
        run: |
          brew update
          brew install ninja cmake python@3.14
          echo "$(brew --prefix python@3.14)/bin" >> "$GITHUB_PATH"          

      - name: Diagnose tools in use
        run: |
          cmake --version
          ninja --version
          "${COMPILER}" --version
          python3 --version          

      - name: Setup virtual environment
        run: |
          python3 -m venv .venv
          source .venv/bin/activate
          python -m pip install -r libcxx/utils/requirements.txt          

      - name: Build the baseline and the candidate
        run: |
          source .venv/bin/activate
          baseline_commit=$(git merge-base $PR_BASE $PR_HEAD)
          ./libcxx/utils/build-at-commit --commit ${baseline_commit} --install-dir install/baseline -- -DCMAKE_CXX_COMPILER="${COMPILER}" -DCMAKE_BUILD_TYPE=RelWithDebInfo
          ./libcxx/utils/build-at-commit --commit $PR_HEAD --install-dir install/candidate -- -DCMAKE_CXX_COMPILER="${COMPILER}" -DCMAKE_BUILD_TYPE=RelWithDebInfo          

      - name: Run baseline and candidate interleaved
        run: |
          source .venv/bin/activate
          # Run 3 times so we can pick the median, and interleave to mitigate the impact of environmental noise
          ./libcxx/utils/test-at-commit --libcxx-installation install/baseline -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS"
          ./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee baseline.lnt
          ./libcxx/utils/test-at-commit --libcxx-installation install/candidate -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS"
          ./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee candidate.lnt

          ./libcxx/utils/test-at-commit --libcxx-installation install/baseline -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS"
          ./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee -a baseline.lnt
          ./libcxx/utils/test-at-commit --libcxx-installation install/candidate -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS"
          ./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee -a candidate.lnt

          ./libcxx/utils/test-at-commit --libcxx-installation install/baseline -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS"
          ./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee -a baseline.lnt
          ./libcxx/utils/test-at-commit --libcxx-installation install/candidate -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS"
          ./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee -a candidate.lnt          

      - name: Compare baseline and candidate runs
        run: |
          source .venv/bin/activate
          ./libcxx/utils/compare-benchmarks baseline.lnt candidate.lnt | tee results.txt          

      - name: Update comment with results
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const fs = require('fs');
            const results = fs.readFileSync('results.txt', 'utf8');
            const { data: comment } = await github.rest.issues.getComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: context.payload.comment.id,
            });
            const details = [
              '<details>',
              '<summary>',
              `Benchmark results for ${process.env.PLATFORM}:`,
              '</summary>',
              '',
              '```',
              results,
              '```',
              '',
              '</details>',
            ].join('\n');
            await github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: context.payload.comment.id,
              body: [comment.body, details].join('\n\n'),
            });            

      - name: Report failure in the comment
        if: failure()
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const { data: comment } = await github.rest.issues.getComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: context.payload.comment.id,
            });
            const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const note = `> _:x: Benchmarks for ${process.env.PLATFORM} failed. See ${run_url} for details._`;
            await github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: context.payload.comment.id,
              body: [comment.body, note].join('\n\n'),
            });