[Utils] Add a python tool for test coverage of patches #17145
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This file defines a workflow that runs the libc++ benchmarks when a comment is added to the PR. | |
| # | |
| # The comment is of the form: | |
| # | |
| # /libcxx-bot benchmark <path-to-benchmarks-to-run> | |
| # | |
| # That will cause the specified benchmarks to be run on the PR and on the pull-request target, and | |
| # their results to be compared. | |
| name: "[libc++] Benchmark PR" | |
| permissions: | |
| contents: read | |
| on: | |
| issue_comment: | |
| types: | |
| - created | |
| - edited | |
| jobs: | |
| permission-check: | |
| if: >- | |
| github.event.issue.pull_request && | |
| contains(github.event.comment.body, '/libcxx-bot benchmark') | |
| runs-on: ubuntu-24.04 | |
| environment: | |
| name: main-branch-only | |
| deployment: false | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout: | | |
| .github/workflows/ | |
| - name: Check Permissions | |
| uses: ./.github/workflows/require-team-membership | |
| with: | |
| team-slug: llvm-committers | |
| LLVM_TOKEN_GENERATOR_CLIENT_ID: ${{ secrets.LLVM_TOKEN_GENERATOR_CLIENT_ID }} | |
| LLVM_TOKEN_GENERATOR_PRIVATE_KEY: ${{ secrets.LLVM_TOKEN_GENERATOR_PRIVATE_KEY }} | |
| extract-info: | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - permission-check | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Extract information from the PR | |
| id: vars | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: | | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.issue.number, | |
| }); | |
| core.setOutput('pr_base', pr.data.base.sha); | |
| core.setOutput('pr_head', pr.data.head.sha); | |
| const match = context.payload.comment.body.match(/\/libcxx-bot benchmark (.+)/); | |
| core.setOutput('benchmarks', match ? match[1] : ''); | |
| - name: Update comment with link to the run | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: | | |
| const { data: comment } = await github.rest.issues.getComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| }); | |
| const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const note = `> _Running benchmarks in ${run_url}_`; | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| body: [comment.body, note].join('\n\n'), | |
| }); | |
| outputs: | |
| pr_base: ${{ steps.vars.outputs.pr_base }} | |
| pr_head: ${{ steps.vars.outputs.pr_head }} | |
| benchmarks: ${{ steps.vars.outputs.benchmarks }} | |
| run-benchmarks: | |
| strategy: | |
| matrix: | |
| include: | |
| - platform: macOS 26.5 arm64 | |
| runner: ["self-hosted", "macOS", "ARM64", "26", "26.5"] | |
| cxx: clang++ | |
| running-on: macos | |
| xcode-version: '26.5' | |
| - platform: Linux x86_64 | |
| runner: llvm-premerge-libcxx-runners | |
| cxx: clang++-22 | |
| running-on: linux | |
| fail-fast: false | |
| permissions: | |
| pull-requests: write | |
| runs-on: ${{ matrix.runner }} | |
| needs: | |
| - extract-info | |
| env: | |
| BENCHMARKS: ${{ needs.extract-info.outputs.benchmarks }} | |
| COMPILER: ${{ matrix.cxx }} | |
| PLATFORM: ${{ matrix.platform }} | |
| PR_HEAD: ${{ needs.extract-info.outputs.pr_head }} | |
| PR_BASE: ${{ needs.extract-info.outputs.pr_base }} | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ needs.extract-info.outputs.pr_head }} | |
| fetch-depth: 0 | |
| fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main | |
| - name: Install Python | |
| if: ${{ matrix.running-on == 'linux' }} # installed via Homebrew on macOS | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: '3.14' | |
| - name: Select Xcode | |
| if: ${{ matrix.running-on == 'macos' }} | |
| run: echo "DEVELOPER_DIR=/Applications/Xcode_${{ matrix.xcode-version }}.app/Contents/Developer" >> $GITHUB_ENV | |
| - name: Install dependencies via Homebrew | |
| if: ${{ matrix.running-on == 'macos' }} | |
| run: | | |
| brew update | |
| brew install ninja cmake python@3.14 | |
| echo "$(brew --prefix python@3.14)/bin" >> "$GITHUB_PATH" | |
| - name: Diagnose tools in use | |
| run: | | |
| cmake --version | |
| ninja --version | |
| "${COMPILER}" --version | |
| python3 --version | |
| - name: Setup virtual environment | |
| run: | | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| python -m pip install -r libcxx/utils/requirements.txt | |
| - name: Build the baseline and the candidate | |
| run: | | |
| source .venv/bin/activate | |
| baseline_commit=$(git merge-base $PR_BASE $PR_HEAD) | |
| ./libcxx/utils/build-at-commit --commit ${baseline_commit} --install-dir install/baseline -- -DCMAKE_CXX_COMPILER="${COMPILER}" -DCMAKE_BUILD_TYPE=RelWithDebInfo | |
| ./libcxx/utils/build-at-commit --commit $PR_HEAD --install-dir install/candidate -- -DCMAKE_CXX_COMPILER="${COMPILER}" -DCMAKE_BUILD_TYPE=RelWithDebInfo | |
| - name: Run baseline and candidate interleaved | |
| run: | | |
| source .venv/bin/activate | |
| # Run 3 times so we can pick the median, and interleave to mitigate the impact of environmental noise | |
| ./libcxx/utils/test-at-commit --libcxx-installation install/baseline -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS" | |
| ./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee baseline.lnt | |
| ./libcxx/utils/test-at-commit --libcxx-installation install/candidate -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS" | |
| ./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee candidate.lnt | |
| ./libcxx/utils/test-at-commit --libcxx-installation install/baseline -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS" | |
| ./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee -a baseline.lnt | |
| ./libcxx/utils/test-at-commit --libcxx-installation install/candidate -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS" | |
| ./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee -a candidate.lnt | |
| ./libcxx/utils/test-at-commit --libcxx-installation install/baseline -B benchmarks/baseline --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS" | |
| ./libcxx/utils/consolidate-benchmarks benchmarks/baseline | tee -a baseline.lnt | |
| ./libcxx/utils/test-at-commit --libcxx-installation install/candidate -B benchmarks/candidate --compiler "${COMPILER}" -- -sv -j1 --param optimization=speed "$BENCHMARKS" | |
| ./libcxx/utils/consolidate-benchmarks benchmarks/candidate | tee -a candidate.lnt | |
| - name: Compare baseline and candidate runs | |
| run: | | |
| source .venv/bin/activate | |
| ./libcxx/utils/compare-benchmarks baseline.lnt candidate.lnt | tee results.txt | |
| - name: Update comment with results | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const results = fs.readFileSync('results.txt', 'utf8'); | |
| const { data: comment } = await github.rest.issues.getComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| }); | |
| const details = [ | |
| '<details>', | |
| '<summary>', | |
| `Benchmark results for ${process.env.PLATFORM}:`, | |
| '</summary>', | |
| '', | |
| '```', | |
| results, | |
| '```', | |
| '', | |
| '</details>', | |
| ].join('\n'); | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| body: [comment.body, details].join('\n\n'), | |
| }); | |
| - name: Report failure in the comment | |
| if: failure() | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: | | |
| const { data: comment } = await github.rest.issues.getComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| }); | |
| const run_url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const note = `> _:x: Benchmarks for ${process.env.PLATFORM} failed. See ${run_url} for details._`; | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| body: [comment.body, note].join('\n\n'), | |
| }); |