[Utils] Add a python tool for test coverage of patches #28657
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # When /test-suite is commented on a PR, checks out the PR, builds clang and | |
| # then the test-suite in several configurations. It then checks out the base of | |
| # the PR, builds clang and the test-suite again, and then uploads the diff of | |
| # the codegen. | |
| name: Diff test-suite codegen | |
| on: | |
| issue_comment: | |
| types: | |
| - created | |
| permissions: | |
| contents: read | |
| jobs: | |
| permission-check: | |
| if: github.event.issue.pull_request && startswith(github.event.comment.body, '/test-suite') | |
| runs-on: ubuntu-24.04 | |
| environment: | |
| name: main-branch-only | |
| deployment: false | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout: | | |
| .github/workflows/ | |
| - name: Check Permissions | |
| uses: ./.github/workflows/require-team-membership | |
| with: | |
| team-slug: llvm-committers | |
| LLVM_TOKEN_GENERATOR_CLIENT_ID: ${{ secrets.LLVM_TOKEN_GENERATOR_CLIENT_ID }} | |
| LLVM_TOKEN_GENERATOR_PRIVATE_KEY: ${{ secrets.LLVM_TOKEN_GENERATOR_PRIVATE_KEY }} | |
| react-to-comment: | |
| needs: | |
| - permission-check | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Thumbs up comment | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: | | |
| github.rest.reactions.createForIssueComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| content: '+1' | |
| }) | |
| test-suite: | |
| name: Build test-suite and diff | |
| runs-on: ubuntu-24.04 | |
| container: | |
| image: ghcr.io/llvm/ci-ubuntu-24.04:35e958d7f0b7 | |
| needs: | |
| - permission-check | |
| env: | |
| # llvm-nm/llvm-ar/llvm-ranlib are needed for the microbenchmarks | |
| NINJA_TARGETS: clang lld llvm-size llvm-nm llvm-ar llvm-ranlib | |
| steps: | |
| - name: Get pull request | |
| id: get-pr | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: | | |
| const { data: pr } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.payload.issue.number | |
| }) | |
| return pr | |
| - name: Check pull request is mergeable | |
| if: ${{ !fromJSON(steps.get-pr.outputs.result).mergeable }} | |
| run: | | |
| cat << EOF > comments | |
| [{"body": "Unable to diff test-suite with PR, PR isn't mergeable"}] | |
| EOF | |
| exit 1 | |
| - name: Checkout pull request | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| ref: ${{ fromJSON(steps.get-pr.outputs.result).merge_commit_sha }} | |
| repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }} | |
| fetch-depth: 2 | |
| path: llvm-project | |
| persist-credentials: false | |
| - name: Checkout llvm/llvm-test-suite | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| repository: llvm/llvm-test-suite | |
| path: llvm-test-suite | |
| persist-credentials: false | |
| - name: Setup environment variables | |
| run: | | |
| echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
| echo "BASE_SHA=$(git rev-parse HEAD^)" >> $GITHUB_ENV | |
| SCRIPTS_DIR="$GITHUB_WORKSPACE/llvm-project/.github/workflows/test-suite" | |
| echo "SCRIPTS_DIR=$SCRIPTS_DIR" >> $GITHUB_ENV | |
| echo "$SCRIPTS_DIR" >> $GITHUB_PATH | |
| echo "$GITHUB_WORKSPACE/llvm-project/build/bin" >> $GITHUB_PATH | |
| working-directory: llvm-project | |
| - name: Install system dependencies | |
| run: | | |
| # Install newer version of CMake (llvm/ci-ubuntu-24.04 image has CMake 3.28) | |
| # TODO(boomanaiden154): Remove once upgraded to Ubuntu 26.04 | |
| sudo apt-get update | |
| sudo apt-get install -y wget | |
| wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null | |
| echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main' | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null | |
| sudo apt-get update | |
| sudo apt-get install -y cmake tcl libc6-dev-arm64-cross libc6-dev-riscv64-cross libgcc-14-dev-arm64-cross libgcc-14-dev-riscv64-cross libstdc++-14-dev-arm64-cross libstdc++-14-dev-riscv64-cross python3-pandas python3-tabulate | |
| - name: Configure Clang | |
| run: cmake -B build -C $SCRIPTS_DIR/llvm.cmake llvm -GNinja | |
| working-directory: llvm-project | |
| - name: Build Clang @ head | |
| run: ninja -C build $NINJA_TARGETS | |
| working-directory: llvm-project | |
| - name: Configure and build test-suite @ head | |
| run: | | |
| configure-and-build.sh rva23u64-O3-head $SCRIPTS_DIR/riscv64.cmake | |
| configure-and-build.sh armv9-a-O3-head $SCRIPTS_DIR/aarch64.cmake | |
| configure-and-build.sh x86-64-v3-O3-head $SCRIPTS_DIR/x86_64.cmake | |
| working-directory: llvm-test-suite | |
| - name: Build Clang @ base | |
| run: git checkout $BASE_SHA && ninja -C build $NINJA_TARGETS | |
| working-directory: llvm-project | |
| - name: Configure and build test-suite @ base | |
| run: | | |
| configure-and-build.sh rva23u64-O3-base $SCRIPTS_DIR/riscv64.cmake | |
| configure-and-build.sh armv9-a-O3-base $SCRIPTS_DIR/aarch64.cmake | |
| configure-and-build.sh x86-64-v3-O3-base $SCRIPTS_DIR/x86_64.cmake | |
| working-directory: llvm-test-suite | |
| - name: Compute diffs | |
| run: | | |
| mkdir diffs | |
| ./utils/tdiff.py -a build.rva23u64-O3-base -b build.rva23u64-O3-head -s all > diffs/rva23u64-O3.diff || true | |
| ./utils/tdiff.py -a build.armv9-a-O3-base -b build.armv9-a-O3-head -s all > diffs/armv9-a-O3.diff || true | |
| ./utils/tdiff.py -a build.x86-64-v3-O3-base -b build.x86-64-v3-O3-head -s all > diffs/x86-64-v3-O3.diff || true | |
| working-directory: llvm-test-suite | |
| - name: Upload diffs | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| id: upload-diffs | |
| with: | |
| name: diffs | |
| path: llvm-test-suite/diffs | |
| - name: Upload results | |
| id: upload-results | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: results | |
| path: llvm-test-suite/results*.json | |
| - name: Compare results | |
| run: | | |
| for ARCH in rva23u64-O3 armv9-a-O3 x86-64-v3-O3 | |
| do | |
| ./utils/compare.py results.$ARCH-base.json vs results.$ARCH-head.json -m size..text --only-changed --lhs-name base --rhs-name head --format=md > ../compare-$ARCH.md || true | |
| done | |
| working-directory: llvm-test-suite | |
| - name: Create comment | |
| env: | |
| ARTIFACT_URL: ${{ steps.upload-diffs.outputs.artifact-url }} | |
| RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }} | |
| run: | | |
| cat << EOF > comment-body | |
| test-suite diff from $BASE_SHA...$HEAD_SHA: $ARTIFACT_URL | |
| Statistics: $RESULTS_URL | |
| ## riscv64-linux-gnu \`-march=rva23u64 -O3\` | |
| $(cat compare-rva23u64-O3.md) | |
| ## aarch64-linux-gnu \`-march=armv9-a -O3\` | |
| $(cat compare-armv9-a-O3.md) | |
| ## x86_64-linux-gnu \`-march=x86-64-v3 -O3\` | |
| $(cat compare-x86-64-v3-O3.md) | |
| EOF | |
| # write escaped markdown to format expected by issue-write.yml | |
| python3 -c 'import json; print(json.dumps([{"body": open("comment-body").read()}]))' > comments | |
| - name: Create comment on failure | |
| if: failure() | |
| env: | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| cat << EOF > comments | |
| [{"body" : "Failed to get test-suite diff from $BASE_SHA...$HEAD_SHA: $RUN_URL"}] | |
| EOF | |
| - name: Save PR number | |
| if: always() | |
| env: | |
| PR_NUMBER: ${{ fromJSON(steps.get-pr.outputs.result).number }} | |
| run: echo $PR_NUMBER > pr_number | |
| - name: Upload comment and PR number | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: always() | |
| with: | |
| name: workflow-args | |
| path: | | |
| comments | |
| pr_number |