Add patch based code coverage tool

Author: xgupta

llvm/utils/patch-coverage/README.md

@@ -0,0 +1,82 @@
+# Patch based Code Coverage git integration
+
+This tool provides patch-level code coverage integration for Git workflows in the LLVM project.
+It helps developers and reviewers to understand which lines introduced by patch are
+covered by tests and which are not.
+
+It is designed to integrate with LLVM’s existing infrastructure (e.g., llvm-lit,
+coverage instrumentation, and CI), and can be used in both local system system and LLVM GitHub
+CI to provide patch coverage for pull requests.
+
+# Overview
+Given a set of recent commits:  
+Extracts the patch (git diff)  
+Identifies modified source files and lines  
+Detects modified tests (lit-based)  
+Rebuilds LLVM with coverage instrumentation  
+Executes relevant tests using llvm-lit --per-test-coverage  
+Collects coverage data using llvm-profdata and llvm-cov  
+Reports:  
+Covered lines introduced by the patch  
+Uncovered lines introduced by the patch  
+
+
+# Setup
+Build LLVM (standard workflow):  
+```shell
+$ git clone https://github.com/llvm/llvm-project.git
+$ cd llvm-project
+$ mkdir build
+$ cmake -G Ninja -B build -S llvm \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_ENABLE_ASSERTIONS=ON
+    -DLLVM_INCLUDE_TESTS=ON
+    -DLLVM_BUILD_TESTS=ON
+
+$ ninja -C build
+```
+
+Add the tool into your path so that you can use it as git subcommand.  
+```shell
+$ export PATH=$PATH:llvm/utils/patch-coverage/ 
+```
+
+# Usage
+Run patch coverage for recent commits:  
+```shell
+$ git patch-coverage -b build opt build/test
+```
+Arguments  
+Argument	Description  
+-b, --build-dir	Path to build directory (default: build)  
+<binary>	Instrumented LLVM binary (e.g., bin/opt)  
+<test-path>	One or more test suite paths (default: build/test)  
+
+Optional Arguments  
+Argument	Description  
+-n, --num-commits	Number of recent commits to analyze (default: 1)  
+-i, --instrumented-build-dir	Use pre-instrumented build directory  
+
+# Example  
+```shell
+$ git patch-coverage -b build -n 3 bin/opt llvm/test
+```
+
+This will:  
+Generate a patch for the last 3 commits  
+Identify modified source and test files  
+Rebuild LLVM with coverage instrumentation (if needed)  
+Run modified lit tests  
+Produce per-line coverage for modified code  
+
+# Output
+The tool reports:  
+Modified files and lines  
+Covered lines (executed by tests)  
+Uncovered lines (not exercised)  
+Example output:  
+```
+Modified File: llvm/lib/IR/Example.cpp
+  Covered Line: 42 : foo();
+  Uncovered Line: 45 : bar();
+```

llvm/utils/patch-coverage/__init__.py

@@ -0,0 +1,163 @@
+import os
+import sys
+import subprocess
+from utils import compute_patch_hash
+from utils import log
+
+
+def is_configured_build(build_dir):
+    return os.path.exists(os.path.join(build_dir, "build.ninja")) or os.path.exists(
+        os.path.join(build_dir, "CMakeCache.txt")
+    )
+
+
+def configure_llvm_build(build_dir):
+    print(f"[patch-coverage] Configuring LLVM build in {build_dir}...")
+
+    subprocess.check_call(
+        [
+            "cmake",
+            "-G",
+            "Ninja",
+            "-S",
+            "llvm",
+            "-B",
+            build_dir,
+            "-DCMAKE_BUILD_TYPE=Release",
+            "-DLLVM_ENABLE_ASSERTIONS=ON",
+            "-DLLVM_INCLUDE_TESTS=ON",
+            "-DLLVM_BUILD_TESTS=ON",
+        ]
+    )
+
+
+def ensure_llvm_tools(build_dir):
+    if not is_configured_build(build_dir):
+        configure_llvm_build(build_dir)
+
+    required_tools = [
+        "bin/llvm-lit",
+        "bin/FileCheck",
+        "bin/count",
+        "bin/not",
+        "bin/llvm-config",
+    ]
+
+    missing = [
+        tool
+        for tool in required_tools
+        if not os.path.exists(os.path.join(build_dir, tool))
+    ]
+
+    if not missing:
+        return
+
+    subprocess.check_call(
+        [
+            "ninja",
+            "-C",
+            build_dir,
+            "FileCheck",
+            "count",
+            "not",
+            "llvm-config",
+        ]
+    )
+
+
+def should_rebuild(build_dir, patch_path):
+    new_hash = compute_patch_hash(patch_path)
+    hash_file = os.path.join(build_dir, ".last_patch_hash")
+
+    if os.path.exists(hash_file):
+        old_hash = open(hash_file).read().strip()
+        if old_hash == new_hash:
+            return False
+
+    with open(hash_file, "w") as f:
+        f.write(new_hash)
+
+    return True
+
+
+def build_llvm(build_dir, binary):
+    try:
+
+        cwd = os.getcwd()
+
+        os.chdir(build_dir)
+
+        command = 'find . -type f -name "*.profraw" -delete'
+
+        try:
+            subprocess.run(command, shell=True, check=True)
+            log(
+                "Files in build directory with '.profraw' extension deleted successfully."
+            )
+        except subprocess.CalledProcessError as e:
+            log(f"Error: {e}")
+        log("")
+
+        needs_cmake = True
+        cache_path = os.path.join(build_dir, "CMakeCache.txt")
+        if os.path.exists(cache_path):
+            with open(cache_path, "r") as f:
+                if "LLVM_BUILD_INSTRUMENTED_COVERAGE:BOOL=ON" in f.read():
+                    needs_cmake = False
+
+        if needs_cmake:
+            cmake_command = [
+                "cmake",
+                "-DLLVM_BUILD_INSTRUMENTED_COVERAGE=ON",
+                "-DLLVM_INDIVIDUAL_TEST_COVERAGE=ON",
+                f"-DCMAKE_C_FLAGS=-fprofile-list={os.path.abspath('fun.list')}",
+                f"-DCMAKE_CXX_FLAGS=-fprofile-list={os.path.abspath('fun.list')}",
+                ".",
+            ]
+
+            subprocess.check_call(cmake_command)
+
+        target_name = os.path.basename(binary)
+        try:
+            print()
+            subprocess.check_call(
+                [
+                    "ninja",
+                    "FileCheck",
+                    "count",
+                    "not",
+                    "llvm-config",
+                    "llvm-cov",
+                    "llvm-profdata",
+                    "llvm-dwarfdump",
+                    "llvm-readobj",
+                    target_name,
+                ]
+            )
+
+        except subprocess.CalledProcessError as ninja_error:
+            log(f"Error during Ninja build: {ninja_error}")
+            log("Attempting to build with 'make' using the available processors.")
+            num_processors = os.cpu_count() or 1
+            make_command = ["make", f"-j{num_processors}"]
+            subprocess.check_call(
+                make_command,
+                "FileCheck",
+                "count",
+                "not",
+                "llvm-config",
+                "llvm-cov",
+                "llvm-profdata",
+                "llvm-dwarfdump",
+                "llvm-readobj",
+                target_name,
+            )
+
+        os.chdir(cwd)
+
+        log("LLVM build completed successfully.")
+        log("")
+
+    except subprocess.CalledProcessError as e:
+        log("Error during LLVM build:", e)
+        sys.exit(1)

llvm/utils/patch-coverage/build.py

@@ -0,0 +1,2 @@
+#!/bin/bash
+python3 "$(dirname "$0")/main.py" "$@"

llvm/utils/patch-coverage/git-patch-coverage

@@ -0,0 +1,47 @@
+import subprocess
+import os
+
+
+def parse_suite_info(s):
+    curr_suite = None
+    res = {}
+
+    for line in s.decode().splitlines():
+        leading_spaces = len(line) - len(line.lstrip(" "))
+
+        if leading_spaces == 2 and line.split():
+            curr_suite = line.split()[0]
+        elif curr_suite is not None and leading_spaces == 4 and "Source Root:" in line:
+            assert curr_suite not in res
+
+            res[curr_suite] = line.split()[-1]
+
+    return res
+
+
+def find_lit_tests(lit_path, test_paths):
+    suites_cmd = [lit_path, "--show-suites"] + test_paths
+    output = subprocess.check_output(suites_cmd)
+
+    test_suites = parse_suite_info(output)
+
+    tests_cmd = [lit_path, "--show-tests"] + test_paths
+    output = subprocess.check_output(tests_cmd)
+
+    lines = [line.decode() for line in output.splitlines()]
+
+    test_info = [line.split() for line in lines if "::" in line]
+
+    if test_info is not None and len(test_info) > 0:
+        if len(test_info[0]) == 3:
+            return [
+                os.path.join(test_suites[suite], test_case)
+                for (suite, sep, test_case) in test_info
+            ]
+        elif len(test_info[0]) == 4:
+            return [
+                os.path.join(test_suites[suite1], test_case)
+                for (suite1, suite2, sep, test_case) in test_info
+            ]
+    else:
+        return []