-
Notifications
You must be signed in to change notification settings - Fork 769
[CI][Benchmarks] Automatically detect component versions in benchmark CI #18339
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
+465
−70
Merged
Changes from all commits
Commits
Show all changes
30 commits
Select commit
Hold shift + click to select a range
eb4e123
Add L0 driver code for detecting compute-runtime versions
ianayl 86263e1
instrument main with detect_versions
ianayl b21be5f
change wording, add frontend
ianayl 0c77780
Fix bug
ianayl 274be61
Hook up benchmark script to detect_versions
ianayl d8ab622
test changes in ci
ianayl 4cb45ce
Fix bug
ianayl 3648a35
Fix bug
ianayl 7d927f7
Remove test code
ianayl a447867
Remove more checking for args
ianayl c67e052
Remove some odd choices
ianayl d611471
Merge branch 'sycl' of https://github.com/intel/llvm into ianayl/benc…
ianayl a07c211
add newline
ianayl 3b0c996
Fix bug
ianayl 233c062
darker format python
ianayl 6915b3e
Apply clang-format
ianayl 2bb4959
Add a way to predefine a cache beforehand
ianayl 9fa6a1c
Add to workflow for testing
ianayl b651ca1
Remove debug messages
ianayl 083cff0
Fix spelling
ianayl 5b02887
apply clang-format
ianayl 54bb593
Add logging for detect versions
ianayl c0949da
fix indent
ianayl a1c75cd
fix typo
ianayl 633f51e
better signifier we are detecting versions
ianayl 23201fe
More logging for detect versions
ianayl 8a29481
More logging for detect versions
ianayl 12cdbf5
I broke darker
ianayl 2de4b33
apply formatting
ianayl 2dd72e4
Merge branch 'sycl' of https://github.com/intel/llvm into ianayl/benc…
ianayl File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| #include <cstdlib> | ||
| #include <cstring> | ||
| #include <iostream> | ||
| #include <vector> | ||
|
|
||
| #include <level_zero/ze_api.h> | ||
|
|
||
| #define _assert(cond, msg) \ | ||
| if (!(cond)) { \ | ||
| std::cout << std::endl << "Error: " << msg << std::endl; \ | ||
| exit(1); \ | ||
| } | ||
|
|
||
| #define _success(res) res == ZE_RESULT_SUCCESS | ||
|
|
||
| std::string query_dpcpp_ver() { return std::string(__clang_version__); } | ||
|
|
||
| std::string query_l0_driver_ver() { | ||
| // Initialize L0 drivers: | ||
| ze_init_driver_type_desc_t driver_type = {}; | ||
| driver_type.stype = ZE_STRUCTURE_TYPE_INIT_DRIVER_TYPE_DESC; | ||
| driver_type.flags = ZE_INIT_DRIVER_TYPE_FLAG_GPU; | ||
| driver_type.pNext = nullptr; | ||
|
|
||
| uint32_t driver_count = 0; | ||
| ze_result_t result = zeInitDrivers(&driver_count, nullptr, &driver_type); | ||
| _assert(_success(result), "Failed to initialize L0."); | ||
| _assert(driver_count > 0, "No L0 drivers available."); | ||
|
|
||
| std::vector<ze_driver_handle_t> drivers(driver_count); | ||
| result = zeInitDrivers(&driver_count, drivers.data(), &driver_type); | ||
| _assert(_success(result), "Could not fetch L0 drivers."); | ||
|
|
||
| // Check support for fetching driver version strings: | ||
| uint32_t ext_count = 0; | ||
| result = zeDriverGetExtensionProperties(drivers[0], &ext_count, nullptr); | ||
| _assert(_success(result), "Failed to obtain L0 extensions count."); | ||
| _assert(ext_count > 0, "No L0 extensions available."); | ||
|
|
||
| std::vector<ze_driver_extension_properties_t> extensions(ext_count); | ||
| result = | ||
| zeDriverGetExtensionProperties(drivers[0], &ext_count, extensions.data()); | ||
| _assert(_success(result), "Failed to obtain L0 extensions."); | ||
| bool version_ext_support = false; | ||
| for (const auto &extension : extensions) { | ||
| // std::cout << extension.name << std::endl; | ||
| if (strcmp(extension.name, "ZE_intel_get_driver_version_string")) { | ||
| version_ext_support = true; | ||
| } | ||
| } | ||
| _assert(version_ext_support, | ||
| "ZE_intel_get_driver_version_string extension is not supported."); | ||
|
|
||
| // Fetch L0 driver version: | ||
| ze_result_t (*pfnGetDriverVersionFn)(ze_driver_handle_t, char *, size_t *); | ||
| result = zeDriverGetExtensionFunctionAddress(drivers[0], | ||
| "zeIntelGetDriverVersionString", | ||
| (void **)&pfnGetDriverVersionFn); | ||
| _assert(_success(result), "Failed to obtain GetDriverVersionString fn."); | ||
|
|
||
| size_t ver_str_len = 0; | ||
| result = pfnGetDriverVersionFn(drivers[0], nullptr, &ver_str_len); | ||
| _assert(_success(result), "Call to GetDriverVersionString failed."); | ||
|
|
||
| std::cout << "ver_str_len: " << ver_str_len << std::endl; | ||
| ver_str_len++; // ver_str_len does not account for '\0' | ||
| char *ver_str = (char *)calloc(ver_str_len, sizeof(char)); | ||
| result = pfnGetDriverVersionFn(drivers[0], ver_str, &ver_str_len); | ||
| _assert(_success(result), "Failed to write driver version string."); | ||
|
|
||
| std::string res(ver_str); | ||
| free(ver_str); | ||
| return res; | ||
| } | ||
|
|
||
| int main() { | ||
| std::string dpcpp_ver = query_dpcpp_ver(); | ||
| std::cout << "DPCPP_VER='" << dpcpp_ver << "'" << std::endl; | ||
|
|
||
| std::string l0_ver = query_l0_driver_ver(); | ||
| std::cout << "L0_VER='" << l0_ver << "'" << std::endl; | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,250 @@ | ||
| import os | ||
| import re | ||
| import sys | ||
| import json | ||
| import urllib | ||
| import tempfile | ||
| import subprocess | ||
| from urllib import request | ||
| from pathlib import Path | ||
| import argparse | ||
|
|
||
| if __name__ == "__main__": | ||
| sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) | ||
| from options import options | ||
|
|
||
|
|
||
| def _get_patch_from_ver(ver: str) -> str: | ||
| """Extract patch from a version string.""" | ||
| # L0 version strings follows semver: major.minor.patch+optional | ||
| # compute-runtime version tags follow year.WW.patch.optional instead, | ||
| # but both follow a quasi-semver versioning where the patch, optional | ||
| # is still the same across both version string formats. | ||
| patch = re.sub(r"^\d+\.\d+\.", "", ver) | ||
| patch = re.sub(r"\+", ".", patch, count=1) | ||
| return patch | ||
|
|
||
|
|
||
| class DetectVersion: | ||
| _instance = None | ||
|
|
||
| def __init__(self): | ||
| raise RuntimeError("Use init() to init and instance() to get instead.") | ||
|
|
||
| @classmethod | ||
| def init(cls, detect_ver_path: Path, dpcpp_exec: str = "clang++"): | ||
| """ | ||
| Constructs the singleton instance for DetectVersion, and initializes by | ||
| building and run detect_version.cpp, which outputs: | ||
| - L0 driver version via ZE_intel_get_driver_version_string extension, | ||
| - DPC++ version via `__clang_version__` builtin. | ||
| Remind: DO NOT allow user input in args. | ||
| Parameters: | ||
| detect_ver_path (Path): Path to detect_version.cpp | ||
| dpcpp_exec (str): Name of DPC++ executable | ||
| """ | ||
| if cls._instance is not None: | ||
| return cls._instance | ||
|
|
||
| detect_ver_exe = tempfile.mktemp() | ||
| result = subprocess.run( | ||
| [dpcpp_exec, "-lze_loader", detect_ver_path, "-o", detect_ver_exe], | ||
| check=True, | ||
| env=os.environ, | ||
| ) | ||
| result = subprocess.run( | ||
| [detect_ver_exe], | ||
| check=True, | ||
| text=True, | ||
| capture_output=True, | ||
| env=os.environ, | ||
| ) | ||
| # Variables are printed to stdout, each var is on its own line | ||
| result_vars = result.stdout.strip().split("\n") | ||
|
|
||
| def get_var(var_name: str) -> str: | ||
| var_str = next( | ||
| filter(lambda v: re.match(f"^{var_name}='.*'", v), result_vars) | ||
| ) | ||
| return var_str[len(f"{var_name}='") : -len("'")] | ||
|
|
||
| cls._instance = cls.__new__(cls) | ||
| cls._instance.l0_ver = get_var("L0_VER") | ||
| cls._instance.dpcpp_ver = get_var("DPCPP_VER") | ||
| cls._instance.dpcpp_exec = dpcpp_exec | ||
|
|
||
| # Populate the computer-runtime version string cache: Since API calls | ||
| # are expensive, we want to avoid API calls when possible, i.e.: | ||
| # - Avoid a second API call if compute_runtime_ver was already obtained | ||
| # - Avoid an API call altogether if the user provides a valid | ||
| # COMPUTE_RUNTIME_TAG_CACHE environment variable. | ||
| cls._instance.compute_runtime_ver_cache = None | ||
| l0_ver_patch = _get_patch_from_ver(get_var("L0_VER")) | ||
| env_cache_ver = os.getenv("COMPUTE_RUNTIME_TAG_CACHE", default="") | ||
| env_cache_patch = _get_patch_from_ver(env_cache_ver) | ||
| # L0 patch often gets padded with 0's: if the environment variable | ||
| # matches up with the prefix of the l0 version patch, the cache is | ||
| # indeed referring to the same version. | ||
| if env_cache_patch == l0_ver_patch[: len(env_cache_patch)]: | ||
| print( | ||
| f"Using compute_runtime tag from COMPUTE_RUNTIME_TAG_CACHE: {env_cache_ver}" | ||
| ) | ||
| cls._instance.compute_runtime_ver_cache = env_cache_ver | ||
| else: | ||
| print( | ||
| f"Mismatch between COMPUTE_RUNTIME_TAG_CACHE {env_cache_ver} and patch reported by level_zero {get_var('L0_VER')}" | ||
| ) | ||
|
|
||
| return cls._instance | ||
|
|
||
| @classmethod | ||
| def instance(cls): | ||
| """ | ||
| Returns singleton instance of DetectVersion if it has been initialized | ||
| via init(), otherwise return None. | ||
| """ | ||
| return cls._instance | ||
|
|
||
| def get_l0_ver(self) -> str: | ||
| """ | ||
| Returns the full L0 version string. | ||
| """ | ||
| return self.l0_ver | ||
|
|
||
| def get_dpcpp_ver(self) -> str: | ||
| """ | ||
| Returns the full DPC++ version / clang version string of DPC++ used. | ||
| """ | ||
| return self.dpcpp_ver | ||
|
|
||
| def get_dpcpp_git_info(self) -> [str, str]: | ||
| """ | ||
| Returns: (git_repo, commit_hash) | ||
| """ | ||
| # clang++ formats are in <clang ver> (<git url> <commit>): if this | ||
| # regex does not match, it is likely this is not upstream clang. | ||
| git_info_match = re.search(r"\(http.+ [0-9a-f]+\)", self.dpcpp_ver) | ||
| if git_info_match is None: | ||
| raise RuntimeError( | ||
| f"detect_version: Unable to obtain git info from {self.dpcpp_exec}, are you sure you are using DPC++?" | ||
| ) | ||
| git_info = git_info_match.group(0) | ||
| return git_info[1:-1].split(" ") | ||
|
|
||
| def get_dpcpp_commit(self) -> str: | ||
| git_info = self.get_dpcpp_git_info() | ||
| if git_info is None: | ||
| return options.detect_versions.not_found_placeholder | ||
| return git_info[1] | ||
|
|
||
| def get_dpcpp_repo(self) -> str: | ||
| git_info = self.get_dpcpp_git_info() | ||
| if git_info is None: | ||
| return options.detect_versions.not_found_placeholder | ||
| return git_info[0] | ||
|
|
||
| def get_compute_runtime_ver_cached(self) -> str: | ||
| return self.compute_runtime_ver_cache | ||
|
|
||
| def get_compute_runtime_ver(self) -> str: | ||
ianayl marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| """ | ||
| Returns the compute-runtime version by deriving from l0 version. | ||
| """ | ||
| if self.compute_runtime_ver_cache is not None: | ||
| print( | ||
| f"Using cached compute-runtime tag {self.compute_runtime_ver_cache}..." | ||
| ) | ||
| return self.compute_runtime_ver_cache | ||
|
|
||
| patch = _get_patch_from_ver(self.l0_ver) | ||
|
|
||
| # TODO unauthenticated users only get 60 API calls per hour: this will | ||
| # not work if we enable benchmark CI in precommit. | ||
| url = options.detect_versions.compute_runtime_tag_api | ||
|
|
||
| print(f"Fetching compute-runtime tag from {url}...") | ||
| try: | ||
| for _ in range(options.detect_versions.max_api_calls): | ||
| res = request.urlopen(url) | ||
| tags = [tag["name"] for tag in json.loads(res.read())] | ||
|
|
||
| for tag in tags: | ||
| tag_patch = _get_patch_from_ver(tag) | ||
| # compute-runtime's cmake files produces "optional" fields | ||
| # padded with 0's: this means e.g. L0 version string | ||
| # 1.6.32961.200000 could be either compute-runtime ver. | ||
| # 25.09.32961.2, 25.09.32961.20, or even 25.09.32961.200. | ||
| # | ||
| # Thus, we take the longest match. Since the github api | ||
| # provides tags from newer -> older, we take the first tag | ||
| # that matches as it would be the "longest" ver. to match. | ||
| if tag_patch == patch[: len(tag_patch)]: | ||
| self.compute_runtime_ver_cache = tag | ||
| return tag | ||
|
|
||
| def get_link_name(link: str) -> str: | ||
| rel_str = re.search(r'rel="\w+"', link).group(0) | ||
| return rel_str[len('rel="') : -len('"')] | ||
|
|
||
| def get_link_url(link: str) -> str: | ||
| return link[link.index("<") + 1 : link.index(">")] | ||
|
|
||
| links = { | ||
| get_link_name(link): get_link_url(link) | ||
| for link in res.getheader("Link").split(", ") | ||
| } | ||
|
|
||
| if "next" in links: | ||
| url = links["next"] | ||
| else: | ||
| break | ||
|
|
||
| except urllib.error.HTTPError as e: | ||
| print(f"HTTP error {e.code}: {e.read().decode('utf-8')}") | ||
|
|
||
| except urllib.error.URLError as e: | ||
| print(f"URL error: {e.reason}") | ||
|
|
||
| print(f"WARNING: unable to find compute-runtime version") | ||
| return options.detect_versions.not_found_placeholder | ||
|
|
||
|
|
||
| def main(components: [str]): | ||
| detect_res = DetectVersion.init(f"{os.path.dirname(__file__)}/detect_versions.cpp") | ||
|
|
||
| str2fn = { | ||
| "dpcpp_repo": detect_res.get_dpcpp_repo, | ||
| "dpcpp_commit": detect_res.get_dpcpp_commit, | ||
| "l0_ver": detect_res.get_l0_ver, | ||
| "compute_runtime_ver": detect_res.get_compute_runtime_ver, | ||
| } | ||
|
|
||
| def remove_undefined_components(component: str) -> bool: | ||
| if component not in str2fn: | ||
| print(f"# Warn: unknown component: {component}", file=sys.stderr) | ||
| return False | ||
| return True | ||
|
|
||
| components_clean = filter(remove_undefined_components, components) | ||
|
|
||
| for s in map(lambda c: f"{c.upper()}={str2fn[c]()}", components_clean): | ||
| print(s) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| parser = argparse.ArgumentParser( | ||
| description="Get version information for specified components." | ||
| ) | ||
| parser.add_argument( | ||
| "components", | ||
| type=str, | ||
| help=""" | ||
| Comma-separated list of components to get version information for. | ||
| Valid options: dpcpp_repo,dpcpp_commit,l0_ver,compute_runtime_ver | ||
| """, | ||
| ) | ||
| args = parser.parse_args() | ||
|
|
||
| main(map(lambda c: c.strip(), args.components.split(","))) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is this envvar needed outside of this current script? if so we need to set it in a different way
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It is only needed in the main.py script ran immediately below this, which is why I haven't put it in GITHUB_ENV