diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py index b0d8e7149e553..87a478a6f184c 100644 --- a/llvm/utils/lit/lit/TestingConfig.py +++ b/llvm/utils/lit/lit/TestingConfig.py @@ -2,6 +2,27 @@ import sys +# config.available_features is a list. This class behaves like a normal list, +# but its "add()" method also checks if the feature being added is known, i.e. +# contained in the set of all alowed features. +class AvailableFeatures(set): + def __init__(self, *args, **kwargs): + from sycl_lit_allowed_features import get_sycl_lit_allowed_features + allowed_features = get_sycl_lit_allowed_features() + super().__init__(*args, **kwargs) + self.allowed_features = allowed_features + + def add(self, entry): + if entry in self.allowed_features: + super().add(entry) + else: + raise ValueError( + "feature " + + entry + + " is not allowed. Please update the set in sycl/test-e2e/sycl_lit_allowed_features.py." + ) + + class TestingConfig(object): """ TestingConfig - Information on the tests inside a suite. diff --git a/sycl/test-e2e/lit.cfg.py b/sycl/test-e2e/lit.cfg.py index 55688b5a29928..b78cc95bd051f 100644 --- a/sycl/test-e2e/lit.cfg.py +++ b/sycl/test-e2e/lit.cfg.py @@ -14,6 +14,7 @@ from lit.llvm import llvm_config from lit.llvm.subst import ToolSubst, FindTool + # Configuration file for the 'lit' test runner. config.backend_to_target = { "level_zero": "target-spir", @@ -33,6 +34,8 @@ k: config.target_to_triple.get(v) for k, v in config.backend_to_target.items() } +config.available_features = AvailableFeatures() + # name: The name of this test suite. config.name = "SYCL" @@ -227,6 +230,7 @@ def get_device_family_from_arch(arch): return device_family return None + def check_igc_tag_and_add_feature(): if os.path.isfile(config.igc_tag_file): with open(config.igc_tag_file, "r") as tag_file: @@ -254,6 +258,7 @@ def check_igc_tag_and_add_feature(): if lit_config.params.get("spirv-backend", False): config.available_features.add("spirv-backend") + # Use this to make sure that any dynamic checks below are done in the build # directory and not where the sources are located. This is important for the # in-tree configuration (as opposite to the standalone one). @@ -781,7 +786,7 @@ def open_check_file(file_name): gpu_intel_pvc_2T_device_id = "3029" _, device_id = line.strip().split(":", 1) device_id = device_id.strip() - if device_id == gpu_intel_pvc_1T_device_id: + if device_id == gpu_intel_pvc_1T_device_id: config.available_features.add("gpu-intel-pvc-1T") if device_id == gpu_intel_pvc_2T_device_id: config.available_features.add("gpu-intel-pvc-2T") diff --git a/sycl/test-e2e/sycl_lit_allowed_features.py b/sycl/test-e2e/sycl_lit_allowed_features.py new file mode 100644 index 0000000000000..f21da1151a666 --- /dev/null +++ b/sycl/test-e2e/sycl_lit_allowed_features.py @@ -0,0 +1,134 @@ +# This script provides the complete set of all possible features that can be +# used in XFAIL, UNSUPPORTED and REQUIRES. + +# To use: +# from sycl_lit_allowed_features import get_sycl_lit_allowed_features +# allowed_features = get_sycl_lit_allowed_features() + +# Note: +# The set below (partial_set_of_features) is maintained manually. If the new +# feature is NOT an aspect or an architecture - it should be added to this set. +# And vice versa - if the feature was deleted, it also should be deleted from +# this set, otherwise the feature is still treated as valid. +# +# Aspects and device architectures are added automatically and require no +# additional changes. + +import os + +partial_set_of_features = { + # for completely disabled tests + "true", + # host OS: + "windows", + "system-windows", + "linux", + "system-linux", + # target device: + "cpu", + "gpu", + "accelerator", + # target backend: + "cuda", + "hip", + "opencl", + "level_zero", + "level-zero", + "native_cpu", + # target: + "target-nvidia", + "target-native_cpu", + "target-amd", + "target-spir", + "spirv-backend", + # tools: + "sycl-ls", + "cm-compiler", + "ocloc", + "opencl-aot", + "llvm-spirv", + "llvm-link", + # dev-kits: + "level_zero_dev_kit", + "cuda_dev_kit", + # manually-set features (deprecated, no new tests should use these features) + "gpu-intel-gen11", + "gpu-intel-gen12", + "gpu-intel-dg1", + "gpu-intel-dg2", + "gpu-intel-pvc", + "gpu-intel-pvc-vg", + "gpu-intel-pvc-1T", + "gpu-intel-pvc-2T", + "gpu-amd-gfx90a", + # any-device-is-: + "any-device-is-cpu", + "any-device-is-gpu", + "any-device-is-accelerator", + "any-device-is-cuda", + "any-device-is-hip", + "any-device-is-opencl", + "any-device-is-level_zero", + "any-device-is-native_cpu", + # any-target-is-: + "any-target-is-spir", + "any-target-is-native_cpu", + "any-target-is-nvidia", + "any-target-is-amd", + "any-target-is-native_cpu", + # sg-sizes (should we allow any sg-X?): + "sg-8", + "sg-16", + "sg-32", + # e2e-modes: + "run-mode", + "build-and-run-mode", + # miscellaneous: + "cl_options", + "opencl_icd", + "dump_ir", + "xptifw", + "has_ndebug", + "zstd", + "preview-breaking-changes-supported", + "vulkan", + "O0", + "ze_debug", + "igc-dev", + "enable-perf-tests", +} + + +# To parse .def files to get aspects and architectures +def parse_defines(path, macro, prefix): + features = set() + with open(path, "r") as file: + for line in file: + if line.startswith(macro): + feature = line.split("(")[1].split(",")[0].strip() + features.add(f"{prefix}-{feature}") + return features + + +def get_sycl_lit_allowed_features(): + current_dir = os.path.dirname(os.path.abspath(__file__)) + aspects = parse_defines( + current_dir + "/../include/sycl/info/aspects.def", "__SYCL_ASPECT", "aspect" + ) + aspects_deprecated = parse_defines( + current_dir + "/../include/sycl/info/aspects_deprecated.def", + "__SYCL_ASPECT", + "aspect", + ) + architectures = parse_defines( + current_dir + + "/../include/sycl/ext/oneapi/experimental/device_architecture.def", + "__SYCL_ARCHITECTURE", + "arch", + ) + + # Combine all sets + all_features = ( + partial_set_of_features | aspects | aspects_deprecated | architectures + ) + return all_features diff --git a/sycl/test/e2e_test_requirements/check-correctness-of-requirements.cpp b/sycl/test/e2e_test_requirements/check-correctness-of-requirements.cpp new file mode 100644 index 0000000000000..f69cf7a588125 --- /dev/null +++ b/sycl/test/e2e_test_requirements/check-correctness-of-requirements.cpp @@ -0,0 +1,16 @@ +// This test checks that all "REQUIRES", "XFAIL" and "UNSUPPORTED" strings +// contain the right feature names. +// +// If this test fails: +// 1. there is some typo/deleted feature requested in the new/modified test. +// 2. ...or, there is some new feature. In this case please update the set of +// features in sycl/test-e2e/sycl_lit_allowed_features.py. +// +// Get a set of all features passed to "REQUIRES", "XFAIL" and "UNSUPPORTED": +// RUN: llvm-lit --show-used-features %S/../../test-e2e > %t +// +// Process this set using a python script: +// RUN: python3 %S/check-correctness-of-requirements.py %t +// +// Small negative test +// RUN: echo non-existing-feature-request > %t | not python3 %S/check-correctness-of-requirements.py %t diff --git a/sycl/test/e2e_test_requirements/check-correctness-of-requirements.py b/sycl/test/e2e_test_requirements/check-correctness-of-requirements.py new file mode 100644 index 0000000000000..295d8ab77c282 --- /dev/null +++ b/sycl/test/e2e_test_requirements/check-correctness-of-requirements.py @@ -0,0 +1,23 @@ +# See check-correctness-of-requirements.cpp + +import os +import sys + +input_data = sys.argv[1] +current_dir = os.path.dirname(os.path.abspath(__file__)) + +# See sycl/test-e2e/sycl_lit_allowed_features.py. +sys.path.append(current_dir + "/../../test-e2e") +from sycl_lit_allowed_features import get_sycl_lit_allowed_features + +allowed_features = get_sycl_lit_allowed_features() + +exit_code = 0 +with open(input_data, "r") as file: + requirements = set(file.read().split()) + for requirement in requirements: + if not requirement in allowed_features: + exit_code = 1 + print("Unsupported requirement: " + requirement) + +sys.exit(exit_code)