From c2a8f0074abe45ae60667aa854d5c334a819d4df Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 13 Jun 2025 11:56:43 +0000 Subject: [PATCH 1/2] mem props API --- .github/workflows/.spellcheck-conf.toml | 2 +- .github/workflows/reusable_basic.yml | 2 + benchmark/ubench.c | 235 +++++++++++++++++++ docs/config/api.rst | 6 + docs/config/spelling_exceptions.txt | 4 +- include/umf/base.h | 27 +++ include/umf/memory_props.h | 44 ++++ include/umf/memory_provider.h | 18 ++ include/umf/memory_provider_ops.h | 18 ++ scripts/qemu/run-tests.sh | 2 + src/CMakeLists.txt | 1 + src/ipc.c | 30 ++- src/libumf.def | 3 + src/libumf.map | 3 + src/memory_pool.c | 12 +- src/memory_props.c | 151 +++++++++++++ src/memory_props_internal.h | 38 ++++ src/memory_provider.c | 12 + src/pool/pool_disjoint.c | 29 ++- src/pool/pool_proxy.c | 19 +- src/provider/provider_cuda.c | 44 ++++ src/provider/provider_devdax_memory.c | 5 +- src/provider/provider_file_memory.c | 5 +- src/provider/provider_fixed_memory.c | 4 +- src/provider/provider_level_zero.c | 46 ++++ src/provider/provider_level_zero_internal.h | 5 + src/provider/provider_os_memory.c | 4 +- src/provider/provider_tracking.c | 140 +++++++----- src/provider/provider_tracking.h | 20 +- src/utils/utils_common.h | 8 +- src/utils/utils_level_zero.cpp | 12 + src/utils/utils_level_zero.h | 4 + test/CMakeLists.txt | 17 ++ test/common/pool_null.c | 3 +- test/common/pool_trace.c | 3 +- test/common/provider_null.c | 17 +- test/common/test_helpers.h | 4 +- test/memoryProviderAPI.cpp | 15 ++ test/props/provider_props.cpp | 74 ++++++ test/props/provider_props.hpp | 238 ++++++++++++++++++++ test/props/provider_props_cuda.cpp | 64 ++++++ test/props/provider_props_level_zero.cpp | 66 ++++++ test/providers/provider_cuda.cpp | 84 +++++++ test/providers/provider_level_zero.cpp | 86 +++++++ 44 files changed, 1526 insertions(+), 98 deletions(-) create mode 100644 include/umf/memory_props.h create mode 100644 src/memory_props.c create mode 100644 src/memory_props_internal.h create mode 100644 test/props/provider_props.cpp create mode 100644 test/props/provider_props.hpp create mode 100644 test/props/provider_props_cuda.cpp create mode 100644 test/props/provider_props_level_zero.cpp diff --git a/.github/workflows/.spellcheck-conf.toml b/.github/workflows/.spellcheck-conf.toml index 288af6a19..bb0f480d6 100644 --- a/.github/workflows/.spellcheck-conf.toml +++ b/.github/workflows/.spellcheck-conf.toml @@ -1,6 +1,6 @@ [default] # Don't correct the following words: -extend-ignore-words-re = ["ASSER", "Tne", "ba", "BA", "PN"] +extend-ignore-words-re = ["ASSER", "Tne", "ba", "BA", "PN", "usm"] [files] # completely exclude those files from consideration: diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 81ca53f10..123813500 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -195,6 +195,8 @@ jobs: ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }} cmake --build ${{env.BUILD_DIR}} -j $(nproc) + # UMF_LOG="level:debug;flush:debug;output:stderr;pid:no" + # -R "test_provider_os_memory" - name: Run tests working-directory: ${{env.BUILD_DIR}} run: | diff --git a/benchmark/ubench.c b/benchmark/ubench.c index 5beaa62be..8c0cd5f4b 100644 --- a/benchmark/ubench.c +++ b/benchmark/ubench.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -438,6 +439,48 @@ static void do_ipc_get_put_benchmark(alloc_t *allocs, size_t num_allocs, } } +static void do_umf_mem_props_benchmark(ze_context_handle_t context, + bool use_umf, alloc_t *allocs, + size_t num_allocs, size_t repeats) { + assert(context != NULL); + + for (size_t r = 0; r < repeats * 10; ++r) { + for (size_t i = 0; i < num_allocs; ++i) { + if (use_umf) { + umf_memory_properties_handle_t props_handle = NULL; + umf_result_t res = + umfGetMemoryPropertiesHandle(allocs[i].ptr, &props_handle); + (void)res; + assert(res == UMF_RESULT_SUCCESS); + + umf_usm_memory_type_t type = UMF_MEMORY_TYPE_UNKNOWN; + res = umfGetMemoryProperty(props_handle, + UMF_MEMORY_PROPERTY_POINTER_TYPE, + sizeof(type), &type); + assert(res == UMF_RESULT_SUCCESS); + if (type != UMF_MEMORY_TYPE_DEVICE) { + fprintf(stderr, + "error: unexpected alloc_props.type value: %d\n", + type); + exit(-1); + } + } else { + ze_memory_allocation_properties_t alloc_props = {0}; + ze_device_handle_t device = 0; + // calls zeMemGetAllocProperties() + utils_ze_get_mem_props(context, allocs[i].ptr, &alloc_props, + &device); + if (alloc_props.type != ZE_MEMORY_TYPE_DEVICE) { + fprintf(stderr, + "error: unexpected alloc_props.type value: %d\n", + alloc_props.type); + exit(-1); + } + } + } + } +} + static int create_level_zero_params(ze_context_handle_t *context, ze_device_handle_t *device) { uint32_t driver_idx = 0; @@ -623,6 +666,198 @@ UBENCH_EX(ipc, disjoint_pool_with_level_zero_provider) { err_destroy_context: utils_ze_destroy_context(context); } + +UBENCH_EX(mem_props, level_zero) { + const size_t BUFFER_SIZE = 100; + const size_t N_BUFFERS = 1000; + + alloc_t *allocs = alloc_array(N_BUFFERS); + if (allocs == NULL) { + fprintf(stderr, "error: alloc_array() failed\n"); + } + + ze_context_handle_t context = NULL; + ze_device_handle_t device = NULL; + int ret = create_level_zero_params(&context, &device); + if (ret != 0) { + fprintf(stderr, "error: create_level_zero_params() failed\n"); + exit(-1); + } + + ze_device_mem_alloc_desc_t dev_desc = { + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + .pNext = NULL, + .flags = 0, + .ordinal = 0}; + + for (size_t i = 0; i < N_BUFFERS; ++i) { + ze_result_t ze_result = zeMemAllocDevice( + context, &dev_desc, BUFFER_SIZE, 0, device, &allocs[i].ptr); + if (ze_result != ZE_RESULT_SUCCESS) { + fprintf(stderr, "error: zeMemAllocDevice() failed\n"); + } + allocs[i].size = BUFFER_SIZE; + } + + do_umf_mem_props_benchmark(context, false, allocs, N_BUFFERS, + 1); // WARMUP + UBENCH_DO_BENCHMARK() { + do_umf_mem_props_benchmark(context, false, allocs, N_BUFFERS, + N_ITERATIONS); + } + + for (size_t i = 0; i < N_BUFFERS; ++i) { + zeMemFree(context, allocs[i].ptr); + } + + free(allocs); + utils_ze_destroy_context(context); +} + +UBENCH_EX(mem_props, disjoint_pool_with_level_zero_provider_use_umf) { + const size_t BUFFER_SIZE = 4 * 1024; + const size_t N_BUFFERS = 1000; + umf_result_t umf_result; + ze_context_handle_t context = NULL; + ze_device_handle_t device = NULL; + umf_level_zero_memory_provider_params_handle_t level_zero_params = NULL; + + int ret = create_level_zero_params(&context, &device); + if (ret != 0) { + fprintf(stderr, "error: create_level_zero_params() failed\n"); + exit(-1); + } + + umf_result = umfLevelZeroMemoryProviderParamsCreate(&level_zero_params); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "error: umfLevelZeroMemoryProviderParamsCreate() failed\n"); + goto err_destroy_context; + } + + umf_result = + umfLevelZeroMemoryProviderParamsSetContext(level_zero_params, context); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "error: umfLevelZeroMemoryProviderParamsSetContext() failed\n"); + goto err_destroy_params; + } + + umf_result = + umfLevelZeroMemoryProviderParamsSetDevice(level_zero_params, device); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "error: umfLevelZeroMemoryProviderParamsSetDevice() failed\n"); + goto err_destroy_params; + } + + umf_result = umfLevelZeroMemoryProviderParamsSetMemoryType( + level_zero_params, UMF_MEMORY_TYPE_DEVICE); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf( + stderr, + "error: umfLevelZeroMemoryProviderParamsSetMemoryType() failed\n"); + goto err_destroy_params; + } + + alloc_t *allocs = alloc_array(N_BUFFERS); + if (allocs == NULL) { + fprintf(stderr, "error: alloc_array() failed\n"); + goto err_destroy_context; + } + + umf_memory_provider_handle_t provider = NULL; + umf_result = umfMemoryProviderCreate(umfLevelZeroMemoryProviderOps(), + level_zero_params, &provider); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "error: umfMemoryProviderCreate() failed\n"); + goto err_free_allocs; + } + + umf_disjoint_pool_params_handle_t disjoint_params = NULL; + umf_result = umfDisjointPoolParamsCreate(&disjoint_params); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "ERROR: umfDisjointPoolParamsCreate failed\n"); + goto err_provider_destroy; + } + + umf_result = + umfDisjointPoolParamsSetSlabMinSize(disjoint_params, BUFFER_SIZE * 10); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "error: umfDisjointPoolParamsSetSlabMinSize() failed\n"); + goto err_params_destroy; + } + + umf_result = umfDisjointPoolParamsSetMaxPoolableSize( + disjoint_params, 4ull * 1024ull * 1024ull); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "error: umfDisjointPoolParamsSetMaxPoolableSize() failed\n"); + goto err_params_destroy; + } + + umf_result = + umfDisjointPoolParamsSetCapacity(disjoint_params, 64ull * 1024ull); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "error: umfDisjointPoolParamsSetCapacity() failed\n"); + goto err_params_destroy; + } + + umf_result = umfDisjointPoolParamsSetMinBucketSize(disjoint_params, 64); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "error: umfDisjointPoolParamsSetMinBucketSize() failed\n"); + goto err_params_destroy; + } + + umf_pool_create_flags_t flags = UMF_POOL_CREATE_FLAG_NONE; + umf_memory_pool_handle_t pool; + umf_result = umfPoolCreate(umfDisjointPoolOps(), provider, disjoint_params, + flags, &pool); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "error: umfPoolCreate() failed\n"); + goto err_params_destroy; + } + + for (size_t i = 0; i < N_BUFFERS; ++i) { + allocs[i].ptr = umfPoolMalloc(pool, BUFFER_SIZE); + if (allocs[i].ptr == NULL) { + goto err_buffer_destroy; + } + allocs[i].size = BUFFER_SIZE; + } + + do_umf_mem_props_benchmark(context, true, allocs, N_BUFFERS, + 1); // WARMUP + UBENCH_DO_BENCHMARK() { + do_umf_mem_props_benchmark(context, true, allocs, N_BUFFERS, + N_ITERATIONS); + } + +err_buffer_destroy: + for (size_t i = 0; i < N_BUFFERS; ++i) { + umfPoolFree(pool, allocs[i].ptr); + } + + umfPoolDestroy(pool); + +err_params_destroy: + umfDisjointPoolParamsDestroy(disjoint_params); + +err_provider_destroy: + umfMemoryProviderDestroy(provider); + +err_free_allocs: + free(allocs); + +err_destroy_params: + umfLevelZeroMemoryProviderParamsDestroy(level_zero_params); + +err_destroy_context: + utils_ze_destroy_context(context); +} + #endif /* (defined UMF_BUILD_LEVEL_ZERO_PROVIDER && defined UMF_BUILD_GPU_TESTS) */ // TODO add IPC benchmark for CUDA diff --git a/docs/config/api.rst b/docs/config/api.rst index 879623428..37aa075d5 100644 --- a/docs/config/api.rst +++ b/docs/config/api.rst @@ -170,6 +170,12 @@ Memtarget .. doxygenfile:: experimental/memtarget.h :sections: define enum typedef func +Memory Properties +========================================== + +.. doxygenfile:: memory_props.h + :sections: define enum typedef func var + Inter-Process Communication ========================================== diff --git a/docs/config/spelling_exceptions.txt b/docs/config/spelling_exceptions.txt index d4e40a3ec..c49ab6db0 100644 --- a/docs/config/spelling_exceptions.txt +++ b/docs/config/spelling_exceptions.txt @@ -47,6 +47,7 @@ partList pid poolable preallocated +propertyId providerIpcData providential ptr @@ -71,4 +72,5 @@ umfPoolMallocUsableSize umfPoolRealloc umfMemspaceUserFilter umfMemspaceMemtargetAdd -unfreed \ No newline at end of file +unfreed +usm diff --git a/include/umf/base.h b/include/umf/base.h index 0bbafc309..9aaf595b0 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -51,6 +51,33 @@ typedef enum umf_result_t { UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown error } umf_result_t; +/// @brief Handle to the memory properties structure +typedef struct umf_memory_properties_t *umf_memory_properties_handle_t; + +/// @brief ID of the memory property +typedef enum umf_memory_property_id_t { + UMF_MEMORY_PROPERTY_INVALID = -1, ///< Invalid property + + // UMF specific + UMF_MEMORY_PROPERTY_PROVIDER_HANDLE = 0, ///< Handle to the memory provider + UMF_MEMORY_PROPERTY_POOL_HANDLE = 1, ///< Handle to the memory pool + + // generic pointer properties + UMF_MEMORY_PROPERTY_POINTER_TYPE = + 2, ///< Type of the pointer (umf_usm_memory_type_t) + UMF_MEMORY_PROPERTY_BASE_ADDRESS = 3, ///< Base address of the allocation + UMF_MEMORY_PROPERTY_BASE_SIZE = 4, ///< Base size of the allocation + UMF_MEMORY_PROPERTY_BUFFER_ID = 5, ///< Unique identifier for the buffer + + // GPU specific + UMF_MEMORY_PROPERTY_CONTEXT = 6, ///< GPU context of the allocation + UMF_MEMORY_PROPERTY_DEVICE = 7, ///< GPU device where the allocation resides + + /// @cond + UMF_MEMORY_PROPERTY_MAX_RESERVED = 0x1000, ///< Maximum reserved value + /// @endcond +} umf_memory_property_id_t; + /// @brief Type of the CTL query typedef enum umf_ctl_query_type { CTL_QUERY_READ, diff --git a/include/umf/memory_props.h b/include/umf/memory_props.h new file mode 100644 index 000000000..56068bfe2 --- /dev/null +++ b/include/umf/memory_props.h @@ -0,0 +1,44 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_MEMORY_PROPS_H +#define UMF_MEMORY_PROPS_H 1 + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// @brief Get the memory properties handle for a given pointer +/// @param ptr pointer to the allocated memory +/// @param props_handle [out] pointer to the memory properties handle +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure +umf_result_t +umfGetMemoryPropertiesHandle(const void *ptr, + umf_memory_properties_handle_t *props_handle); + +/// @brief Get a specific memory property from the properties handle +/// @param props_handle handle to the memory properties +/// @param memory_property_id ID of the memory property to get +/// @param max_property_size size of the property value buffer +/// @param property_value [out] pointer to the value of the memory property +/// which will be filled +// TODO check return type +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure +umf_result_t umfGetMemoryProperty(umf_memory_properties_handle_t props_handle, + umf_memory_property_id_t memory_property_id, + size_t max_property_size, + void *property_value); + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_MEMORY_PROPS_H */ diff --git a/include/umf/memory_provider.h b/include/umf/memory_provider.h index b98b89346..c68bee32c 100644 --- a/include/umf/memory_provider.h +++ b/include/umf/memory_provider.h @@ -265,6 +265,24 @@ umf_result_t umfMemoryProviderAllocationMerge(umf_memory_provider_handle_t hProvider, void *lowPtr, void *highPtr, size_t totalSize); +/// +/// @brief Retrieve properties of the memory allocation. +/// @param hProvider pointer to the memory provider +/// @param ptr pointer to the allocated memory +/// @param propertyId identifier of the memory property to retrieve +/// @param max_property_size size of the property value buffer +/// @param property_value [out] pointer to the value of the memory property +/// which will be filled +// TODO check +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure, +/// UMF_RESULT_ERROR_INVALID_ARGUMENT if propertyId is invalid or value is NULL, +/// UMF_RESULT_ERROR_NOT_SUPPORTED if the property is not supported by this provider. +/// +umf_result_t umfMemoryProviderGetAllocationProperties( + umf_memory_provider_handle_t hProvider, const void *ptr, + umf_memory_property_id_t propertyId, size_t max_property_size, + void *property_value); + #ifdef __cplusplus } #endif diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index adb468160..2d83c9928 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -275,6 +275,24 @@ typedef struct umf_memory_provider_ops_t { void *arg, size_t size, umf_ctl_query_type_t queryType); + /// + /// @brief Retrieve properties of the memory allocation. + /// @param provider pointer to the memory provider + /// @param ptr pointer to the allocated memory + /// @param memory_property_id identifier of the memory property to retrieve + /// @param max_property_size size of the property value buffer + /// @param property_value [out] pointer to the value of the memory property + /// which will be filled + // TODO check + /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure, + /// UMF_RESULT_ERROR_INVALID_ARGUMENT if memory_property_id is invalid or value is NULL, + /// UMF_RESULT_ERROR_NOT_SUPPORTED if the property is not supported by this provider. + /// + umf_result_t (*ext_get_allocation_properties)( + void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, size_t max_property_size, + void *value); + } umf_memory_provider_ops_t; #ifdef __cplusplus diff --git a/scripts/qemu/run-tests.sh b/scripts/qemu/run-tests.sh index 341e2f9ab..35caf05dc 100755 --- a/scripts/qemu/run-tests.sh +++ b/scripts/qemu/run-tests.sh @@ -28,7 +28,9 @@ numactl -H cd build echo "## Running all tests ..." +#UMF_LOG="level:debug;flush:debug;output:stderr;pid:no" ctest --verbose +# --output-on-failure -R "memoryPool" echo "## Running tests bound to a numa node 0 and node 1 ..." numactl -N 0 ctest --output-on-failure diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d11e04c4f..99bb7e2cf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -44,6 +44,7 @@ set(UMF_SOURCES ipc.c ipc_cache.c memory_pool.c + memory_props.c memory_provider.c memory_provider_get_last_failed.c memtarget.c diff --git a/src/ipc.c b/src/ipc.c index d4e5cc806..aa7388af6 100644 --- a/src/ipc.c +++ b/src/ipc.c @@ -58,14 +58,19 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, } size_t ipcHandleSize = 0; - umf_alloc_info_t allocInfo; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); + umf_memory_properties_handle_t props = NULL; + umf_result_t ret = umfGetMemoryPropertiesHandle(ptr, &props); if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("cannot get alloc info for ptr = %p.", ptr); + LOG_ERR("cannot get alloc props for ptr = %p.", ptr); return ret; } - ret = umfPoolGetIPCHandleSize(allocInfo.pool, &ipcHandleSize); + if (props == NULL || props->pool == NULL) { + LOG_ERR("cannot get pool from alloc info for ptr = %p.", ptr); + return UMF_RESULT_ERROR_UNKNOWN; + } + + ret = umfPoolGetIPCHandleSize(props->pool, &ipcHandleSize); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("cannot get IPC handle size."); return ret; @@ -79,11 +84,14 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, // We cannot use umfPoolGetMemoryProvider function because it returns // upstream provider but we need tracking one - umf_memory_provider_handle_t provider = allocInfo.pool->provider; - assert(provider); + if (props == NULL || props->pool == NULL || props->pool->provider == NULL) { + LOG_ERR("cannot get memory provider from pool"); + umf_ba_global_free(ipcData); + return UMF_RESULT_ERROR_UNKNOWN; + } + umf_memory_provider_handle_t provider = props->pool->provider; - ret = umfMemoryProviderGetIPCHandle(provider, allocInfo.base, - allocInfo.baseSize, + ret = umfMemoryProviderGetIPCHandle(provider, props->base, props->base_size, (void *)ipcData->providerIpcData); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to get IPC handle."); @@ -92,10 +100,10 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, } // ipcData->handle_id is filled by tracking provider - ipcData->base = allocInfo.base; + ipcData->base = props->base; ipcData->pid = utils_getpid(); - ipcData->baseSize = allocInfo.baseSize; - ipcData->offset = (uintptr_t)ptr - (uintptr_t)allocInfo.base; + ipcData->baseSize = props->base_size; + ipcData->offset = (uintptr_t)ptr - (uintptr_t)props->base; *umfIPCHandle = ipcData; *size = ipcHandleSize; diff --git a/src/libumf.def b/src/libumf.def index 10b0326b8..31217c3d4 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -140,7 +140,10 @@ EXPORTS umfCtlExec umfCtlGet umfCtlSet + umfGetMemoryPropertiesHandle + umfGetMemoryProperty umfJemallocPoolParamsCreate umfJemallocPoolParamsDestroy umfJemallocPoolParamsSetNumArenas + umfMemoryProviderGetAllocationProperties umfPoolGetName diff --git a/src/libumf.map b/src/libumf.map index e283c3853..8550ddf13 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -140,8 +140,11 @@ UMF_0.12 { umfCtlExec; umfCtlGet; umfCtlSet; + umfGetMemoryPropertiesHandle; + umfGetMemoryProperty; umfJemallocPoolParamsCreate; umfJemallocPoolParamsDestroy; umfJemallocPoolParamsSetNumArenas; + umfMemoryProviderGetAllocationProperties; umfPoolGetName; } UMF_0.11; diff --git a/src/memory_pool.c b/src/memory_pool.c index 0aa0ef5ab..df568da82 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -286,8 +286,16 @@ umf_result_t umfFree(void *ptr) { umf_result_t umfPoolByPtr(const void *ptr, umf_memory_pool_handle_t *pool) { UMF_CHECK((pool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); - *pool = umfMemoryTrackerGetPool(ptr); - return *pool ? UMF_RESULT_SUCCESS : UMF_RESULT_ERROR_INVALID_ARGUMENT; + + umf_memory_properties_handle_t props = NULL; + umf_result_t ret = umfGetMemoryPropertiesHandle(ptr, &props); + if (ret != UMF_RESULT_SUCCESS || props == NULL || props->pool == NULL) { + *pool = NULL; + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + *pool = props->pool; + return UMF_RESULT_SUCCESS; } umf_result_t umfPoolGetMemoryProvider(umf_memory_pool_handle_t hPool, diff --git a/src/memory_props.c b/src/memory_props.c new file mode 100644 index 000000000..31e79a9bf --- /dev/null +++ b/src/memory_props.c @@ -0,0 +1,151 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include + +#include +#include +#include + +#include "memory_props_internal.h" +#include "memory_provider_internal.h" +#include "provider/provider_tracking.h" + +// TODO flag - IPC? +umf_result_t +umfGetMemoryPropertiesHandle(const void *ptr, + umf_memory_properties_handle_t *props_handle) { + + //LOG_DEBUG("umfGetMemoryPropertiesHandle: ptr=%p, props_handle=%p", ptr, + // props_handle); + + if (UNLIKELY(props_handle == NULL)) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + tracker_alloc_info_t *info = NULL; + umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &info); + if (ret == UMF_RESULT_SUCCESS) { + *props_handle = &info->props; + //LOG_DEBUG("umfGetMemoryPropertiesHandle: props_handle=%p, id=%" PRIu64, + // *props_handle, (*props_handle)->id); + return UMF_RESULT_SUCCESS; + } + + // try to get IPC info + umf_ipc_info_t ipc_info; + ret = umfMemoryTrackerGetIpcInfo(ptr, &ipc_info); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("Failed to get memory properties handle for ptr=%p", ptr); + return ret; + } + + *props_handle = ipc_info.props; + //LOG_DEBUG( + // "umfGetMemoryPropertiesHandle (IPC info): props_handle=%p, id=%" PRIu64, + // *props_handle, (*props_handle)->id); + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfGetMemoryProperty(umf_memory_properties_handle_t props_handle, + umf_memory_property_id_t memory_property_id, + size_t max_property_size, void *value) { + + /* + LOG_DEBUG("umfGetMemoryProperty: props_handle=%p, memory_property_id=%d, " + "max_property_size=%zu, value=%p", + props_handle, memory_property_id, max_property_size, value); + */ + + if (UNLIKELY((value == NULL) || (props_handle == NULL) || + (max_property_size == 0))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_memory_provider_t *provider = props_handle->provider; + + /* + LOG_DEBUG("umfGetMemoryProperty: provider=%p", provider); + LOG_DEBUG("dereferencing value..."); + + LOG_DEBUG("value: %zu", *(size_t *)value); + */ + + /* + int x = (int)memory_property_id; + for (int i = 0; i < 1000; i++) { + x += (double)i / 10000000.0; + memory_property_id = x; + } + */ + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_INVALID: + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + + case UMF_MEMORY_PROPERTY_POOL_HANDLE: + if (UNLIKELY(max_property_size < sizeof(umf_memory_pool_handle_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_memory_pool_handle_t *)value = props_handle->pool; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_PROVIDER_HANDLE: + if (UNLIKELY(max_property_size < + sizeof(umf_memory_provider_handle_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_memory_provider_handle_t *)value = provider; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_BUFFER_ID: + if (UNLIKELY(max_property_size < sizeof(uint64_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(uint64_t *)value = props_handle->id; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_BASE_ADDRESS: + if (UNLIKELY(max_property_size < sizeof(uintptr_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(uintptr_t *)value = (uintptr_t)props_handle->base; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_BASE_SIZE: + if (UNLIKELY(max_property_size < sizeof(size_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(size_t *)value = props_handle->base_size; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + // NOTE: this property is "cached" in the props_handle but the value is + // determined by the memory provider and set during addition to the + // tracker. + if (UNLIKELY(max_property_size < sizeof(umf_usm_memory_type_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_usm_memory_type_t *)value = props_handle->memory_type; + return UMF_RESULT_SUCCESS; + + // GPU Memory Provider specific properties + case UMF_MEMORY_PROPERTY_CONTEXT: + case UMF_MEMORY_PROPERTY_DEVICE: + return provider->ops.ext_get_allocation_properties( + provider->provider_priv, props_handle->ptr, memory_property_id, + max_property_size, value); + + default: + break; + }; + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} diff --git a/src/memory_props_internal.h b/src/memory_props_internal.h new file mode 100644 index 000000000..9bc8d0ea1 --- /dev/null +++ b/src/memory_props_internal.h @@ -0,0 +1,38 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_MEMORY_PROPS_INTERNAL_H +#define UMF_MEMORY_PROPS_INTERNAL_H 1 + +#include + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct umf_memory_properties_t { + void *ptr; + umf_memory_pool_handle_t pool; + umf_memory_provider_handle_t provider; + uint64_t id; + void *base; + size_t base_size; + umf_usm_memory_type_t memory_type; +} umf_memory_properties_t; + +#ifdef __cplusplus +} +#endif + +#endif // UMF_MEMORY_PROPS_INTERNAL_H diff --git a/src/memory_provider.c b/src/memory_provider.c index 27f33f86f..7d9d97ce7 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -18,6 +18,7 @@ #include "base_alloc.h" #include "base_alloc_global.h" #include "libumf.h" +#include "memory_props_internal.h" #include "memory_provider_internal.h" #include "utils_assert.h" @@ -427,3 +428,14 @@ umfMemoryProviderCloseIPCHandle(umf_memory_provider_handle_t hProvider, return hProvider->ops.ext_close_ipc_handle(hProvider->provider_priv, ptr, size); } + +umf_result_t umfMemoryProviderGetAllocationProperties( + umf_memory_provider_handle_t hProvider, const void *ptr, + umf_memory_property_id_t propertyId, size_t max_property_size, + void *property_value) { + UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((property_value != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + return hProvider->ops.ext_get_allocation_properties( + hProvider->provider_priv, ptr, propertyId, max_property_size, + property_value); +} diff --git a/src/pool/pool_disjoint.c b/src/pool/pool_disjoint.c index 3689330d5..0beb85200 100644 --- a/src/pool/pool_disjoint.c +++ b/src/pool/pool_disjoint.c @@ -895,14 +895,19 @@ umf_result_t disjoint_pool_malloc_usable_size(void *pool, const void *ptr, critnib_release(disjoint_pool->known_slabs, ref_slab); } - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); - if (ret != UMF_RESULT_SUCCESS) { - *size = 0; - return ret; + umf_memory_properties_handle_t props = NULL; + umf_result_t umf_result = umfGetMemoryPropertiesHandle(ptr, &props); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + if (props == NULL) { + TLS_last_allocation_error = UMF_RESULT_ERROR_UNKNOWN; + LOG_ERR("failed to get allocation info from the memory tracker"); + return UMF_RESULT_ERROR_UNKNOWN; } - *size = allocInfo.baseSize; + *size = props->base_size; return UMF_RESULT_SUCCESS; } @@ -938,15 +943,21 @@ umf_result_t disjoint_pool_free(void *pool, void *ptr) { critnib_release(disjoint_pool->known_slabs, ref_slab); } - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); + umf_memory_properties_handle_t props = NULL; + umf_result_t ret = umfGetMemoryPropertiesHandle(ptr, &props); if (ret != UMF_RESULT_SUCCESS) { TLS_last_allocation_error = ret; LOG_ERR("failed to get allocation info from the memory tracker"); return ret; } - size_t size = allocInfo.baseSize; + if (props == NULL) { + TLS_last_allocation_error = UMF_RESULT_ERROR_UNKNOWN; + LOG_ERR("failed to get allocation info from the memory tracker"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + size_t size = props->base_size; umf_memory_provider_handle_t provider = disjoint_pool->provider; ret = umfMemoryProviderFree(provider, ptr, size); if (ret != UMF_RESULT_SUCCESS) { diff --git a/src/pool/pool_proxy.c b/src/pool/pool_proxy.c index c6bf74124..3da9a4ee6 100644 --- a/src/pool/pool_proxy.c +++ b/src/pool/pool_proxy.c @@ -100,11 +100,22 @@ static umf_result_t proxy_free(void *pool, void *ptr) { struct proxy_memory_pool *hPool = (struct proxy_memory_pool *)pool; if (ptr) { - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t umf_result = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); - if (umf_result == UMF_RESULT_SUCCESS) { - size = allocInfo.baseSize; + umf_memory_properties_handle_t props = NULL; + umf_result_t umf_result = umfGetMemoryPropertiesHandle(ptr, &props); + + if (umf_result != UMF_RESULT_SUCCESS) { + TLS_last_allocation_error = umf_result; + LOG_ERR("failed to get allocation info from the memory tracker"); + return umf_result; + } + + if (props == NULL) { + TLS_last_allocation_error = UMF_RESULT_ERROR_UNKNOWN; + LOG_ERR("failed to get allocation info from the memory tracker"); + return UMF_RESULT_ERROR_UNKNOWN; } + + size = props->base_size; } return umfMemoryProviderFree(hPool->hProvider, ptr, size); diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index d024ae0ea..420c64e17 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -12,6 +12,8 @@ #include #include +#include "memory_props_internal.h" +#include "memory_provider_internal.h" #include "provider_cuda_internal.h" #include "utils_load_library.h" #include "utils_log.h" @@ -685,6 +687,45 @@ cu_memory_provider_close_ipc_handle(void *provider, void *ptr, size_t size) { return UMF_RESULT_SUCCESS; } +static umf_result_t cu_memory_provider_get_allocation_properties( + void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, size_t max_property_size, + void *value) { + + // unused + (void)ptr; + + cu_memory_provider_t *cuda_provider = (cu_memory_provider_t *)provider; + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + if (UNLIKELY(max_property_size < sizeof(umf_usm_memory_type_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_usm_memory_type_t *)value = cuda_provider->memory_type; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_CONTEXT: + if (UNLIKELY(max_property_size < sizeof(CUcontext))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(CUcontext *)value = cuda_provider->context; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_DEVICE: + if (UNLIKELY(max_property_size < sizeof(CUdevice))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(CUdevice *)value = cuda_provider->device; + return UMF_RESULT_SUCCESS; + + default: + break; + }; + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} + static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = cu_memory_provider_initialize, @@ -707,6 +748,9 @@ static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .ext_put_ipc_handle = cu_memory_provider_put_ipc_handle, .ext_open_ipc_handle = cu_memory_provider_open_ipc_handle, .ext_close_ipc_handle = cu_memory_provider_close_ipc_handle, + .ext_ctl = NULL, + .ext_get_allocation_properties = + cu_memory_provider_get_allocation_properties, }; const umf_memory_provider_ops_t *umfCUDAMemoryProviderOps(void) { diff --git a/src/provider/provider_devdax_memory.c b/src/provider/provider_devdax_memory.c index 995f50cf5..18dbd2d50 100644 --- a/src/provider/provider_devdax_memory.c +++ b/src/provider/provider_devdax_memory.c @@ -555,7 +555,10 @@ static umf_memory_provider_ops_t UMF_DEVDAX_MEMORY_PROVIDER_OPS = { .ext_get_ipc_handle = devdax_get_ipc_handle, .ext_put_ipc_handle = devdax_put_ipc_handle, .ext_open_ipc_handle = devdax_open_ipc_handle, - .ext_close_ipc_handle = devdax_close_ipc_handle}; + .ext_close_ipc_handle = devdax_close_ipc_handle, + .ext_ctl = NULL, + .ext_get_allocation_properties = NULL, +}; const umf_memory_provider_ops_t *umfDevDaxMemoryProviderOps(void) { return &UMF_DEVDAX_MEMORY_PROVIDER_OPS; diff --git a/src/provider/provider_file_memory.c b/src/provider/provider_file_memory.c index 78cfb2cc2..7956b3c79 100644 --- a/src/provider/provider_file_memory.c +++ b/src/provider/provider_file_memory.c @@ -882,7 +882,10 @@ static umf_memory_provider_ops_t UMF_FILE_MEMORY_PROVIDER_OPS = { .ext_get_ipc_handle = file_get_ipc_handle, .ext_put_ipc_handle = file_put_ipc_handle, .ext_open_ipc_handle = file_open_ipc_handle, - .ext_close_ipc_handle = file_close_ipc_handle}; + .ext_close_ipc_handle = file_close_ipc_handle, + .ext_ctl = NULL, + .ext_get_allocation_properties = NULL, +}; const umf_memory_provider_ops_t *umfFileMemoryProviderOps(void) { return &UMF_FILE_MEMORY_PROVIDER_OPS; diff --git a/src/provider/provider_fixed_memory.c b/src/provider/provider_fixed_memory.c index 5f047442a..b25a165cb 100644 --- a/src/provider/provider_fixed_memory.c +++ b/src/provider/provider_fixed_memory.c @@ -309,7 +309,9 @@ static umf_memory_provider_ops_t UMF_FIXED_MEMORY_PROVIDER_OPS = { .ext_put_ipc_handle = NULL, .ext_open_ipc_handle = NULL, .ext_close_ipc_handle = NULL, - .ext_ctl = fixed_ctl}; + .ext_ctl = fixed_ctl, + .ext_get_allocation_properties = NULL, +}; const umf_memory_provider_ops_t *umfFixedMemoryProviderOps(void) { return &UMF_FIXED_MEMORY_PROVIDER_OPS; diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index ba071a45e..c0c50804d 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -11,9 +11,12 @@ #include #include +#include #include #include +#include "memory_props_internal.h" +#include "memory_provider_internal.h" #include "provider_level_zero_internal.h" #include "utils_load_library.h" #include "utils_log.h" @@ -31,6 +34,7 @@ void fini_ze_global_state(void) { #include "base_alloc_global.h" #include "libumf.h" +#include "provider_level_zero_internal.h" #include "utils_assert.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -790,6 +794,45 @@ ze_memory_provider_close_ipc_handle(void *provider, void *ptr, size_t size) { return UMF_RESULT_SUCCESS; } +static umf_result_t ze_memory_provider_get_allocation_properties( + void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, size_t max_property_size, + void *value) { + (void)ptr; + + struct ze_memory_provider_t *ze_provider = + (struct ze_memory_provider_t *)provider; + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + if (UNLIKELY(max_property_size < sizeof(umf_usm_memory_type_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(umf_usm_memory_type_t *)value = + ze2umf_memory_type(ze_provider->memory_type); + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_CONTEXT: + if (UNLIKELY(max_property_size < sizeof(ze_context_handle_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(ze_context_handle_t *)value = ze_provider->context; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_DEVICE: + if (UNLIKELY(max_property_size < sizeof(ze_device_handle_t))) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + *(ze_device_handle_t *)value = ze_provider->device; + return UMF_RESULT_SUCCESS; + + default: + break; + } + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} + static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = ze_memory_provider_initialize, @@ -809,6 +852,9 @@ static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .ext_put_ipc_handle = ze_memory_provider_put_ipc_handle, .ext_open_ipc_handle = ze_memory_provider_open_ipc_handle, .ext_close_ipc_handle = ze_memory_provider_close_ipc_handle, + .ext_ctl = NULL, + .ext_get_allocation_properties = + ze_memory_provider_get_allocation_properties, }; const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { diff --git a/src/provider/provider_level_zero_internal.h b/src/provider/provider_level_zero_internal.h index 7da299ffd..3bd76a0a9 100644 --- a/src/provider/provider_level_zero_internal.h +++ b/src/provider/provider_level_zero_internal.h @@ -7,4 +7,9 @@ * */ +#ifndef UMF_LEVEL_ZERO_PROVIDER_INTERNAL_H +#define UMF_LEVEL_ZERO_PROVIDER_INTERNAL_H + void fini_ze_global_state(void); + +#endif // UMF_LEVEL_ZERO_PROVIDER_INTERNAL_H diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 2b3ca2534..470f34bac 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -1413,7 +1413,8 @@ static umf_result_t os_open_ipc_handle(void *provider, void *providerIpcData, os_ipc_data->visibility, fd, os_ipc_data->fd_offset); if (*ptr == NULL) { os_store_last_native_error(UMF_OS_RESULT_ERROR_ALLOC_FAILED, errno); - LOG_PERR("memory mapping failed"); + LOG_PERR("memory mapping failed %zu bytes at fd=%d, offset=%zu", + os_ipc_data->size, fd, os_ipc_data->fd_offset); ret = UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; } @@ -1471,6 +1472,7 @@ static umf_memory_provider_ops_t UMF_OS_MEMORY_PROVIDER_OPS = { .ext_open_ipc_handle = os_open_ipc_handle, .ext_close_ipc_handle = os_close_ipc_handle, .ext_ctl = os_ctl, + .ext_get_allocation_properties = NULL, }; const umf_memory_provider_ops_t *umfOsMemoryProviderOps(void) { diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 386eef0ba..ff51ed4ba 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -15,6 +15,7 @@ #include #include +#include #include #include "base_alloc_global.h" @@ -22,6 +23,8 @@ #include "ipc_cache.h" #include "ipc_internal.h" #include "memory_pool_internal.h" +#include "memory_props_internal.h" +#include "memory_provider_internal.h" #include "provider_tracking.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -32,6 +35,8 @@ uint64_t IPC_HANDLE_ID = 0; +uint64_t unique_alloc_id = 0; // requires atomic access + struct umf_memory_tracker_t { umf_ba_pool_t *alloc_info_allocator; // Multilevel maps are needed to support the case @@ -43,19 +48,8 @@ struct umf_memory_tracker_t { critnib *ipc_segments_map; }; -typedef struct tracker_alloc_info_t { - umf_memory_pool_handle_t pool; - size_t size; - // number of overlapping memory regions - // in the next level of map - // falling within the current range - size_t n_children; -#if !defined(NDEBUG) && defined(UMF_DEVELOPER_MODE) - uint64_t is_freed; -#endif -} tracker_alloc_info_t; - typedef struct tracker_ipc_info_t { + umf_memory_properties_t props; size_t size; umf_memory_provider_handle_t provider; ipc_opened_cache_value_t *ipc_cache_value; @@ -113,7 +107,8 @@ static tracker_alloc_info_t *get_most_nested_alloc_segment( continue; } - utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, + &rsize); utils_atomic_load_acquire_size_t(&rvalue->n_children, &n_children); if (found && ((uintptr_t)ptr < rkey + rsize) && n_children) { if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { @@ -192,8 +187,27 @@ umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - value->pool = pool; - value->size = size; + // get provider + umf_memory_provider_handle_t provider = NULL; + umfPoolGetMemoryProvider(pool, &provider); + + // get memory type + umf_usm_memory_type_t memory_type = UMF_MEMORY_TYPE_UNKNOWN; + if (provider && provider->ops.ext_get_allocation_properties) { + provider->ops.ext_get_allocation_properties( + provider->provider_priv, ptr, UMF_MEMORY_PROPERTY_POINTER_TYPE, + sizeof(umf_usm_memory_type_t), &memory_type); + } + + memset(&value->props, 0, sizeof(umf_memory_properties_t)); + value->props.id = utils_atomic_increment_u64(&unique_alloc_id); + value->props.base = (void *)ptr; + value->props.base_size = size; + value->props.pool = pool; + value->props.provider = provider; + value->props.ptr = (void *)ptr; + value->props.memory_type = memory_type; + value->n_children = 0; #if !defined(NDEBUG) && defined(UMF_DEVELOPER_MODE) value->is_freed = 0; @@ -214,8 +228,8 @@ umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, "child #%zu added to memory region: tracker=%p, level=%i, " "pool=%p, ptr=%p, size=%zu", n_children, (void *)hTracker, level - 1, - (void *)parent_value->pool, (void *)parent_key, - parent_value->size); + (void *)parent_value->props.pool, (void *)parent_key, + parent_value->props.base_size); assert(ref_parent_value); critnib_release(hTracker->alloc_segments_map[level - 1], ref_parent_value); @@ -286,7 +300,8 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, assert(is_freed != 0xDEADBEEF); #endif - utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, + &rsize); if ((uintptr_t)ptr < rkey + rsize) { if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { @@ -300,8 +315,8 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, "cannot insert to the tracker value (pool=%p, ptr=%p, " "size=%zu) " "that exceeds the parent value (pool=%p, ptr=%p, size=%zu)", - (void *)pool, ptr, size, (void *)rvalue->pool, (void *)rkey, - (size_t)rsize); + (void *)pool, ptr, size, (void *)rvalue->props.pool, + (void *)rkey, (size_t)rsize); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } parent_key = rkey; @@ -363,7 +378,8 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, LOG_DEBUG("memory region removed: tracker=%p, level=%i, pool=%p, ptr=%p, " "size=%zu", - (void *)hTracker, level, (void *)value->pool, ptr, value->size); + (void *)hTracker, level, (void *)value->props.pool, ptr, + value->props.base_size); // release the reference to the value got from critnib_remove() assert(ref_value); @@ -375,8 +391,9 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, LOG_DEBUG( "child #%zu removed from memory region: tracker=%p, level=%i, " "pool=%p, ptr=%p, size=%zu", - n_children, (void *)hTracker, level - 1, (void *)parent_value->pool, - (void *)parent_key, parent_value->size); + n_children, (void *)hTracker, level - 1, + (void *)parent_value->props.pool, (void *)parent_key, + parent_value->props.base_size); assert(ref_parent_value); assert(level >= 1); @@ -409,6 +426,14 @@ umfMemoryTrackerAddIpcSegment(umf_memory_tracker_handle_t hTracker, value->provider = provider; value->ipc_cache_value = cache_entry; + memset(&value->props, 0, sizeof(umf_memory_properties_t)); + value->props.id = utils_atomic_increment_u64(&unique_alloc_id); + value->props.base = (void *)ptr; + value->props.base_size = size; + value->props.pool = NULL; // unknown + value->props.provider = provider; + value->props.ptr = (void *)ptr; + int ret = critnib_insert(hTracker->ipc_segments_map, (uintptr_t)ptr, value, 0); if (ret == 0) { @@ -458,37 +483,27 @@ umfMemoryTrackerRemoveIpcSegment(umf_memory_tracker_handle_t hTracker, return UMF_RESULT_SUCCESS; } -umf_memory_pool_handle_t umfMemoryTrackerGetPool(const void *ptr) { - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); - if (ret != UMF_RESULT_SUCCESS) { - return NULL; - } - - return allocInfo.pool; -} - umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, - umf_alloc_info_t *pAllocInfo) { - assert(pAllocInfo); + tracker_alloc_info_t **info) { + assert(info); - if (ptr == NULL) { + if (UNLIKELY(ptr == NULL)) { return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (TRACKER == NULL) { + if (UNLIKELY(TRACKER == NULL)) { LOG_ERR("tracker does not exist"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } - if (TRACKER->alloc_segments_map[0] == NULL) { + if (UNLIKELY(TRACKER->alloc_segments_map[0] == NULL)) { LOG_ERR("tracker's alloc_segments_map does not exist"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } tracker_alloc_info_t *top_most_value = NULL; tracker_alloc_info_t *rvalue = NULL; - uintptr_t top_most_key = 0; + //uintptr_t top_most_key = 0; uintptr_t rkey = 0; uint64_t rsize = 0; size_t n_children = 0; @@ -514,7 +529,7 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, critnib_release(TRACKER->alloc_segments_map[level], ref_value); } top_most_value = NULL; - top_most_key = 0; + //top_most_key = 0; rkey = 0; rsize = 0; level = 0; @@ -525,10 +540,11 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, continue; } - utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, + &rsize); utils_atomic_load_acquire_size_t(&rvalue->n_children, &n_children); if (found && (uintptr_t)ptr < rkey + rsize) { - top_most_key = rkey; + //top_most_key = rkey; top_most_value = rvalue; if (ref_top_most_value) { assert(level >= 1); @@ -555,9 +571,7 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - pAllocInfo->base = (void *)top_most_key; - pAllocInfo->baseSize = top_most_value->size; - pAllocInfo->pool = top_most_value->pool; + *info = top_most_value; assert(ref_top_most_value); critnib_release(TRACKER->alloc_segments_map[ref_level], ref_top_most_value); @@ -603,6 +617,8 @@ umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, pIpcInfo->baseSize = rvalue->size; pIpcInfo->provider = rvalue->provider; + pIpcInfo->props = &rvalue->props; + if (ref_value) { critnib_release(TRACKER->ipc_segments_map, ref_value); } @@ -692,9 +708,9 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; } - if (value->size != totalSize) { + if (value->props.base_size != totalSize) { LOG_ERR("tracked size=%zu does not match requested size to split: %zu", - value->size, totalSize); + value->props.base_size, totalSize); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; } @@ -732,7 +748,8 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, } // update the size of the first part - utils_atomic_store_release_u64((uint64_t *)&value->size, firstSize); + utils_atomic_store_release_u64((uint64_t *)&value->props.base_size, + firstSize); critnib_release(provider->hTracker->alloc_segments_map[level], ref_value); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); @@ -805,12 +822,12 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_fatal; } - if (lowValue->pool != highValue->pool) { + if (lowValue->props.pool != highValue->props.pool) { LOG_FATAL("pool mismatch"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_fatal; } - if (lowValue->size + highValue->size != totalSize) { + if (lowValue->props.base_size + highValue->props.base_size != totalSize) { LOG_FATAL("lowValue->size + highValue->size != totalSize"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_fatal; @@ -824,7 +841,8 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, } // we only need to update the size of the first part - utils_atomic_store_release_u64((uint64_t *)&lowValue->size, totalSize); + utils_atomic_store_release_u64((uint64_t *)&lowValue->props.base_size, + totalSize); size_t low_children = lowValue->n_children; size_t high_children = highValue->n_children; @@ -950,12 +968,13 @@ static void check_if_tracker_is_empty(umf_memory_tracker_handle_t hTracker, while (1 == critnib_find(hTracker->alloc_segments_map[i], last_key, FIND_G, &rkey, (void **)&rvalue, &ref_value)) { - if (rvalue && ((rvalue->pool == pool) || pool == NULL)) { + if (rvalue && ((rvalue->props.pool == pool) || pool == NULL)) { n_items++; LOG_DEBUG( "found abandoned allocation in the tracking provider: " "pool=%p, ptr=%p, size=%zu", - (void *)rvalue->pool, (void *)rkey, (size_t)rvalue->size); + (void *)rvalue->props.pool, (void *)rkey, + (size_t)rvalue->props.base_size); } if (ref_value) { @@ -1295,6 +1314,16 @@ static umf_result_t trackingCloseIpcHandle(void *provider, void *ptr, return umf_result; } +static umf_result_t +trackingGetAllocationProperties(void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, + size_t max_property_size, void *value) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderGetAllocationProperties( + p->hUpstream, ptr, memory_property_id, max_property_size, value); +} + umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = trackingInitialize, @@ -1313,7 +1342,10 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .ext_get_ipc_handle = trackingGetIpcHandle, .ext_put_ipc_handle = trackingPutIpcHandle, .ext_open_ipc_handle = trackingOpenIpcHandle, - .ext_close_ipc_handle = trackingCloseIpcHandle}; + .ext_close_ipc_handle = trackingCloseIpcHandle, + .ext_ctl = NULL, + .ext_get_allocation_properties = trackingGetAllocationProperties, +}; static void free_ipc_cache_value(void *unused, void *ipc_cache_value) { (void)unused; diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index d7ee06c1b..9b6884928 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -20,6 +20,7 @@ #include "base_alloc.h" #include "critnib.h" +#include "memory_props_internal.h" #include "utils_concurrency.h" #ifdef __cplusplus @@ -34,18 +35,23 @@ extern umf_memory_tracker_handle_t TRACKER; umf_result_t umfMemoryTrackerCreate(umf_memory_tracker_handle_t *handle); void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle); -umf_memory_pool_handle_t umfMemoryTrackerGetPool(const void *ptr); +typedef struct tracker_alloc_info_t { + umf_memory_properties_t props; -typedef struct umf_alloc_info_t { - void *base; - size_t baseSize; - umf_memory_pool_handle_t pool; -} umf_alloc_info_t; + // number of overlapping memory regions in the next level of map falling + // within the current range + size_t n_children; +#if !defined(NDEBUG) && defined(UMF_DEVELOPER_MODE) + uint64_t is_freed; +#endif +} tracker_alloc_info_t; umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, - umf_alloc_info_t *pAllocInfo); + tracker_alloc_info_t **info); typedef struct umf_ipc_info_t { + umf_memory_properties_handle_t props; + void *base; size_t baseSize; umf_memory_provider_handle_t provider; diff --git a/src/utils/utils_common.h b/src/utils/utils_common.h index 0fa860a0e..17dd006ad 100644 --- a/src/utils/utils_common.h +++ b/src/utils/utils_common.h @@ -57,11 +57,17 @@ typedef enum umf_purge_advise_t { #define __TLS __declspec(thread) +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) + #else /* Linux */ #define __TLS __thread -#endif /* _WIN32 */ +#define LIKELY(x) __builtin_expect(!!(x), 1) +#define UNLIKELY(x) __builtin_expect(!!(x), 0) + +#endif /* !_WIN32 */ // get the address of the given string in the environment variable (or NULL) char *utils_env_var_get_str(const char *envvar, const char *str); diff --git a/src/utils/utils_level_zero.cpp b/src/utils/utils_level_zero.cpp index 6daab3e69..e2b62064c 100644 --- a/src/utils/utils_level_zero.cpp +++ b/src/utils/utils_level_zero.cpp @@ -752,6 +752,18 @@ ze_memory_type_t utils_ze_get_mem_type(ze_context_handle_t context, void *ptr) { return alloc_props.type; } +void utils_ze_get_mem_props(ze_context_handle_t context, void *ptr, + ze_memory_allocation_properties_t *alloc_props, + ze_device_handle_t *device) { + alloc_props->stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES; + alloc_props->pNext = NULL; + alloc_props->type = ZE_MEMORY_TYPE_UNKNOWN; + alloc_props->id = 0; + alloc_props->pageSize = 0; + + libze_ops.zeMemGetAllocProperties(context, ptr, alloc_props, device); +} + int64_t utils_ze_get_num_memory_properties(ze_device_handle_t device) { uint32_t pCount = 0; ze_result_t ze_result = diff --git a/src/utils/utils_level_zero.h b/src/utils/utils_level_zero.h index 00f55b351..41e183288 100644 --- a/src/utils/utils_level_zero.h +++ b/src/utils/utils_level_zero.h @@ -44,6 +44,10 @@ int utils_ze_destroy_context(ze_context_handle_t context); ze_memory_type_t utils_ze_get_mem_type(ze_context_handle_t context, void *ptr); +void utils_ze_get_mem_props(ze_context_handle_t context, void *ptr, + ze_memory_allocation_properties_t *alloc_props, + ze_device_handle_t *device); + int64_t utils_ze_get_num_memory_properties(ze_device_handle_t device); #ifdef __cplusplus diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bda44ea04..e6d92fdad 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -54,11 +54,13 @@ function(build_umf_test) if(UMF_CUDA_ENABLED) set(INC_DIRS ${INC_DIRS} ${CUDA_INCLUDE_DIRS}) set(LIB_DIRS ${LIB_DIRS} ${CUDA_LIBRARY_DIRS}) + set(CPL_DEFS ${CPL_DEFS} UMF_CUDA_ENABLED=1) endif() if(UMF_LEVEL_ZERO_ENABLED) set(INC_DIRS ${INC_DIRS} ${LEVEL_ZERO_INCLUDE_DIRS}) set(LIB_DIRS ${LIB_DIRS} ${ZE_LOADER_LIBRARY_DIRS}) + set(CPL_DEFS ${CPL_DEFS} UMF_LEVEL_ZERO_ENABLED=1) endif() if(NOT UMF_DISABLE_HWLOC) @@ -358,6 +360,10 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented NAME provider_tracking_fixture_tests SRCS provider_tracking_fixture_tests.cpp malloc_compliance_tests.cpp LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) + add_umf_test( + NAME provider_props + SRCS props/provider_props.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) @@ -429,6 +435,12 @@ if(UMF_BUILD_GPU_TESTS AND UMF_LEVEL_ZERO_ENABLED) LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) target_compile_definitions(test_provider_level_zero_dlopen_local PUBLIC USE_DLOPEN=1 OPEN_ZE_LIBRARY_GLOBAL=0) + + add_umf_test( + NAME provider_props_level_zero + SRCS props/provider_props_level_zero.cpp + ${UMF_UTILS_DIR}/utils_level_zero.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST} ze_loader) endif() if(NOT UMF_BUILD_LEVEL_ZERO_PROVIDER) @@ -461,6 +473,11 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_CUDA_PROVIDER) LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) target_compile_definitions(test_provider_cuda_dlopen_local PUBLIC USE_DLOPEN=1 OPEN_CU_LIBRARY_GLOBAL=0) + + add_umf_test( + NAME provider_props_cuda + SRCS props/provider_props_cuda.cpp providers/cuda_helpers.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST} cuda) else() message( STATUS diff --git a/test/common/pool_null.c b/test/common/pool_null.c index ccabfee56..60433bee6 100644 --- a/test/common/pool_null.c +++ b/test/common/pool_null.c @@ -5,9 +5,10 @@ #include #include -#include "pool_null.h" #include +#include "pool_null.h" + static umf_result_t nullInitialize(umf_memory_provider_handle_t provider, const void *params, void **pool) { (void)provider; diff --git a/test/common/pool_trace.c b/test/common/pool_trace.c index ce944479f..04b2ded88 100644 --- a/test/common/pool_trace.c +++ b/test/common/pool_trace.c @@ -5,9 +5,10 @@ #include #include -#include "pool_trace.h" #include +#include "pool_trace.h" + typedef struct trace_pool { umf_pool_trace_params_t params; } trace_pool_t; diff --git a/test/common/provider_null.c b/test/common/provider_null.c index 380cba47d..5940e4565 100644 --- a/test/common/provider_null.c +++ b/test/common/provider_null.c @@ -5,9 +5,11 @@ #include #include -#include "provider_null.h" +#include #include +#include "provider_null.h" + static umf_result_t nullInitialize(const void *params, void **pool) { (void)params; *pool = NULL; @@ -133,6 +135,18 @@ static umf_result_t nullCloseIpcHandle(void *provider, void *ptr, size_t size) { return UMF_RESULT_SUCCESS; } +static umf_result_t +nullGetAllocationProperties(void *provider, const void *ptr, + umf_memory_property_id_t propertyId, + size_t max_property_size, void *value) { + (void)provider; + (void)ptr; + (void)propertyId; + (void)max_property_size; + (void)value; + return UMF_RESULT_SUCCESS; +} + umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = nullInitialize, @@ -152,4 +166,5 @@ umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .ext_put_ipc_handle = nullPutIpcHandle, .ext_open_ipc_handle = nullOpenIpcHandle, .ext_close_ipc_handle = nullCloseIpcHandle, + .ext_get_allocation_properties = nullGetAllocationProperties, }; diff --git a/test/common/test_helpers.h b/test/common/test_helpers.h index 494528b57..4f1318c55 100644 --- a/test/common/test_helpers.h +++ b/test/common/test_helpers.h @@ -1,4 +1,4 @@ -// Copyright (C) 2023-2024 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // This file contains helpers for tests for UMF pool API @@ -9,8 +9,10 @@ #include #include #include + #include #include +#include #include #include "provider_trace.h" diff --git a/test/memoryProviderAPI.cpp b/test/memoryProviderAPI.cpp index 33e298dc6..d03498fda 100644 --- a/test/memoryProviderAPI.cpp +++ b/test/memoryProviderAPI.cpp @@ -330,6 +330,21 @@ TEST_F(test, memoryProviderOpsNullAllocationSplitAllocationMergeNegative) { umfMemoryProviderDestroy(hProvider); } +TEST_F(test, memoryProviderOpsNullGetAllocationProperties) { + umf_memory_provider_ops_t provider_ops = UMF_NULL_PROVIDER_OPS; + umf_memory_provider_handle_t hProvider; + + umf_result_t ret = + umfMemoryProviderCreate(&provider_ops, nullptr, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + ret = umfMemoryProviderGetAllocationProperties( + hProvider, nullptr, UMF_MEMORY_PROPERTY_PROVIDER_HANDLE, 0, nullptr); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umfMemoryProviderDestroy(hProvider); +} + struct providerInitializeTest : umf_test::test, ::testing::WithParamInterface {}; diff --git a/test/props/provider_props.cpp b/test/props/provider_props.cpp new file mode 100644 index 000000000..a9e6865e9 --- /dev/null +++ b/test/props/provider_props.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include "provider_props.hpp" + +void createFixedProvider(umf_memory_provider_handle_t *out_provider, + void *out_data) { + constexpr size_t buffer_size = 1024 * 1024; + + void *memory_buffer = malloc(buffer_size); + ASSERT_NE(memory_buffer, nullptr); + + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = + umfFixedMemoryProviderParamsCreate(memory_buffer, buffer_size, ¶ms); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + res = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + out_provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(out_provider, nullptr); + + umfFixedMemoryProviderParamsDestroy(params); + + *(uintptr_t *)out_data = (uintptr_t)memory_buffer; +} + +void destroyFixedProvider(umf_memory_provider_handle_t provider, void *data) { + umfMemoryProviderDestroy(provider); + free(data); +} + +void createOsMemoryProvider(umf_memory_provider_handle_t *out_provider, + void *out_data) { + + umf_os_memory_provider_params_handle_t os_memory_provider_params = nullptr; + umf_result_t res = + umfOsMemoryProviderParamsCreate(&os_memory_provider_params); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(os_memory_provider_params, nullptr); + + umf_memory_provider_handle_t os_memory_provider = nullptr; + res = + umfMemoryProviderCreate(umfOsMemoryProviderOps(), + os_memory_provider_params, &os_memory_provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(os_memory_provider, nullptr); + + res = umfOsMemoryProviderParamsDestroy(os_memory_provider_params); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + + *out_provider = os_memory_provider; + *(uintptr_t *)out_data = (uintptr_t)NULL; +} + +void destroyOsMemoryProvider(umf_memory_provider_handle_t provider, + void *data) { + (void)data; // unused + + umfMemoryProviderDestroy(provider); +} + +INSTANTIATE_TEST_SUITE_P( + providerPropsTest, ProviderPropsTest, + ::testing::Values(testParams{createFixedProvider, destroyFixedProvider, + "fixedProvider"}, + testParams{createOsMemoryProvider, + destroyOsMemoryProvider, "osMemoryProvider"}), + nameGen); diff --git a/test/props/provider_props.hpp b/test/props/provider_props.hpp new file mode 100644 index 000000000..5a9ba67c9 --- /dev/null +++ b/test/props/provider_props.hpp @@ -0,0 +1,238 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include +#include +#include + +#include "base.hpp" +#include "test_helpers.h" + +using umf_test::test; + +using testParams = + std::tuple, + std::function, + const char *>; + +std::string nameGen(const testing::TestParamInfo param) { + return std::get<2>(param.param); +} + +struct ProviderPropsTest : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + + auto [create_fun, destroy_fun, name] = this->GetParam(); + provider_create = create_fun; + provider_destroy = destroy_fun; + (void)name; // unused + + provider_create(&provider, &data); + ASSERT_NE(provider, nullptr); + + umf_result_t umf_result = + umfPoolCreate(umfProxyPoolOps(), provider, nullptr, 0, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + void TearDown() override { + umfPoolDestroy(pool); + provider_destroy(provider, data); + test::TearDown(); + } + + umf_memory_provider_handle_t provider; + umf_memory_pool_handle_t pool; + + std::function provider_create; + std::function provider_destroy; + void *data; +}; + +TEST_P(ProviderPropsTest, genericProps) { + umf_result_t umf_result; + const size_t alloc_size = 8; + + void *ptr = umfPoolMalloc(pool, alloc_size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + umf_memory_provider_handle_t param_provider = nullptr; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_PROVIDER_HANDLE, + sizeof(param_provider), ¶m_provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(param_provider, provider); + + umf_memory_pool_handle_t param_pool = nullptr; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_POOL_HANDLE, + sizeof(param_pool), ¶m_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(param_pool, pool); + + void *base_address = nullptr; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(base_address), &base_address); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(base_address, ptr); + + size_t size = 0; + umf_result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_BASE_SIZE, sizeof(size), &size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(size, alloc_size); + + uint64_t buffer_id = 0; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + sizeof(buffer_id), &buffer_id); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_GE(buffer_id, 0); + + umf_result = umfPoolFree(pool, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, baseAddressFromMiddle) { + umf_result_t umf_result; + const size_t alloc_size = 8; + + void *ptr = umfPoolMalloc(pool, alloc_size); + ASSERT_NE(ptr, nullptr); + + void *ptr_mid = (void *)((uintptr_t)ptr + (alloc_size / 2)); + umf_memory_properties_handle_t props_handle = nullptr; + umf_result = umfGetMemoryPropertiesHandle(ptr_mid, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + uintptr_t param_base_address = 0; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(param_base_address), ¶m_base_address); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(param_base_address, (uintptr_t)ptr); + + umf_result = umfPoolFree(pool, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, uniqueBufferId) { + size_t alloc_size = 8; + size_t num_allocs = 10; + umf_result_t umf_result; + std::set buffer_ids; + + for (size_t i = 0; i < num_allocs; ++i) { + void *ptr = umfPoolMalloc(pool, alloc_size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + uint64_t buffer_id = 0; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + sizeof(buffer_id), &buffer_id); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_GE(buffer_id, 0); + + // Ensure that the buffer ID is unique by inserting it into a set and + // checking if it was already present + ASSERT_TRUE(buffer_ids.find(buffer_id) == buffer_ids.end()); + ASSERT_TRUE(buffer_ids.insert(buffer_id).second); + + umf_result = umfPoolFree(pool, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } +} + +// Negative tests + +TEST_P(ProviderPropsTest, invalidPointer) { + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t umf_result = + umfGetMemoryPropertiesHandle(nullptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(props_handle, nullptr); + + uintptr_t invalid_ptr = 0xdeadbeef; + umf_result = + umfGetMemoryPropertiesHandle((void *)invalid_ptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(props_handle, nullptr); +} + +TEST_P(ProviderPropsTest, invalidPropertyId) { + void *ptr = umfPoolMalloc(pool, 8); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t res = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + void *value = nullptr; + res = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_INVALID, + sizeof(value), &value); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, invalidPropertyValue) { + void *ptr = umfPoolMalloc(pool, 8); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t res = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + res = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(int), NULL); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, invalidPropertySize) { + void *ptr = umfPoolMalloc(pool, 8); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t res = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + int value = 0; + res = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + size_t(0), &value); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, nullPropertiesHandle) { + int val = 0; + umf_result_t res = umfGetMemoryProperty( + NULL, UMF_MEMORY_PROPERTY_BASE_ADDRESS, sizeof(val), &val); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} diff --git a/test/props/provider_props_cuda.cpp b/test/props/provider_props_cuda.cpp new file mode 100644 index 000000000..72f6cf17d --- /dev/null +++ b/test/props/provider_props_cuda.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include "provider_props.hpp" +#include "providers/cuda_helpers.h" + +void createCudaMemoryProvider(umf_memory_provider_handle_t *out_provider, + void *out_data) { + CUdevice hDevice = -1; + CUcontext hContext = NULL; + + int ret = init_cuda(); + ASSERT_EQ(ret, 0); + + ret = get_cuda_device(&hDevice); + ASSERT_EQ(ret, 0); + ASSERT_NE(hDevice, -1); + + ret = create_context(hDevice, &hContext); + ASSERT_EQ(ret, 0); + ASSERT_NE(hContext, nullptr); + + umf_cuda_memory_provider_params_handle_t cu_params = NULL; + umf_result_t umf_result = umfCUDAMemoryProviderParamsCreate(&cu_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(cu_params, nullptr); + + umf_result = umfCUDAMemoryProviderParamsSetContext(cu_params, hContext); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfCUDAMemoryProviderParamsSetDevice(cu_params, hDevice); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfCUDAMemoryProviderParamsSetMemoryType( + cu_params, UMF_MEMORY_TYPE_DEVICE); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_memory_provider_handle_t provider = nullptr; + umf_result = umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), cu_params, + &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umfCUDAMemoryProviderParamsDestroy(cu_params); + + *out_provider = provider; + *(uintptr_t *)out_data = (uintptr_t)hContext; +} + +void destroyCudaMemoryProvider(umf_memory_provider_handle_t provider, + void *data) { + destroy_context((CUcontext)data); + umfMemoryProviderDestroy(provider); +} + +INSTANTIATE_TEST_SUITE_P(providerPropsTest, ProviderPropsTest, + ::testing::Values(testParams{createCudaMemoryProvider, + destroyCudaMemoryProvider, + "cudaMemoryProvider"}), + nameGen); diff --git a/test/props/provider_props_level_zero.cpp b/test/props/provider_props_level_zero.cpp new file mode 100644 index 000000000..23f1bbc4e --- /dev/null +++ b/test/props/provider_props_level_zero.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include "provider_props.hpp" +#include "utils/utils_level_zero.h" + +void levelZeroMemoryProviderCreate(umf_memory_provider_handle_t *out_provider, + void *out_data) { + + ze_driver_handle_t hDriver = nullptr; + ze_device_handle_t hDevice = nullptr; + ze_context_handle_t hContext = nullptr; + uint32_t driver_idx = 0; + + int ret = utils_ze_init_level_zero(); + ASSERT_EQ(ret, 0); + + ret = utils_ze_find_driver_with_gpu(&driver_idx, &hDriver); + ASSERT_EQ(ret, 0); + + ret = utils_ze_find_gpu_device(hDriver, &hDevice); + ASSERT_EQ(ret, 0); + + ret = utils_ze_create_context(hDriver, &hContext); + ASSERT_EQ(ret, 0); + + umf_level_zero_memory_provider_params_handle_t params = nullptr; + umf_result_t result = umfLevelZeroMemoryProviderParamsCreate(¶ms); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + result = umfLevelZeroMemoryProviderParamsSetContext(params, hContext); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + result = umfLevelZeroMemoryProviderParamsSetDevice(params, hDevice); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + result = umfLevelZeroMemoryProviderParamsSetMemoryType( + params, UMF_MEMORY_TYPE_DEVICE); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + + umf_memory_provider_handle_t provider = nullptr; + umf_result_t umf_result = umfMemoryProviderCreate( + umfLevelZeroMemoryProviderOps(), params, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + result = umfLevelZeroMemoryProviderParamsDestroy(params); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + + *out_provider = provider; + *(uintptr_t *)out_data = (uintptr_t)hContext; +} + +void levelZeroMemoryProviderDestroy(umf_memory_provider_handle_t provider, + void *data) { + umfMemoryProviderDestroy(provider); + utils_ze_destroy_context((ze_context_handle_t)data); +} + +INSTANTIATE_TEST_SUITE_P(providerPropsTest, ProviderPropsTest, + ::testing::Values(testParams{ + levelZeroMemoryProviderCreate, + levelZeroMemoryProviderDestroy, + "levelZeroProvider"}), + nameGen); diff --git a/test/providers/provider_cuda.cpp b/test/providers/provider_cuda.cpp index 9b0d29f4b..4d1b94ea9 100644 --- a/test/providers/provider_cuda.cpp +++ b/test/providers/provider_cuda.cpp @@ -9,6 +9,7 @@ #include +#include #include #include "cuda_helpers.h" @@ -496,6 +497,89 @@ TEST_P(umfCUDAProviderTest, multiContext) { ASSERT_EQ(ret, 0); } +TEST_P(umfCUDAProviderTest, memProps) { + umf_memory_provider_handle_t provider = nullptr; + umf_result_t umf_result = + umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), params, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umf_memory_pool_handle_t pool = NULL; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider, NULL, 0, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_t size = 1024; + void *ptr = umfPoolMalloc(pool, size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = NULL; + umf_result_t result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + umf_usm_memory_type_t type = UMF_MEMORY_TYPE_UNKNOWN; + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, sizeof(type), &type); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, expected_memory_type); + + // TODO move generic tests for memory properties to some common code + // base address and size + void *baseAddress = nullptr; + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(baseAddress), &baseAddress); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + size_t baseSize = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_SIZE, + sizeof(baseSize), &baseSize); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(baseSize, size); + + int64_t bufferId = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + sizeof(bufferId), &bufferId); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(bufferId, 0); + + if (expected_memory_type != UMF_MEMORY_TYPE_HOST) { + CUdevice device = -1; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_DEVICE, + sizeof(device), &device); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(device, cudaTestHelper.get_test_device()); + } + + CUcontext context = nullptr; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_CONTEXT, + sizeof(context), &context); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(context, cudaTestHelper.get_test_context()); + + // check the props of pointer from the middle of alloc + void *midPtr = static_cast(ptr) + size / 2; + result = umfGetMemoryPropertiesHandle(midPtr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, sizeof(type), &type); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, expected_memory_type); + + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(baseAddress), &baseAddress); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + umfFree(ptr); + + umfPoolDestroy(pool); + umfMemoryProviderDestroy(provider); +} + struct umfCUDAProviderAllocFlagsTest : umf_test::test, ::testing::WithParamInterface< diff --git a/test/providers/provider_level_zero.cpp b/test/providers/provider_level_zero.cpp index d6d479a14..8629989f0 100644 --- a/test/providers/provider_level_zero.cpp +++ b/test/providers/provider_level_zero.cpp @@ -9,6 +9,7 @@ #include +#include #include #include "ipcFixtures.hpp" @@ -246,6 +247,7 @@ struct umfLevelZeroProviderTest test::SetUp(); umf_usm_memory_type_t memory_type = this->GetParam(); + umfExpectedMemoryType = memory_type; params = nullptr; memAccessor = nullptr; @@ -298,6 +300,7 @@ struct umfLevelZeroProviderTest std::unique_ptr memAccessor = nullptr; ze_context_handle_t hContext = nullptr; ze_memory_type_t zeMemoryTypeExpected = ZE_MEMORY_TYPE_UNKNOWN; + umf_usm_memory_type_t umfExpectedMemoryType = UMF_MEMORY_TYPE_UNKNOWN; }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(umfLevelZeroProviderTest); @@ -484,6 +487,89 @@ TEST_P(umfLevelZeroProviderTest, setDeviceOrdinalValid) { } } +TEST_P(umfLevelZeroProviderTest, memProps) { + umf_memory_provider_handle_t provider = nullptr; + umf_result_t umf_result = umfMemoryProviderCreate( + umfLevelZeroMemoryProviderOps(), params, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umf_memory_pool_handle_t pool = NULL; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider, NULL, 0, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_t size = 1024; + void *ptr = umfPoolMalloc(pool, size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = NULL; + umf_result_t result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + umf_usm_memory_type_t type = UMF_MEMORY_TYPE_UNKNOWN; + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, sizeof(type), &type); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, umfExpectedMemoryType); + + // TODO move generic tests for memory properties to some common code + // base address and size + void *baseAddress = nullptr; + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(baseAddress), &baseAddress); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + size_t baseSize = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_SIZE, + sizeof(baseSize), &baseSize); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(baseSize, size); + + int64_t bufferId = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + sizeof(bufferId), &bufferId); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(bufferId, 0); + + if (umfExpectedMemoryType != UMF_MEMORY_TYPE_HOST) { + ze_device_handle_t device = nullptr; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_DEVICE, + sizeof(device), &device); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(device, l0TestHelper.get_test_device()); + } + + ze_context_handle_t context = nullptr; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_CONTEXT, + sizeof(context), &context); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(context, l0TestHelper.get_test_context()); + + // check the props of pointer from the middle of alloc + void *midPtr = static_cast(ptr) + size / 2; + result = umfGetMemoryPropertiesHandle(midPtr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, sizeof(type), &type); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, umfExpectedMemoryType); + + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + sizeof(baseAddress), &baseAddress); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + umfFree(ptr); + + umfPoolDestroy(pool); + umfMemoryProviderDestroy(provider); +} + // TODO add tests that mixes Level Zero Memory Provider and Disjoint Pool INSTANTIATE_TEST_SUITE_P(umfLevelZeroProviderTestSuite, From 84b24db7bc397ad240c000d3642d7d00febdc99e Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 4 Jul 2025 16:20:26 +0000 Subject: [PATCH 2/2] opt critnib --- src/critnib/critnib.c | 5 ++- src/critnib/critnib.h | 2 +- src/provider/provider_file_memory.c | 2 +- src/provider/provider_tracking.c | 68 +++++++++++++++++++++-------- 4 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index e8f5961c9..3daacfbec 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -1009,7 +1009,7 @@ static struct critnib_leaf *find_ge(struct critnib_node *__restrict n, * critnib_find() returns 0 if ref is NULL */ int critnib_find(struct critnib *c, uintptr_t key, enum find_dir_t dir, - uintptr_t *rkey, void **rvalue, void **ref) { + uintptr_t *rkey, void **rvalue, void **ref, bool weak) { uint64_t wrs1, wrs2; struct critnib_leaf *k; uintptr_t _rkey = (uintptr_t)0x0; @@ -1067,7 +1067,8 @@ int critnib_find(struct critnib *c, uintptr_t key, enum find_dir_t dir, } while (wrs1 + DELETED_LIFE <= wrs2); if (k) { - if (c->cb_free_leaf) { + + if (weak == false && c->cb_free_leaf) { if (increment_ref_count(k)) { return 0; } diff --git a/src/critnib/critnib.h b/src/critnib/critnib.h index 690d75fae..fb76e1cf9 100644 --- a/src/critnib/critnib.h +++ b/src/critnib/critnib.h @@ -51,7 +51,7 @@ void *critnib_remove(critnib *c, uintptr_t key, void **ref); void *critnib_get(critnib *c, uintptr_t key, void **ref); void *critnib_find_le(critnib *c, uintptr_t key, void **ref); int critnib_find(critnib *c, uintptr_t key, enum find_dir_t dir, - uintptr_t *rkey, void **rvalue, void **ref); + uintptr_t *rkey, void **rvalue, void **ref, bool weak); int critnib_release(struct critnib *c, void *ref); #ifdef __cplusplus diff --git a/src/provider/provider_file_memory.c b/src/provider/provider_file_memory.c index 7956b3c79..ddcc67c25 100644 --- a/src/provider/provider_file_memory.c +++ b/src/provider/provider_file_memory.c @@ -313,7 +313,7 @@ static umf_result_t file_finalize(void *provider) { void *rvalue = NULL; umf_result_t ret = UMF_RESULT_SUCCESS; while (1 == critnib_find(file_provider->mmaps, key, FIND_G, &rkey, &rvalue, - NULL)) { + NULL, false /* weak */)) { utils_munmap((void *)rkey, (size_t)rvalue); critnib_remove(file_provider->mmaps, rkey, NULL); key = rkey; diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index ff51ed4ba..b70b963ff 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -84,9 +84,9 @@ static tracker_alloc_info_t *get_most_nested_alloc_segment( do { assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); - found = - critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, - FIND_LE, (void *)&rkey, (void **)&rvalue, &ref_value); + found = critnib_find(hTracker->alloc_segments_map[level], + (uintptr_t)ptr, FIND_LE, (void *)&rkey, + (void **)&rvalue, &ref_value, false /* weak */); if (!found) { assert(ref_value == NULL); break; @@ -270,9 +270,9 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, // in the critnib maps that contains the given 'ptr' pointer. do { assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); - found = - critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, - FIND_LE, (void *)&rkey, (void **)&rvalue, &ref_value); + found = critnib_find(hTracker->alloc_segments_map[level], + (uintptr_t)ptr, FIND_LE, (void *)&rkey, + (void **)&rvalue, &ref_value, false /* weak */); if (!found) { assert(ref_value == NULL); break; @@ -501,7 +501,11 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, return UMF_RESULT_ERROR_NOT_SUPPORTED; } +#define CCC 0 + +#if CCC tracker_alloc_info_t *top_most_value = NULL; +#endif tracker_alloc_info_t *rvalue = NULL; //uintptr_t top_most_key = 0; uintptr_t rkey = 0; @@ -511,19 +515,24 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, int found = 0; void *ref_value = NULL; + +#if CCC void *ref_top_most_value = NULL; int ref_level = 0; do { assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); - found = - critnib_find(TRACKER->alloc_segments_map[level], (uintptr_t)ptr, - FIND_LE, (void *)&rkey, (void **)&rvalue, &ref_value); +#endif + + found = critnib_find(TRACKER->alloc_segments_map[level], (uintptr_t)ptr, + FIND_LE, (void *)&rkey, (void **)&rvalue, + &ref_value, true /* weak */); if (!found) { assert(ref_value == NULL); - break; + return UMF_RESULT_ERROR_INVALID_ARGUMENT; } +#if CCC if (!rvalue) { if (ref_value) { critnib_release(TRACKER->alloc_segments_map[level], ref_value); @@ -539,10 +548,13 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, ref_level = 0; continue; } +#endif utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, &rsize); utils_atomic_load_acquire_size_t(&rvalue->n_children, &n_children); + +#if CCC if (found && (uintptr_t)ptr < rkey + rsize) { //top_most_key = rkey; top_most_value = rvalue; @@ -560,7 +572,19 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, level++; } } while (found && (uintptr_t)ptr < rkey + rsize && n_children); +#else + if ((uintptr_t)ptr < rkey + rsize) { + *info = rvalue; + return UMF_RESULT_SUCCESS; + } else { + *info = NULL; + LOG_DEBUG("pointer %p not found in the tracker, TRACKER=%p", ptr, + (void *)TRACKER); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } +#endif +#if CCC if (!top_most_value) { if (ref_value) { critnib_release(TRACKER->alloc_segments_map[level], ref_value); @@ -570,13 +594,16 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, (void *)TRACKER); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } +#endif +#if CCC *info = top_most_value; assert(ref_top_most_value); critnib_release(TRACKER->alloc_segments_map[ref_level], ref_top_most_value); return UMF_RESULT_SUCCESS; +#endif } umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, @@ -601,15 +628,18 @@ umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, tracker_ipc_info_t *rvalue = NULL; void *ref_value = NULL; int found = critnib_find(TRACKER->ipc_segments_map, (uintptr_t)ptr, FIND_LE, - (void *)&rkey, (void **)&rvalue, &ref_value); + (void *)&rkey, (void **)&rvalue, &ref_value, + true /* weak */); if (!found || (found && (rvalue == NULL)) || ((uintptr_t)ptr >= rkey + rvalue->size)) { LOG_DEBUG("pointer %p not found in the tracker, TRACKER=%p", ptr, (void *)TRACKER); + /* if (ref_value) { critnib_release(TRACKER->ipc_segments_map, ref_value); } + */ return UMF_RESULT_ERROR_INVALID_ARGUMENT; } @@ -619,9 +649,11 @@ umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, pIpcInfo->props = &rvalue->props; + /* if (ref_value) { critnib_release(TRACKER->ipc_segments_map, ref_value); } + */ return UMF_RESULT_SUCCESS; } @@ -754,9 +786,9 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, utils_mutex_unlock(&provider->hTracker->splitMergeMutex); - LOG_DEBUG( - "split memory region (level=%i): ptr=%p, totalSize=%zu, firstSize=%zu", - level, ptr, totalSize, firstSize); + LOG_DEBUG("split memory region (level=%i): ptr=%p, totalSize=%zu, " + "firstSize=%zu", + level, ptr, totalSize, firstSize); return UMF_RESULT_SUCCESS; @@ -764,9 +796,9 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, utils_mutex_unlock(&provider->hTracker->splitMergeMutex); err_lock: - LOG_ERR( - "failed to split memory region: ptr=%p, totalSize=%zu, firstSize=%zu", - ptr, totalSize, firstSize); + LOG_ERR("failed to split memory region: ptr=%p, totalSize=%zu, " + "firstSize=%zu", + ptr, totalSize, firstSize); return ret; } @@ -967,7 +999,7 @@ static void check_if_tracker_is_empty(umf_memory_tracker_handle_t hTracker, void *ref_value = NULL; while (1 == critnib_find(hTracker->alloc_segments_map[i], last_key, - FIND_G, &rkey, (void **)&rvalue, &ref_value)) { + FIND_G, &rkey, (void **)&rvalue, &ref_value, false /* weak */)) { if (rvalue && ((rvalue->props.pool == pool) || pool == NULL)) { n_items++; LOG_DEBUG(