Skip to content

Commit bac2f44

Browse files
martygrantmartygrant
committed
Change implementation to a new URRefCount class and add it as member to any relevant handles that need ref counting.
1 parent 156bd2c commit bac2f44

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+231
-105
lines changed

‎unified-runtime/source/adapters/level_zero/adapter.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
301301
ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED;
302302
ZesResult = ZE_RESULT_ERROR_UNINITIALIZED;
303303

304-
resetRefCount(0);
304+
RefCount.reset(0);
305305

306306
#ifdef UR_STATIC_LEVEL_ZERO
307307
// Given static linking of the L0 Loader, we must delay the loader's
@@ -677,7 +677,7 @@ ur_result_t urAdapterGet(
677677
}
678678
*Adapters = GlobalAdapter;
679679

680-
if (GlobalAdapter->incrementRefCount() == 0) {
680+
if (GlobalAdapter->getRefCount().increment() == 0) {
681681
adapterStateInit();
682682
}
683683
}
@@ -694,7 +694,7 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) {
694694

695695
// NOTE: This does not require guarding with a mutex; the instant the ref
696696
// count hits zero, both Get and Retain are UB.
697-
if (GlobalAdapter->decrementRefCount() == 0) {
697+
if (GlobalAdapter->getRefCount().decrementAndTest()) {
698698
auto result = adapterStateTeardown();
699699
#ifdef UR_STATIC_LEVEL_ZERO
700700
// Given static linking of the L0 Loader, we must delay the loader's
@@ -711,9 +711,9 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) {
711711
return UR_RESULT_SUCCESS;
712712
}
713713

714-
ur_result_t urAdapterRetain(ur_adapter_handle_t) {
714+
ur_result_t urAdapterRetain([[maybe_unused]] ur_adapter_handle_t Adapter) {
715715
assert(GlobalAdapter && GlobalAdapter == Adapter);
716-
GlobalAdapter->incrementRefCount();
716+
GlobalAdapter->getRefCount().increment();
717717

718718
return UR_RESULT_SUCCESS;
719719
}
@@ -742,12 +742,12 @@ ur_result_t urAdapterGetInfo(ur_adapter_handle_t, ur_adapter_info_t PropName,
742742
case UR_ADAPTER_INFO_BACKEND:
743743
return ReturnValue(UR_BACKEND_LEVEL_ZERO);
744744
case UR_ADAPTER_INFO_REFERENCE_COUNT:
745-
return ReturnValue(GlobalAdapter->getRefCount());
745+
return ReturnValue(GlobalAdapter->getRefCount().getCount());
746746
case UR_ADAPTER_INFO_VERSION: {
747747
#ifdef UR_ADAPTER_LEVEL_ZERO_V2
748748
uint32_t adapterVersion = 2;
749749
#else
750-
uint32_t adapterVersion = 1;
750+
uint32_t adapterVersion = 1;
751751
#endif
752752
return ReturnValue(adapterVersion);
753753
}

‎unified-runtime/source/adapters/level_zero/adapter.hpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
//===----------------------------------------------------------------------===//
1010
#pragma once
1111

12+
#include "common/ur_ref_count.hpp"
1213
#include "logger/ur_logger.hpp"
1314
#include "ur_interface_loader.hpp"
1415
#include <loader/ur_loader.hpp>
@@ -43,6 +44,11 @@ struct ur_adapter_handle_t_ : ur::handle_base<ur::level_zero::ddi_getter> {
4344
ZeCache<Result<PlatformVec>> PlatformCache;
4445
logger::Logger &logger;
4546
HMODULE processHandle = nullptr;
47+
48+
URRefCount &getRefCount() noexcept { return RefCount; }
49+
50+
private:
51+
URRefCount RefCount;
4652
};
4753

4854
extern ur_adapter_handle_t_ *GlobalAdapter;

‎unified-runtime/source/adapters/level_zero/async_alloc.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ ur_result_t urEnqueueUSMFreeExp(
247247
}
248248

249249
size_t size = umfPoolMallocUsableSize(hPool, Mem);
250-
(*Event)->incrementRefCount();
250+
(*Event)->getRefCount().increment();
251251
usmPool->AsyncPool.insert(Mem, size, *Event, Queue);
252252

253253
// Signal that USM free event was finished

‎unified-runtime/source/adapters/level_zero/command_buffer.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -842,13 +842,13 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
842842

843843
ur_result_t
844844
urCommandBufferRetainExp(ur_exp_command_buffer_handle_t CommandBuffer) {
845-
CommandBuffer->incrementRefCount();
845+
CommandBuffer->getRefCount().increment();
846846
return UR_RESULT_SUCCESS;
847847
}
848848

849849
ur_result_t
850850
urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t CommandBuffer) {
851-
if (!CommandBuffer->decrementAndTest())
851+
if (!CommandBuffer->getRefCount().decrementAndTest())
852852
return UR_RESULT_SUCCESS;
853853

854854
UR_CALL(waitForOngoingExecution(CommandBuffer));
@@ -1643,7 +1643,7 @@ ur_result_t enqueueImmediateAppendPath(
16431643
if (CommandBuffer->CurrentSubmissionEvent) {
16441644
UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent));
16451645
}
1646-
(*Event)->incrementRefCount();
1646+
(*Event)->getRefCount().increment();
16471647
CommandBuffer->CurrentSubmissionEvent = *Event;
16481648

16491649
UR_CALL(Queue->executeCommandList(CommandListHelper, false, false));
@@ -1726,7 +1726,7 @@ ur_result_t enqueueWaitEventPath(ur_exp_command_buffer_handle_t CommandBuffer,
17261726
if (CommandBuffer->CurrentSubmissionEvent) {
17271727
UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent));
17281728
}
1729-
(*Event)->incrementRefCount();
1729+
(*Event)->getRefCount().increment();
17301730
CommandBuffer->CurrentSubmissionEvent = *Event;
17311731

17321732
UR_CALL(Queue->executeCommandList(SignalCommandList, false /*IsBlocking*/,
@@ -1850,7 +1850,7 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer,
18501850

18511851
switch (propName) {
18521852
case UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT:
1853-
return ReturnValue(uint32_t{hCommandBuffer->getRefCount()});
1853+
return ReturnValue(uint32_t{hCommandBuffer->getRefCount().getCount()});
18541854
case UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR: {
18551855
ur_exp_command_buffer_desc_t Descriptor{};
18561856
Descriptor.stype = UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC;

‎unified-runtime/source/adapters/level_zero/command_buffer.hpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include "common.hpp"
1919

20+
#include "common/ur_ref_count.hpp"
2021
#include "context.hpp"
2122
#include "kernel.hpp"
2223
#include "queue.hpp"
@@ -149,4 +150,9 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object {
149150
// Track handle objects to free when command-buffer is destroyed.
150151
std::vector<std::unique_ptr<ur_exp_command_buffer_command_handle_t_>>
151152
CommandHandles;
153+
154+
URRefCount &getRefCount() noexcept { return RefCount; }
155+
156+
private:
157+
URRefCount RefCount;
152158
};

‎unified-runtime/source/adapters/level_zero/common.hpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include <level_zero/ze_intel_gpu.h>
3535
#include <umf_pools/disjoint_pool_config_parser.hpp>
3636

37+
#include "common/ur_ref_count.hpp"
3738
#include "logger/ur_logger.hpp"
3839
#include "ur_interface_loader.hpp"
3940

@@ -250,6 +251,11 @@ struct MemAllocRecord : ur_object {
250251
// TODO: this should go away when memory isolation issue is fixed in the Level
251252
// Zero runtime.
252253
ur_context_handle_t Context;
254+
255+
URRefCount &getRefCount() noexcept { return RefCount; }
256+
257+
private:
258+
URRefCount RefCount;
253259
};
254260

255261
extern usm::DisjointPoolAllConfigs DisjointPoolConfigInstance;

‎unified-runtime/source/adapters/level_zero/context.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ ur_result_t urContextRetain(
6161

6262
/// [in] handle of the context to get a reference of.
6363
ur_context_handle_t Context) {
64-
Context->incrementRefCount();
64+
Context->getRefCount().increment();
6565
return UR_RESULT_SUCCESS;
6666
}
6767

@@ -113,7 +113,7 @@ ur_result_t urContextGetInfo(
113113
case UR_CONTEXT_INFO_NUM_DEVICES:
114114
return ReturnValue(uint32_t(Context->Devices.size()));
115115
case UR_CONTEXT_INFO_REFERENCE_COUNT:
116-
return ReturnValue(uint32_t{Context->getRefCount()});
116+
return ReturnValue(uint32_t{Context->getRefCount().getCount()});
117117
case UR_CONTEXT_INFO_USM_MEMCPY2D_SUPPORT:
118118
// 2D USM memcpy is supported.
119119
return ReturnValue(uint8_t{UseMemcpy2DOperations});
@@ -251,7 +251,7 @@ ur_device_handle_t ur_context_handle_t_::getRootDevice() const {
251251
// from the list of tracked contexts.
252252
ur_result_t ContextReleaseHelper(ur_context_handle_t Context) {
253253

254-
if (!Context->decrementAndTest())
254+
if (!Context->getRefCount().decrementAndTest())
255255
return UR_RESULT_SUCCESS;
256256

257257
if (IndirectAccessTrackingEnabled) {

‎unified-runtime/source/adapters/level_zero/context.hpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "queue.hpp"
2727
#include "usm.hpp"
2828

29+
#include "common/ur_ref_count.hpp"
2930
#include <umf_helpers.hpp>
3031

3132
struct l0_command_list_cache_info {
@@ -358,6 +359,8 @@ struct ur_context_handle_t_ : ur_object {
358359
// Get handle to the L0 context
359360
ze_context_handle_t getZeHandle() const;
360361

362+
URRefCount &getRefCount() noexcept { return RefCount; }
363+
361364
private:
362365
enum EventFlags {
363366
EVENT_FLAG_HOST_VISIBLE = UR_BIT(0),
@@ -404,6 +407,8 @@ struct ur_context_handle_t_ : ur_object {
404407

405408
return &EventCaches[index];
406409
}
410+
411+
URRefCount RefCount;
407412
};
408413

409414
// Helper function to release the context, a caller must lock the platform-level

‎unified-runtime/source/adapters/level_zero/device.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -470,7 +470,7 @@ ur_result_t urDeviceGetInfo(
470470
return ReturnValue((uint32_t)Device->SubDevices.size());
471471
}
472472
case UR_DEVICE_INFO_REFERENCE_COUNT:
473-
return ReturnValue(uint32_t{Device->getRefCount()});
473+
return ReturnValue(uint32_t{Device->getRefCount().getCount()});
474474
case UR_DEVICE_INFO_SUPPORTED_PARTITIONS: {
475475
// SYCL spec says: if this SYCL device cannot be partitioned into at least
476476
// two sub devices then the returned vector must be empty.
@@ -1643,15 +1643,15 @@ ur_result_t urDeviceGetGlobalTimestamps(
16431643
ur_result_t urDeviceRetain(ur_device_handle_t Device) {
16441644
// The root-device ref-count remains unchanged (always 1).
16451645
if (Device->isSubDevice()) {
1646-
Device->incrementRefCount();
1646+
Device->getRefCount().increment();
16471647
}
16481648
return UR_RESULT_SUCCESS;
16491649
}
16501650

16511651
ur_result_t urDeviceRelease(ur_device_handle_t Device) {
16521652
// Root devices are destroyed during the piTearDown process.
16531653
if (Device->isSubDevice()) {
1654-
if (Device->decrementAndTest()) {
1654+
if (Device->getRefCount().decrementAndTest()) {
16551655
delete Device;
16561656
}
16571657
}

‎unified-runtime/source/adapters/level_zero/device.hpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
#include "adapters/level_zero/platform.hpp"
2222
#include "common.hpp"
23+
#include "common/ur_ref_count.hpp"
2324
#include <ur/ur.hpp>
2425
#include <ur_ddi.h>
2526
#include <ze_api.h>
@@ -238,6 +239,11 @@ struct ur_device_handle_t_ : ur_object {
238239

239240
// unique ephemeral identifer of the device in the adapter
240241
std::optional<DeviceId> Id;
242+
243+
URRefCount &getRefCount() noexcept { return RefCount; }
244+
245+
private:
246+
URRefCount RefCount;
241247
};
242248

243249
inline std::vector<ur_device_handle_t>

‎unified-runtime/source/adapters/level_zero/event.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -505,7 +505,7 @@ ur_result_t urEventGetInfo(
505505
return ReturnValue(Result);
506506
}
507507
case UR_EVENT_INFO_REFERENCE_COUNT: {
508-
return ReturnValue(Event->getRefCount());
508+
return ReturnValue(Event->getRefCount().getCount());
509509
}
510510
default:
511511
UR_LOG(ERR, "Unsupported ParamName in urEventGetInfo: ParamName={}(0x{})",
@@ -874,7 +874,7 @@ ur_result_t
874874
/// [in] handle of the event object
875875
urEventRetain(/** [in] handle of the event object */ ur_event_handle_t Event) {
876876
Event->RefCountExternal++;
877-
Event->incrementRefCount();
877+
Event->getRefCount().increment();
878878

879879
return UR_RESULT_SUCCESS;
880880
}
@@ -1088,7 +1088,7 @@ ur_event_handle_t_::~ur_event_handle_t_() {
10881088

10891089
ur_result_t urEventReleaseInternal(ur_event_handle_t Event,
10901090
bool *isEventDeleted) {
1091-
if (!Event->decrementAndTest())
1091+
if (!Event->getRefCount().decrementAndTest())
10921092
return UR_RESULT_SUCCESS;
10931093

10941094
if (Event->OriginAllocEvent) {
@@ -1429,7 +1429,7 @@ ur_result_t ur_event_handle_t_::reset() {
14291429
CommandType = UR_EXT_COMMAND_TYPE_USER;
14301430
WaitList = {};
14311431
RefCountExternal = 0;
1432-
resetRefCount();
1432+
RefCount.reset();
14331433
CommandList = std::nullopt;
14341434
completionBatch = std::nullopt;
14351435
OriginAllocEvent = nullptr;
@@ -1524,7 +1524,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList(
15241524
std::shared_lock<ur_shared_mutex> Lock(CurQueue->LastCommandEvent->Mutex);
15251525
this->ZeEventList[0] = CurQueue->LastCommandEvent->ZeEvent;
15261526
this->UrEventList[0] = CurQueue->LastCommandEvent;
1527-
this->UrEventList[0]->incrementRefCount();
1527+
this->UrEventList[0]->getRefCount().increment();
15281528
TmpListLength = 1;
15291529
} else if (EventListLength > 0) {
15301530
this->ZeEventList = new ze_event_handle_t[EventListLength];
@@ -1660,7 +1660,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList(
16601660
IsInternal, IsMultiDevice));
16611661
MultiDeviceZeEvent = MultiDeviceEvent->ZeEvent;
16621662
const auto &ZeCommandList = CommandList->first;
1663-
EventList[I]->incrementRefCount();
1663+
EventList[I]->getRefCount().increment();
16641664

16651665
// Append a Barrier to wait on the original event while signalling the
16661666
// new multi device event.
@@ -1676,11 +1676,11 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList(
16761676

16771677
this->ZeEventList[TmpListLength] = MultiDeviceZeEvent;
16781678
this->UrEventList[TmpListLength] = MultiDeviceEvent;
1679-
this->UrEventList[TmpListLength]->incrementRefCount();
1679+
this->UrEventList[TmpListLength]->getRefCount().increment();
16801680
} else {
16811681
this->ZeEventList[TmpListLength] = EventList[I]->ZeEvent;
16821682
this->UrEventList[TmpListLength] = EventList[I];
1683-
this->UrEventList[TmpListLength]->incrementRefCount();
1683+
this->UrEventList[TmpListLength]->getRefCount().increment();
16841684
}
16851685

16861686
if (QueueLock.has_value()) {

‎unified-runtime/source/adapters/level_zero/event.hpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <zes_api.h>
2626

2727
#include "common.hpp"
28+
#include "common/ur_ref_count.hpp"
2829
#include "queue.hpp"
2930
#include "ur_api.h"
3031

@@ -262,6 +263,11 @@ struct ur_event_handle_t_ : ur_object {
262263
// Used only for asynchronous allocations. This is the event originally used
263264
// on async free to indicate when the allocation can be used again.
264265
ur_event_handle_t OriginAllocEvent = nullptr;
266+
267+
URRefCount &getRefCount() noexcept { return RefCount; }
268+
269+
private:
270+
URRefCount RefCount;
265271
};
266272

267273
// Helper function to implement zeHostSynchronize.

‎unified-runtime/source/adapters/level_zero/kernel.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -787,7 +787,7 @@ ur_result_t urKernelGetInfo(
787787
}
788788
}
789789
case UR_KERNEL_INFO_REFERENCE_COUNT:
790-
return ReturnValue(uint32_t{Kernel->getRefCount()});
790+
return ReturnValue(uint32_t{Kernel->getRefCount().getCount()});
791791
case UR_KERNEL_INFO_ATTRIBUTES:
792792
try {
793793
uint32_t Size;
@@ -938,15 +938,15 @@ ur_result_t urKernelGetSubGroupInfo(
938938
ur_result_t urKernelRetain(
939939
/// [in] handle for the Kernel to retain
940940
ur_kernel_handle_t Kernel) {
941-
Kernel->incrementRefCount();
941+
Kernel->getRefCount().increment();
942942

943943
return UR_RESULT_SUCCESS;
944944
}
945945

946946
ur_result_t urKernelRelease(
947947
/// [in] handle for the Kernel to release
948948
ur_kernel_handle_t Kernel) {
949-
if (!Kernel->decrementAndTest())
949+
if (!Kernel->getRefCount().decrementAndTest())
950950
return UR_RESULT_SUCCESS;
951951

952952
auto KernelProgram = Kernel->Program;

0 commit comments

Comments
 (0)