| 1 |
/* |
| 2 |
* Copyright (C) 2020-2024 Intel Corporation |
| 3 |
* |
| 4 |
* SPDX-License-Identifier: MIT |
| 5 |
* |
| 6 |
*/ |
| 7 |
|
| 8 |
#pragma once |
| 9 |
|
| 10 |
#include "level_zero/experimental/source/tracing/tracing.h" |
| 11 |
#include "level_zero/experimental/source/tracing/tracing_barrier_imp.h" |
| 12 |
#include "level_zero/experimental/source/tracing/tracing_cmdlist_imp.h" |
| 13 |
#include "level_zero/experimental/source/tracing/tracing_cmdqueue_imp.h" |
| 14 |
#include "level_zero/experimental/source/tracing/tracing_copy_imp.h" |
| 15 |
#include "level_zero/experimental/source/tracing/tracing_device_imp.h" |
| 16 |
#include "level_zero/experimental/source/tracing/tracing_driver_imp.h" |
| 17 |
#include "level_zero/experimental/source/tracing/tracing_event_imp.h" |
| 18 |
#include "level_zero/experimental/source/tracing/tracing_fence_imp.h" |
| 19 |
#include "level_zero/experimental/source/tracing/tracing_global_imp.h" |
| 20 |
#include "level_zero/experimental/source/tracing/tracing_image_imp.h" |
| 21 |
#include "level_zero/experimental/source/tracing/tracing_memory_imp.h" |
| 22 |
#include "level_zero/experimental/source/tracing/tracing_module_imp.h" |
| 23 |
#include "level_zero/experimental/source/tracing/tracing_residency_imp.h" |
| 24 |
#include "level_zero/experimental/source/tracing/tracing_sampler_imp.h" |
| 25 |
#include <level_zero/ze_api.h> |
| 26 |
|
| 27 |
#include "ze_ddi_tables.h" |
| 28 |
|
| 29 |
#include <atomic> |
| 30 |
#include <chrono> |
| 31 |
#include <list> |
| 32 |
#include <mutex> |
| 33 |
#include <vector> |
| 34 |
|
| 35 |
extern ze_gpu_driver_dditable_t driverDdiTable; |
| 36 |
|
| 37 |
namespace L0 { |
| 38 |
|
| 39 |
extern thread_local ze_bool_t tracingInProgress; |
| 40 |
extern struct APITracerContextImp *pGlobalAPITracerContextImp; |
| 41 |
|
| 42 |
typedef struct TracerArrayEntry { |
| 43 |
zet_core_callbacks_t corePrologues; |
| 44 |
zet_core_callbacks_t coreEpilogues; |
| 45 |
zet_device_handle_t hDevice; |
| 46 |
void *pUserData; |
| 47 |
} tracer_array_entry_t; |
| 48 |
|
| 49 |
typedef struct TracerArray { |
| 50 |
size_t tracerArrayCount; |
| 51 |
tracer_array_entry_t *tracerArrayEntries; |
| 52 |
} tracer_array_t; |
| 53 |
|
| 54 |
enum TracingState { |
| 55 |
disabledState, // tracing has never been enabled |
| 56 |
enabledState, // tracing is enabled. |
| 57 |
disabledWaitingState, // tracing has been disabled, but not waited for |
| 58 |
}; |
| 59 |
|
| 60 |
struct APITracerImp : APITracer { |
| 61 |
ze_result_t destroyTracer(zet_tracer_exp_handle_t phTracer) override; |
| 62 |
ze_result_t setPrologues(zet_core_callbacks_t *pCoreCbs) override; |
| 63 |
ze_result_t setEpilogues(zet_core_callbacks_t *pCoreCbs) override; |
| 64 |
ze_result_t enableTracer(ze_bool_t enable) override; |
| 65 |
|
| 66 |
tracer_array_entry_t tracerFunctions{}; |
| 67 |
TracingState tracingState = disabledState; |
| 68 |
|
| 69 |
private: |
| 70 |
}; |
| 71 |
|
| 72 |
class ThreadPrivateTracerData { |
| 73 |
public: |
| 74 |
void clearThreadTracerDataOnList(void) { onList = false; } |
| 75 |
void removeThreadTracerDataFromList(void); |
| 76 |
bool testAndSetThreadTracerDataInitializedAndOnList(void); |
| 77 |
bool onList; |
| 78 |
bool isInitialized; |
| 79 |
ThreadPrivateTracerData(); |
| 80 |
~ThreadPrivateTracerData(); |
| 81 |
|
| 82 |
std::atomic<tracer_array_t *> tracerArrayPointer; |
| 83 |
|
| 84 |
private: |
| 85 |
ThreadPrivateTracerData(const ThreadPrivateTracerData &); |
| 86 |
ThreadPrivateTracerData &operator=(const ThreadPrivateTracerData &); |
| 87 |
}; |
| 88 |
|
| 89 |
struct APITracerContextImp { |
| 90 |
public: |
| 91 |
APITracerContextImp() { |
| 92 |
activeTracerArray.store(&emptyTracerArray, std::memory_order_relaxed); |
| 93 |
}; |
| 94 |
|
| 95 |
~APITracerContextImp(); |
| 96 |
|
| 97 |
static void apiTracingEnable(ze_init_flag_t flag); |
| 98 |
|
| 99 |
void *getActiveTracersList(); |
| 100 |
void releaseActivetracersList(); |
| 101 |
|
| 102 |
ze_result_t enableTracingImp(struct APITracerImp *newTracer, ze_bool_t enable); |
| 103 |
ze_result_t finalizeDisableImpTracingWait(struct APITracerImp *oldTracer); |
| 104 |
|
| 105 |
bool isTracingEnabled(); |
| 106 |
|
| 107 |
void addThreadTracerDataToList(ThreadPrivateTracerData *threadDataP); |
| 108 |
void removeThreadTracerDataFromList(ThreadPrivateTracerData *threadDataP); |
| 109 |
|
| 110 |
private: |
| 111 |
std::mutex traceTableMutex; |
| 112 |
tracer_array_t emptyTracerArray = {0, NULL}; |
| 113 |
std::atomic<tracer_array_t *> activeTracerArray; |
| 114 |
|
| 115 |
// |
| 116 |
// a list of tracer arrays that were once active, but |
| 117 |
// have been replaced by a new active array. These |
| 118 |
// once-active tracer arrays may continue for some time |
| 119 |
// to have references to them among the per-thread |
| 120 |
// tracer array pointers. |
| 121 |
// |
| 122 |
std::list<tracer_array_t *> retiringTracerArrayList; |
| 123 |
|
| 124 |
std::list<struct APITracerImp *> enabledTracerImpList; |
| 125 |
|
| 126 |
ze_bool_t testForTracerArrayReferences(tracer_array_t *tracerArray); |
| 127 |
size_t testAndFreeRetiredTracers(); |
| 128 |
size_t updateTracerArrays(); |
| 129 |
|
| 130 |
std::list<ThreadPrivateTracerData *> threadTracerDataList; |
| 131 |
std::mutex threadTracerDataListMutex; |
| 132 |
}; |
| 133 |
|
| 134 |
extern thread_local ThreadPrivateTracerData myThreadPrivateTracerData; |
| 135 |
|
| 136 |
template <class T> |
| 137 |
class APITracerCallbackStateImp { |
| 138 |
public: |
| 139 |
T currentApiCallback; |
| 140 |
void *pUserData; |
| 141 |
}; |
| 142 |
|
| 143 |
template <class T> |
| 144 |
class APITracerCallbackDataImp { |
| 145 |
public: |
| 146 |
T apiOrdinal = {}; |
| 147 |
std::vector<L0::APITracerCallbackStateImp<T>> prologCallbacks; |
| 148 |
std::vector<L0::APITracerCallbackStateImp<T>> epilogCallbacks; |
| 149 |
}; |
| 150 |
|
| 151 |
#define ZE_HANDLE_TRACER_RECURSION(ze_api_ptr, ...) \ |
| 152 |
do { \ |
| 153 |
if (L0::tracingInProgress) { \ |
| 154 |
return ze_api_ptr(__VA_ARGS__); \ |
| 155 |
} \ |
| 156 |
L0::tracingInProgress = 1; \ |
| 157 |
} while (0) |
| 158 |
|
| 159 |
#define ZE_GEN_TRACER_ARRAY_ENTRY(callbackPtr, tracerArray, tracerArrayIndex, callbackType, callbackCategory, callbackFunction) \ |
| 160 |
do { \ |
| 161 |
callbackPtr = tracerArray->tracerArrayEntries[tracerArrayIndex].callbackType.callbackCategory.callbackFunction; \ |
| 162 |
} while (0) |
| 163 |
|
| 164 |
#define ZE_GEN_PER_API_CALLBACK_STATE(perApiCallbackData, tracerType, callbackCategory, callbackFunctionType) \ |
| 165 |
L0::tracer_array_t *currentTracerArray; \ |
| 166 |
currentTracerArray = (L0::tracer_array_t *)L0::pGlobalAPITracerContextImp->getActiveTracersList(); \ |
| 167 |
if (currentTracerArray) { \ |
| 168 |
for (size_t i = 0; i < currentTracerArray->tracerArrayCount; i++) { \ |
| 169 |
tracerType prologueCallbackPtr; \ |
| 170 |
tracerType epilogue_callback_ptr; \ |
| 171 |
ZE_GEN_TRACER_ARRAY_ENTRY(prologueCallbackPtr, currentTracerArray, i, corePrologues, callbackCategory, callbackFunctionType); \ |
| 172 |
ZE_GEN_TRACER_ARRAY_ENTRY(epilogue_callback_ptr, currentTracerArray, i, coreEpilogues, callbackCategory, callbackFunctionType); \ |
| 173 |
\ |
| 174 |
L0::APITracerCallbackStateImp<tracerType> prologCallback; \ |
| 175 |
prologCallback.currentApiCallback = prologueCallbackPtr; \ |
| 176 |
prologCallback.pUserData = currentTracerArray->tracerArrayEntries[i].pUserData; \ |
| 177 |
perApiCallbackData.prologCallbacks.push_back(prologCallback); \ |
| 178 |
\ |
| 179 |
L0::APITracerCallbackStateImp<tracerType> epilogCallback; \ |
| 180 |
epilogCallback.currentApiCallback = epilogue_callback_ptr; \ |
| 181 |
epilogCallback.pUserData = currentTracerArray->tracerArrayEntries[i].pUserData; \ |
| 182 |
perApiCallbackData.epilogCallbacks.push_back(epilogCallback); \ |
| 183 |
} \ |
| 184 |
} |
| 185 |
|
| 186 |
template <typename TFunctionPointer, typename TParams, typename TTracer, typename TTracerPrologCallbacks, typename TTracerEpilogCallbacks, typename... Args> |
| 187 |
ze_result_t apiTracerWrapperImp(TFunctionPointer zeApiPtr, |
| 188 |
TParams paramsStruct, |
| 189 |
TTracer apiOrdinal, |
| 190 |
TTracerPrologCallbacks prologCallbacks, |
| 191 |
TTracerEpilogCallbacks epilogCallbacks, |
| 192 |
Args &&...args) { |
| 193 |
ze_result_t ret = ZE_RESULT_SUCCESS; |
| 194 |
std::vector<APITracerCallbackStateImp<TTracer>> *callbacksPrologs = &prologCallbacks; |
| 195 |
|
| 196 |
std::vector<void *> ppTracerInstanceUserData; |
| 197 |
ppTracerInstanceUserData.resize(callbacksPrologs->size()); |
| 198 |
|
| 199 |
for (size_t i = 0; i < callbacksPrologs->size(); i++) { |
| 200 |
if (callbacksPrologs->at(i).currentApiCallback != nullptr) |
| 201 |
callbacksPrologs->at(i).currentApiCallback(paramsStruct, ret, callbacksPrologs->at(i).pUserData, &ppTracerInstanceUserData[i]); |
| 202 |
} |
| 203 |
ret = zeApiPtr(args...); |
| 204 |
std::vector<APITracerCallbackStateImp<TTracer>> *callbacksEpilogs = &epilogCallbacks; |
| 205 |
for (size_t i = 0; i < callbacksEpilogs->size(); i++) { |
| 206 |
if (callbacksEpilogs->at(i).currentApiCallback != nullptr) |
| 207 |
callbacksEpilogs->at(i).currentApiCallback(paramsStruct, ret, callbacksEpilogs->at(i).pUserData, &ppTracerInstanceUserData[i]); |
| 208 |
} |
| 209 |
L0::tracingInProgress = 0; |
| 210 |
L0::pGlobalAPITracerContextImp->releaseActivetracersList(); |
| 211 |
return ret; |
| 212 |
} |
| 213 |
|
| 214 |
} // namespace L0 |