Debian Codegraph @

shared/source/direct_submission/windows/wddm_direct_submission.inl

raw
1 /*
2 * Copyright (C) 2020-2025 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 */
7
8 #include "shared/source/device/device.h"
9 #include "shared/source/direct_submission/windows/wddm_direct_submission.h"
10 #include "shared/source/execution_environment/root_device_environment.h"
11 #include "shared/source/gmm_helper/gmm_helper.h"
12 #include "shared/source/os_interface/windows/os_context_win.h"
13 #include "shared/source/os_interface/windows/wddm/wddm.h"
14 #include "shared/source/os_interface/windows/wddm/wddm_interface.h"
15 #include "shared/source/os_interface/windows/wddm/wddm_residency_logger.h"
16 #include "shared/source/os_interface/windows/wddm_allocation.h"
17 #include "shared/source/os_interface/windows/wddm_memory_operations_handler.h"
18 #include "shared/source/utilities/arrayref.h"
19
20 namespace NEO {
21
22 // Initialize COMMAND_BUFFER_HEADER Type PatchList Streamer Perf Tag
23 DECLARE_COMMAND_BUFFER(CommandBufferHeader, UMD_OCL, FALSE, FALSE, PERFTAG_OCL);
24
25 template <typename GfxFamily, typename Dispatcher>
26 WddmDirectSubmission<GfxFamily, Dispatcher>::WddmDirectSubmission(const DirectSubmissionInputParams &inputParams)
27 : DirectSubmissionHw<GfxFamily, Dispatcher>(inputParams) {
28 osContextWin = reinterpret_cast<OsContextWin *>(&this->osContext);
29 wddm = osContextWin->getWddm();
30 commandBufferHeader = std::make_unique<COMMAND_BUFFER_HEADER_REC>();
31 *(commandBufferHeader.get()) = CommandBufferHeader;
32 if (osContextWin->getPreemptionMode() != PreemptionMode::Disabled) {
33 commandBufferHeader->NeedsMidBatchPreEmptionSupport = true;
34 }
35 perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "Starting Wddm ULLS. Placement ring buffer: %d semaphore %d\n",
36 debugManager.flags.DirectSubmissionBufferPlacement.get(),
37 debugManager.flags.DirectSubmissionSemaphorePlacement.get());
38
39 this->completionFenceAllocation = inputParams.completionFenceAllocation;
40 UNRECOVERABLE_IF(!this->completionFenceAllocation);
41 if (this->miMemFenceRequired) {
42 this->gpuVaForAdditionalSynchronizationWA = this->completionFenceAllocation->getGpuAddress() + 8u;
43 }
44 }
45
46 template <typename GfxFamily, typename Dispatcher>
47 WddmDirectSubmission<GfxFamily, Dispatcher>::~WddmDirectSubmission() {
48 perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "Stopping Wddm ULLS\n");
49 if (this->ringStart) {
50 this->stopRingBuffer(true);
51 }
52 this->deallocateResources();
53 wddm->getWddmInterface()->destroyMonitorFence(ringFence);
54 }
55
56 template <typename GfxFamily, typename Dispatcher>
57 inline void WddmDirectSubmission<GfxFamily, Dispatcher>::flushMonitorFence(bool notifyKmd) {
58 auto needStart = !this->ringStart;
59
60 size_t requiredMinimalSize = this->getSizeSemaphoreSection(false) +
61 Dispatcher::getSizeMonitorFence(this->rootDeviceEnvironment) +
62 this->getSizeNewResourceHandler() +
63 this->getSizeSwitchRingBufferSection() +
64 this->getSizeEnd(false);
65 this->switchRingBuffersNeeded(requiredMinimalSize, nullptr);
66
67 auto startVA = this->ringCommandStream.getCurrentGpuAddressPosition();
68
69 this->handleNewResourcesSubmission();
70
71 TagData currentTagData = {};
72 this->getTagAddressValue(currentTagData);
73 Dispatcher::dispatchMonitorFence(this->ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, this->rootDeviceEnvironment, this->partitionedMode, this->dcFlushRequired, notifyKmd);
74
75 this->dispatchSemaphoreSection(this->currentQueueWorkCount + 1);
76 this->submitCommandBufferToGpu(needStart, startVA, requiredMinimalSize, true, nullptr);
77 this->currentQueueWorkCount++;
78
79 this->updateTagValueImpl(this->currentRingBuffer);
80 }
81
82 template <typename GfxFamily, typename Dispatcher>
83 void WddmDirectSubmission<GfxFamily, Dispatcher>::ensureRingCompletion() {
84 WddmDirectSubmission<GfxFamily, Dispatcher>::handleCompletionFence(ringFence.lastSubmittedFence, ringFence);
85 }
86
87 template <typename GfxFamily, typename Dispatcher>
88 bool WddmDirectSubmission<GfxFamily, Dispatcher>::allocateOsResources() {
89 bool ret = wddm->getWddmInterface()->createFenceForDirectSubmission(ringFence, *this->osContextWin);
90 perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "ULLS resource allocation finished with: %d\n", ret);
91
92 this->ringBufferEndCompletionTagData.tagAddress = this->semaphoreGpuVa + offsetof(RingSemaphoreData, tagAllocation);
93 this->ringBufferEndCompletionTagData.tagValue = 0u;
94
95 DirectSubmissionHw<GfxFamily, Dispatcher>::allocateOsResources();
96
97 return ret;
98 }
99
100 template <typename GfxFamily, typename Dispatcher>
101 bool WddmDirectSubmission<GfxFamily, Dispatcher>::submit(uint64_t gpuAddress, size_t size, const ResidencyContainer *allocationsForResidency) {
102 perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "ULLS Submit to GPU\n");
103 COMMAND_BUFFER_HEADER *pHeader = reinterpret_cast<COMMAND_BUFFER_HEADER *>(commandBufferHeader.get());
104 pHeader->RequiresCoherency = false;
105
106 pHeader->UmdRequestedSliceState = 0;
107 pHeader->UmdRequestedEUCount = wddm->getRequestedEUCount();
108
109 pHeader->UmdRequestedSubsliceCount = 0;
110 pHeader->NeedsMidBatchPreEmptionSupport = true;
111
112 WddmSubmitArguments submitArgs = {};
113 submitArgs.contextHandle = osContextWin->getWddmContextHandle();
114 submitArgs.hwQueueHandle = osContextWin->getHwQueue().handle;
115 submitArgs.monitorFence = &ringFence;
116
117 return wddm->submit(gpuAddress, size, pHeader, submitArgs);
118 }
119
120 template <typename GfxFamily, typename Dispatcher>
121 bool WddmDirectSubmission<GfxFamily, Dispatcher>::handleResidency() {
122 wddm->waitOnPagingFenceFromCpu(this->lastSubmittedThrottle == QueueThrottle::LOW);
123 perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "ULLS residency wait exit\n");
124 return true;
125 }
126
127 template <typename GfxFamily, typename Dispatcher>
128 void WddmDirectSubmission<GfxFamily, Dispatcher>::handleStopRingBuffer() {
129 updateTagValueImplForSwitchRingBuffer(this->currentRingBuffer);
130 updateTagValueImpl(this->currentRingBuffer);
131 }
132
133 template <typename GfxFamily, typename Dispatcher>
134 void WddmDirectSubmission<GfxFamily, Dispatcher>::handleSwitchRingBuffers(ResidencyContainer *allocationsForResidency) {
135 updateTagValueImplForSwitchRingBuffer(this->previousRingBuffer);
136 }
137
138 template <typename GfxFamily, typename Dispatcher>
139 uint64_t WddmDirectSubmission<GfxFamily, Dispatcher>::updateTagValue(bool requireMonitorFence) {
140 if (this->detectGpuHang) {
141 bool osHang = wddm->isGpuHangDetected(*osContextWin);
142 bool ringHang = *ringFence.cpuAddress == Wddm::gpuHangIndication;
143
144 if (osHang || ringHang) {
145 wddm->getDeviceState();
146 return DirectSubmissionHw<GfxFamily, Dispatcher>::updateTagValueFail;
147 }
148 }
149
150 if (requireMonitorFence) {
151 return this->updateTagValueImpl(this->currentRingBuffer);
152 }
153 MonitoredFence &currentFence = osContextWin->getMonitoredFence();
154 return currentFence.currentFenceValue;
155 }
156
157 template <typename GfxFamily, typename Dispatcher>
158 bool WddmDirectSubmission<GfxFamily, Dispatcher>::dispatchMonitorFenceRequired(bool requireMonitorFence) {
159 return requireMonitorFence;
160 }
161
162 template <typename GfxFamily, typename Dispatcher>
163 uint64_t WddmDirectSubmission<GfxFamily, Dispatcher>::updateTagValueImpl(uint32_t completionBufferIndex) {
164 MonitoredFence &currentFence = osContextWin->getMonitoredFence();
165
166 currentFence.lastSubmittedFence = currentFence.currentFenceValue;
167 currentFence.currentFenceValue++;
168 this->ringBuffers[completionBufferIndex].completionFence = currentFence.lastSubmittedFence;
169
170 return currentFence.lastSubmittedFence;
171 }
172 template <typename GfxFamily, typename Dispatcher>
173 void WddmDirectSubmission<GfxFamily, Dispatcher>::updateTagValueImplForSwitchRingBuffer(uint32_t completionBufferIndex) {
174 this->ringBufferEndCompletionTagData.tagValue++;
175 this->ringBuffers[completionBufferIndex].completionFenceForSwitch = this->ringBufferEndCompletionTagData.tagValue;
176 }
177
178 template <typename GfxFamily, typename Dispatcher>
179 void WddmDirectSubmission<GfxFamily, Dispatcher>::handleCompletionFence(uint64_t completionValue, MonitoredFence &fence) {
180 wddm->waitFromCpu(completionValue, fence, false);
181 }
182
183 template <typename GfxFamily, typename Dispatcher>
184 void WddmDirectSubmission<GfxFamily, Dispatcher>::getTagAddressValue(TagData &tagData) {
185 MonitoredFence &currentFence = osContextWin->getMonitoredFence();
186 auto gmmHelper = wddm->getRootDeviceEnvironment().getGmmHelper();
187
188 tagData.tagAddress = gmmHelper->canonize(currentFence.gpuAddress);
189 tagData.tagValue = currentFence.currentFenceValue;
190 }
191
192 template <typename GfxFamily, typename Dispatcher>
193 void WddmDirectSubmission<GfxFamily, Dispatcher>::getTagAddressValueForRingSwitch(TagData &tagData) {
194 tagData.tagAddress = this->ringBufferEndCompletionTagData.tagAddress;
195 tagData.tagValue = this->ringBufferEndCompletionTagData.tagValue + 1;
196 }
197
198 template <typename GfxFamily, typename Dispatcher>
199 void WddmDirectSubmission<GfxFamily, Dispatcher>::dispatchStopRingBufferSection() {
200 TagData currentTagData = {};
201 getTagAddressValueForRingSwitch(currentTagData);
202 Dispatcher::dispatchMonitorFence(this->ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, this->rootDeviceEnvironment, this->partitionedMode, this->dcFlushRequired, this->notifyKmdDuringMonitorFence);
203 getTagAddressValue(currentTagData);
204 Dispatcher::dispatchMonitorFence(this->ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, this->rootDeviceEnvironment, this->partitionedMode, this->dcFlushRequired, this->notifyKmdDuringMonitorFence);
205 }
206
207 template <typename GfxFamily, typename Dispatcher>
208 size_t WddmDirectSubmission<GfxFamily, Dispatcher>::dispatchStopRingBufferSectionSize() {
209 return 2 * Dispatcher::getSizeMonitorFence(this->rootDeviceEnvironment);
210 }
211 template <typename GfxFamily, typename Dispatcher>
212 inline bool WddmDirectSubmission<GfxFamily, Dispatcher>::isCompleted(uint32_t ringBufferIndex) {
213 auto taskCount = this->ringBuffers[ringBufferIndex].completionFenceForSwitch;
214 auto pollAddress = this->tagAddress;
215 for (uint32_t i = 0; i < this->activeTiles; i++) {
216 if (*pollAddress < taskCount) {
217 return false;
218 }
219 pollAddress = ptrOffset(pollAddress, this->immWritePostSyncOffset);
220 }
221 return true;
222 }
223
224 template <typename GfxFamily, typename Dispatcher>
225 void WddmDirectSubmission<GfxFamily, Dispatcher>::updateMonitorFenceValueForResidencyList(ResidencyContainer *allocationsForResidency) {
226 if (allocationsForResidency == nullptr) {
227 return;
228 }
229 const auto currentFence = osContextWin->getMonitoredFence().currentFenceValue;
230 auto contextId = osContextWin->getContextId();
231 for (uint32_t i = 0; i < allocationsForResidency->size(); i++) {
232 WddmAllocation *allocation = static_cast<WddmAllocation *>((*allocationsForResidency)[i]);
233 // Update fence value not to early destroy / evict allocation
234 allocation->updateCompletionDataForAllocationAndFragments(currentFence, contextId);
235 }
236 }
237
238 /**
239 * @brief Unblocks pipeline by waiting for paging fence value and signals semaphore with latest value.
240 * Signals semaphore with latest fence value instead of passed one to unblock immediately subsequent semaphores.
241 *
242 * @param[in] pagingFenceValue paging fence submitted from user thread to wait for.
243 */
244 template <typename GfxFamily, typename Dispatcher>
245 inline void WddmDirectSubmission<GfxFamily, Dispatcher>::unblockPagingFenceSemaphore(uint64_t pagingFenceValue) {
246 this->wddm->waitOnPagingFenceFromCpu(pagingFenceValue, true);
247
248 typename DirectSubmissionHw<GfxFamily, Dispatcher>::SemaphoreFenceHelper fence(*this);
249 this->semaphoreData->pagingFenceCounter = static_cast<uint32_t>(*this->wddm->getPagingFenceAddress());
250 }
251
252 template <typename GfxFamily, typename Dispatcher>
253 inline void WddmDirectSubmission<GfxFamily, Dispatcher>::makeGlobalFenceAlwaysResident() {
254 if (this->globalFenceAllocation != nullptr) {
255 DirectSubmissionAllocations allocations;
256 allocations.push_back(this->globalFenceAllocation);
257 UNRECOVERABLE_IF(!this->makeResourcesResident(allocations));
258 }
259 }
260
261 } // namespace NEO