/* * Copyright (C) 2020-2025 Intel Corporation * * SPDX-License-Identifier: MIT * */ #include "shared/source/device/device.h" #include "shared/source/direct_submission/windows/wddm_direct_submission.h" #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/os_interface/windows/os_context_win.h" #include "shared/source/os_interface/windows/wddm/wddm.h" #include "shared/source/os_interface/windows/wddm/wddm_interface.h" #include "shared/source/os_interface/windows/wddm/wddm_residency_logger.h" #include "shared/source/os_interface/windows/wddm_allocation.h" #include "shared/source/os_interface/windows/wddm_memory_operations_handler.h" #include "shared/source/utilities/arrayref.h" namespace NEO { // Initialize COMMAND_BUFFER_HEADER Type PatchList Streamer Perf Tag DECLARE_COMMAND_BUFFER(CommandBufferHeader, UMD_OCL, FALSE, FALSE, PERFTAG_OCL); template WddmDirectSubmission::WddmDirectSubmission(const DirectSubmissionInputParams &inputParams) : DirectSubmissionHw(inputParams) { osContextWin = reinterpret_cast(&this->osContext); wddm = osContextWin->getWddm(); commandBufferHeader = std::make_unique(); *(commandBufferHeader.get()) = CommandBufferHeader; if (osContextWin->getPreemptionMode() != PreemptionMode::Disabled) { commandBufferHeader->NeedsMidBatchPreEmptionSupport = true; } perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "Starting Wddm ULLS. Placement ring buffer: %d semaphore %d\n", debugManager.flags.DirectSubmissionBufferPlacement.get(), debugManager.flags.DirectSubmissionSemaphorePlacement.get()); this->completionFenceAllocation = inputParams.completionFenceAllocation; UNRECOVERABLE_IF(!this->completionFenceAllocation); if (this->miMemFenceRequired) { this->gpuVaForAdditionalSynchronizationWA = this->completionFenceAllocation->getGpuAddress() + 8u; } } template WddmDirectSubmission::~WddmDirectSubmission() { perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "Stopping Wddm ULLS\n"); if (this->ringStart) { this->stopRingBuffer(true); } this->deallocateResources(); wddm->getWddmInterface()->destroyMonitorFence(ringFence); } template inline void WddmDirectSubmission::flushMonitorFence(bool notifyKmd) { auto needStart = !this->ringStart; size_t requiredMinimalSize = this->getSizeSemaphoreSection(false) + Dispatcher::getSizeMonitorFence(this->rootDeviceEnvironment) + this->getSizeNewResourceHandler() + this->getSizeSwitchRingBufferSection() + this->getSizeEnd(false); this->switchRingBuffersNeeded(requiredMinimalSize, nullptr); auto startVA = this->ringCommandStream.getCurrentGpuAddressPosition(); this->handleNewResourcesSubmission(); TagData currentTagData = {}; this->getTagAddressValue(currentTagData); Dispatcher::dispatchMonitorFence(this->ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, this->rootDeviceEnvironment, this->partitionedMode, this->dcFlushRequired, notifyKmd); this->dispatchSemaphoreSection(this->currentQueueWorkCount + 1); this->submitCommandBufferToGpu(needStart, startVA, requiredMinimalSize, true, nullptr); this->currentQueueWorkCount++; this->updateTagValueImpl(this->currentRingBuffer); } template void WddmDirectSubmission::ensureRingCompletion() { WddmDirectSubmission::handleCompletionFence(ringFence.lastSubmittedFence, ringFence); } template bool WddmDirectSubmission::allocateOsResources() { bool ret = wddm->getWddmInterface()->createFenceForDirectSubmission(ringFence, *this->osContextWin); perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "ULLS resource allocation finished with: %d\n", ret); this->ringBufferEndCompletionTagData.tagAddress = this->semaphoreGpuVa + offsetof(RingSemaphoreData, tagAllocation); this->ringBufferEndCompletionTagData.tagValue = 0u; DirectSubmissionHw::allocateOsResources(); return ret; } template bool WddmDirectSubmission::submit(uint64_t gpuAddress, size_t size, const ResidencyContainer *allocationsForResidency) { perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "ULLS Submit to GPU\n"); COMMAND_BUFFER_HEADER *pHeader = reinterpret_cast(commandBufferHeader.get()); pHeader->RequiresCoherency = false; pHeader->UmdRequestedSliceState = 0; pHeader->UmdRequestedEUCount = wddm->getRequestedEUCount(); pHeader->UmdRequestedSubsliceCount = 0; pHeader->NeedsMidBatchPreEmptionSupport = true; WddmSubmitArguments submitArgs = {}; submitArgs.contextHandle = osContextWin->getWddmContextHandle(); submitArgs.hwQueueHandle = osContextWin->getHwQueue().handle; submitArgs.monitorFence = &ringFence; return wddm->submit(gpuAddress, size, pHeader, submitArgs); } template bool WddmDirectSubmission::handleResidency() { wddm->waitOnPagingFenceFromCpu(this->lastSubmittedThrottle == QueueThrottle::LOW); perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "ULLS residency wait exit\n"); return true; } template void WddmDirectSubmission::handleStopRingBuffer() { updateTagValueImplForSwitchRingBuffer(this->currentRingBuffer); updateTagValueImpl(this->currentRingBuffer); } template void WddmDirectSubmission::handleSwitchRingBuffers(ResidencyContainer *allocationsForResidency) { updateTagValueImplForSwitchRingBuffer(this->previousRingBuffer); } template uint64_t WddmDirectSubmission::updateTagValue(bool requireMonitorFence) { if (this->detectGpuHang) { bool osHang = wddm->isGpuHangDetected(*osContextWin); bool ringHang = *ringFence.cpuAddress == Wddm::gpuHangIndication; if (osHang || ringHang) { wddm->getDeviceState(); return DirectSubmissionHw::updateTagValueFail; } } if (requireMonitorFence) { return this->updateTagValueImpl(this->currentRingBuffer); } MonitoredFence ¤tFence = osContextWin->getMonitoredFence(); return currentFence.currentFenceValue; } template bool WddmDirectSubmission::dispatchMonitorFenceRequired(bool requireMonitorFence) { return requireMonitorFence; } template uint64_t WddmDirectSubmission::updateTagValueImpl(uint32_t completionBufferIndex) { MonitoredFence ¤tFence = osContextWin->getMonitoredFence(); currentFence.lastSubmittedFence = currentFence.currentFenceValue; currentFence.currentFenceValue++; this->ringBuffers[completionBufferIndex].completionFence = currentFence.lastSubmittedFence; return currentFence.lastSubmittedFence; } template void WddmDirectSubmission::updateTagValueImplForSwitchRingBuffer(uint32_t completionBufferIndex) { this->ringBufferEndCompletionTagData.tagValue++; this->ringBuffers[completionBufferIndex].completionFenceForSwitch = this->ringBufferEndCompletionTagData.tagValue; } template void WddmDirectSubmission::handleCompletionFence(uint64_t completionValue, MonitoredFence &fence) { wddm->waitFromCpu(completionValue, fence, false); } template void WddmDirectSubmission::getTagAddressValue(TagData &tagData) { MonitoredFence ¤tFence = osContextWin->getMonitoredFence(); auto gmmHelper = wddm->getRootDeviceEnvironment().getGmmHelper(); tagData.tagAddress = gmmHelper->canonize(currentFence.gpuAddress); tagData.tagValue = currentFence.currentFenceValue; } template void WddmDirectSubmission::getTagAddressValueForRingSwitch(TagData &tagData) { tagData.tagAddress = this->ringBufferEndCompletionTagData.tagAddress; tagData.tagValue = this->ringBufferEndCompletionTagData.tagValue + 1; } template void WddmDirectSubmission::dispatchStopRingBufferSection() { TagData currentTagData = {}; getTagAddressValueForRingSwitch(currentTagData); Dispatcher::dispatchMonitorFence(this->ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, this->rootDeviceEnvironment, this->partitionedMode, this->dcFlushRequired, this->notifyKmdDuringMonitorFence); getTagAddressValue(currentTagData); Dispatcher::dispatchMonitorFence(this->ringCommandStream, currentTagData.tagAddress, currentTagData.tagValue, this->rootDeviceEnvironment, this->partitionedMode, this->dcFlushRequired, this->notifyKmdDuringMonitorFence); } template size_t WddmDirectSubmission::dispatchStopRingBufferSectionSize() { return 2 * Dispatcher::getSizeMonitorFence(this->rootDeviceEnvironment); } template inline bool WddmDirectSubmission::isCompleted(uint32_t ringBufferIndex) { auto taskCount = this->ringBuffers[ringBufferIndex].completionFenceForSwitch; auto pollAddress = this->tagAddress; for (uint32_t i = 0; i < this->activeTiles; i++) { if (*pollAddress < taskCount) { return false; } pollAddress = ptrOffset(pollAddress, this->immWritePostSyncOffset); } return true; } template void WddmDirectSubmission::updateMonitorFenceValueForResidencyList(ResidencyContainer *allocationsForResidency) { if (allocationsForResidency == nullptr) { return; } const auto currentFence = osContextWin->getMonitoredFence().currentFenceValue; auto contextId = osContextWin->getContextId(); for (uint32_t i = 0; i < allocationsForResidency->size(); i++) { WddmAllocation *allocation = static_cast((*allocationsForResidency)[i]); // Update fence value not to early destroy / evict allocation allocation->updateCompletionDataForAllocationAndFragments(currentFence, contextId); } } /** * @brief Unblocks pipeline by waiting for paging fence value and signals semaphore with latest value. * Signals semaphore with latest fence value instead of passed one to unblock immediately subsequent semaphores. * * @param[in] pagingFenceValue paging fence submitted from user thread to wait for. */ template inline void WddmDirectSubmission::unblockPagingFenceSemaphore(uint64_t pagingFenceValue) { this->wddm->waitOnPagingFenceFromCpu(pagingFenceValue, true); typename DirectSubmissionHw::SemaphoreFenceHelper fence(*this); this->semaphoreData->pagingFenceCounter = static_cast(*this->wddm->getPagingFenceAddress()); } template inline void WddmDirectSubmission::makeGlobalFenceAlwaysResident() { if (this->globalFenceAllocation != nullptr) { DirectSubmissionAllocations allocations; allocations.push_back(this->globalFenceAllocation); UNRECOVERABLE_IF(!this->makeResourcesResident(allocations)); } } } // namespace NEO