From a36316e7c5d2823b306574041e4e50b3f9f21775 Mon Sep 17 00:00:00 2001 From: Magnus Norddahl Date: Sun, 26 Jan 2025 13:59:40 +0100 Subject: [PATCH] Log vulkan device lost faults --- .../ZVulkan/include/zvulkan/vulkandevice.h | 24 ++++++++++ .../ZVulkan/include/zvulkan/vulkaninstance.h | 22 ++++----- .../ZVulkan/include/zvulkan/vulkanobjects.h | 22 ++++----- libraries/ZVulkan/src/vulkanbuilders.cpp | 31 ++++++------ libraries/ZVulkan/src/vulkandevice.cpp | 47 +++++++++++++++++-- .../vulkan/commands/vk_commandbuffer.cpp | 8 ++-- .../rendering/vulkan/vk_renderstate.cpp | 2 +- 7 files changed, 111 insertions(+), 45 deletions(-) diff --git a/libraries/ZVulkan/include/zvulkan/vulkandevice.h b/libraries/ZVulkan/include/zvulkan/vulkandevice.h index 87aa2cbe4..2bcb1a3c4 100644 --- a/libraries/ZVulkan/include/zvulkan/vulkandevice.h +++ b/libraries/ZVulkan/include/zvulkan/vulkandevice.h @@ -7,6 +7,7 @@ #include #include #include +#include class VulkanSwapChain; class VulkanSemaphore; @@ -15,6 +16,14 @@ class VulkanPhysicalDevice; class VulkanSurface; class VulkanCompatibleDevice; +class VulkanDeviceFaultInfo +{ +public: + std::string description; + // std::vector addressInfos; + std::vector vendorInfos; +}; + class VulkanDevice { public: @@ -43,6 +52,21 @@ public: void SetObjectName(const char* name, uint64_t handle, VkObjectType type); + VulkanDeviceFaultInfo GetDeviceFaultInfo(); + + inline void CheckVulkanError(VkResult result, const char* text) + { + if (result >= VK_SUCCESS) return; + if (result == VK_ERROR_DEVICE_LOST) + { + VulkanDeviceFaultInfo info = GetDeviceFaultInfo(); + VulkanPrintLog("fault", info.description); + for (const std::string& vendorInfo : info.vendorInfos) + VulkanPrintLog("fault", vendorInfo); + } + VulkanError((text + std::string(": ") + VkResultToString(result)).c_str()); + } + private: void CreateDevice(); void CreateAllocator(); diff --git a/libraries/ZVulkan/include/zvulkan/vulkaninstance.h b/libraries/ZVulkan/include/zvulkan/vulkaninstance.h index ae8ec33e3..d75efcd27 100644 --- a/libraries/ZVulkan/include/zvulkan/vulkaninstance.h +++ b/libraries/ZVulkan/include/zvulkan/vulkaninstance.h @@ -21,6 +21,11 @@ #include #include +std::string VkResultToString(VkResult result); + +void VulkanPrintLog(const char* typestr, const std::string& msg); +void VulkanError(const char* text); + class VulkanDeviceFeatures { public: @@ -75,6 +80,12 @@ public: bool DebugLayerActive = false; + static void CheckVulkanError(VkResult result, const char* text) + { + if (result >= VK_SUCCESS) return; + VulkanError((text + std::string(": ") + VkResultToString(result)).c_str()); + } + private: bool WantDebugLayer = false; VkDebugUtilsMessengerEXT debugMessenger = VK_NULL_HANDLE; @@ -90,14 +101,3 @@ private: static VKAPI_ATTR VkBool32 VKAPI_CALL DebugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity, VkDebugUtilsMessageTypeFlagsEXT messageType, const VkDebugUtilsMessengerCallbackDataEXT* pCallbackData, void* pUserData); static std::vector SplitString(const std::string& s, const std::string& seperator); }; - -std::string VkResultToString(VkResult result); - -void VulkanPrintLog(const char* typestr, const std::string& msg); -void VulkanError(const char* text); - -inline void CheckVulkanError(VkResult result, const char* text) -{ - if (result >= VK_SUCCESS) return; - VulkanError((text + std::string(": ") + VkResultToString(result)).c_str()); -} diff --git a/libraries/ZVulkan/include/zvulkan/vulkanobjects.h b/libraries/ZVulkan/include/zvulkan/vulkanobjects.h index c7842f670..0097d1abd 100644 --- a/libraries/ZVulkan/include/zvulkan/vulkanobjects.h +++ b/libraries/ZVulkan/include/zvulkan/vulkanobjects.h @@ -469,7 +469,7 @@ inline VulkanSemaphore::VulkanSemaphore(VulkanDevice *device) : device(device) VkSemaphoreCreateInfo semaphoreInfo = {}; semaphoreInfo.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO; VkResult result = vkCreateSemaphore(device->device, &semaphoreInfo, nullptr, &semaphore); - CheckVulkanError(result, "Could not create semaphore"); + device->CheckVulkanError(result, "Could not create semaphore"); } inline VulkanSemaphore::~VulkanSemaphore() @@ -484,7 +484,7 @@ inline VulkanFence::VulkanFence(VulkanDevice *device) : device(device) VkFenceCreateInfo fenceInfo = {}; fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; VkResult result = vkCreateFence(device->device, &fenceInfo, nullptr, &fence); - CheckVulkanError(result, "Could not create fence!"); + device->CheckVulkanError(result, "Could not create fence!"); } inline VulkanFence::~VulkanFence() @@ -525,7 +525,7 @@ inline VulkanCommandPool::VulkanCommandPool(VulkanDevice *device, int queueFamil poolInfo.flags = 0; VkResult result = vkCreateCommandPool(device->device, &poolInfo, nullptr, &pool); - CheckVulkanError(result, "Could not create command pool"); + device->CheckVulkanError(result, "Could not create command pool"); } inline VulkanCommandPool::~VulkanCommandPool() @@ -627,7 +627,7 @@ inline VulkanCommandBuffer::VulkanCommandBuffer(VulkanCommandPool *pool) : pool( allocInfo.commandBufferCount = 1; VkResult result = vkAllocateCommandBuffers(pool->device->device, &allocInfo, &buffer); - CheckVulkanError(result, "Could not create command buffer"); + pool->device->CheckVulkanError(result, "Could not create command buffer"); } inline VulkanCommandBuffer::~VulkanCommandBuffer() @@ -643,13 +643,13 @@ inline void VulkanCommandBuffer::begin() beginInfo.pInheritanceInfo = nullptr; VkResult result = vkBeginCommandBuffer(buffer, &beginInfo); - CheckVulkanError(result, "Could not begin recording command buffer"); + pool->device->CheckVulkanError(result, "Could not begin recording command buffer"); } inline void VulkanCommandBuffer::end() { VkResult result = vkEndCommandBuffer(buffer); - CheckVulkanError(result, "Could not end command buffer recording"); + pool->device->CheckVulkanError(result, "Could not end command buffer recording"); } inline void VulkanCommandBuffer::debugFullPipelineBarrier() @@ -1042,7 +1042,7 @@ inline std::unique_ptr VulkanDescriptorPool::allocate(Vulka if (allocType == AllocType::TryAllocate && result != VK_SUCCESS) return nullptr; else - CheckVulkanError(result, "Could not allocate descriptor sets"); + device->CheckVulkanError(result, "Could not allocate descriptor sets"); return std::make_unique(device, this, descriptorSet); } @@ -1062,7 +1062,7 @@ inline std::unique_ptr VulkanDescriptorPool::allocate(Vulka if (allocType == AllocType::TryAllocate && result != VK_SUCCESS) return nullptr; else - CheckVulkanError(result, "Could not allocate descriptor sets"); + device->CheckVulkanError(result, "Could not allocate descriptor sets"); return std::make_unique(device, this, descriptorSet); } @@ -1100,7 +1100,7 @@ inline VulkanQueryPool::~VulkanQueryPool() inline bool VulkanQueryPool::getResults(uint32_t firstQuery, uint32_t queryCount, size_t dataSize, void *data, VkDeviceSize stride, VkQueryResultFlags flags) { VkResult result = vkGetQueryPoolResults(device->device, pool, firstQuery, queryCount, dataSize, data, stride, flags); - CheckVulkanError(result, "vkGetQueryPoolResults failed"); + device->CheckVulkanError(result, "vkGetQueryPoolResults failed"); return result == VK_SUCCESS; } @@ -1210,14 +1210,14 @@ inline std::vector VulkanPipelineCache::GetCacheData() { size_t dataSize = 0; VkResult result = vkGetPipelineCacheData(device->device, cache, &dataSize, nullptr); - CheckVulkanError(result, "Could not get cache data size"); + device->CheckVulkanError(result, "Could not get cache data size"); std::vector buffer; buffer.resize(dataSize); result = vkGetPipelineCacheData(device->device, cache, &dataSize, buffer.data()); if (result == VK_INCOMPLETE) VulkanError("Could not get cache data (incomplete)"); - CheckVulkanError(result, "Could not get cache data"); + device->CheckVulkanError(result, "Could not get cache data"); buffer.resize(dataSize); return buffer; } diff --git a/libraries/ZVulkan/src/vulkanbuilders.cpp b/libraries/ZVulkan/src/vulkanbuilders.cpp index 07129d16f..3165b423c 100644 --- a/libraries/ZVulkan/src/vulkanbuilders.cpp +++ b/libraries/ZVulkan/src/vulkanbuilders.cpp @@ -358,7 +358,7 @@ std::unique_ptr ImageBuilder::Create(VulkanDevice* device, VkDevice VmaAllocation allocation; VkResult result = vmaCreateImage(device->allocator, &imageInfo, &allocInfo, &image, &allocation, nullptr); - CheckVulkanError(result, "Could not create vulkan image"); + device->CheckVulkanError(result, "Could not create vulkan image"); if (allocatedBytes != nullptr) { @@ -423,7 +423,7 @@ std::unique_ptr ImageViewBuilder::Create(VulkanDevice* device) { VkImageView view; VkResult result = vkCreateImageView(device->device, &viewInfo, nullptr, &view); - CheckVulkanError(result, "Could not create texture image view"); + device->CheckVulkanError(result, "Could not create texture image view"); auto obj = std::make_unique(device, view); if (debugName) @@ -510,7 +510,7 @@ std::unique_ptr SamplerBuilder::Create(VulkanDevice* device) { VkSampler sampler; VkResult result = vkCreateSampler(device->device, &samplerInfo, nullptr, &sampler); - CheckVulkanError(result, "Could not create texture sampler"); + device->CheckVulkanError(result, "Could not create texture sampler"); auto obj = std::make_unique(device, sampler); if (debugName) obj->SetDebugName(debugName); @@ -561,12 +561,12 @@ std::unique_ptr BufferBuilder::Create(VulkanDevice* device) if (minAlignment == 0) { VkResult result = vmaCreateBuffer(device->allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr); - CheckVulkanError(result, "Could not allocate memory for vulkan buffer"); + device->CheckVulkanError(result, "Could not allocate memory for vulkan buffer"); } else { VkResult result = vmaCreateBufferWithAlignment(device->allocator, &bufferInfo, &allocInfo, minAlignment, &buffer, &allocation, nullptr); - CheckVulkanError(result, "Could not allocate memory for vulkan buffer"); + device->CheckVulkanError(result, "Could not allocate memory for vulkan buffer"); } auto obj = std::make_unique(device, buffer, allocation, (size_t)bufferInfo.size); @@ -694,7 +694,7 @@ std::unique_ptr DescriptorSetLayoutBuilder::Create(Vu { VkDescriptorSetLayout layout; VkResult result = vkCreateDescriptorSetLayout(device->device, &layoutInfo, nullptr, &layout); - CheckVulkanError(result, "Could not create descriptor set layout"); + device->CheckVulkanError(result, "Could not create descriptor set layout"); auto obj = std::make_unique(device, layout); if (debugName) obj->SetDebugName(debugName); @@ -738,7 +738,7 @@ std::unique_ptr DescriptorPoolBuilder::Create(VulkanDevice { VkDescriptorPool descriptorPool; VkResult result = vkCreateDescriptorPool(device->device, &poolInfo, nullptr, &descriptorPool); - CheckVulkanError(result, "Could not create descriptor pool"); + device->CheckVulkanError(result, "Could not create descriptor pool"); auto obj = std::make_unique(device, descriptorPool); if (debugName) obj->SetDebugName(debugName); @@ -764,7 +764,7 @@ std::unique_ptr QueryPoolBuilder::Create(VulkanDevice* device) { VkQueryPool queryPool; VkResult result = vkCreateQueryPool(device->device, &poolInfo, nullptr, &queryPool); - CheckVulkanError(result, "Could not create query pool"); + device->CheckVulkanError(result, "Could not create query pool"); auto obj = std::make_unique(device, queryPool); if (debugName) obj->SetDebugName(debugName); @@ -814,7 +814,7 @@ std::unique_ptr FramebufferBuilder::Create(VulkanDevice* devi { VkFramebuffer framebuffer = 0; VkResult result = vkCreateFramebuffer(device->device, &framebufferInfo, nullptr, &framebuffer); - CheckVulkanError(result, "Could not create framebuffer"); + device->CheckVulkanError(result, "Could not create framebuffer"); auto obj = std::make_unique(device, framebuffer); if (debugName) obj->SetDebugName(debugName); @@ -1141,7 +1141,7 @@ std::unique_ptr GraphicsPipelineBuilder::Create(VulkanDevice* de VkPipeline pipeline = 0; VkResult result = vkCreateGraphicsPipelines(device->device, cache ? cache->cache : VK_NULL_HANDLE, 1, &pipelineInfo, nullptr, &pipeline); - CheckVulkanError(result, "Could not create graphics pipeline"); + device->CheckVulkanError(result, "Could not create graphics pipeline"); auto obj = std::make_unique(device, pipeline); if (debugName) obj->SetDebugName(debugName); @@ -1179,7 +1179,7 @@ std::unique_ptr PipelineLayoutBuilder::Create(VulkanDevice { VkPipelineLayout pipelineLayout; VkResult result = vkCreatePipelineLayout(device->device, &pipelineLayoutInfo, nullptr, &pipelineLayout); - CheckVulkanError(result, "Could not create pipeline layout"); + device->CheckVulkanError(result, "Could not create pipeline layout"); auto obj = std::make_unique(device, pipelineLayout); if (debugName) obj->SetDebugName(debugName); @@ -1227,7 +1227,7 @@ std::unique_ptr PipelineCacheBuilder::Create(VulkanDevice* VkPipelineCache pipelineCache; VkResult result = vkCreatePipelineCache(device->device, &pipelineCacheInfo, nullptr, &pipelineCache); - CheckVulkanError(result, "Could not create pipeline cache"); + device->CheckVulkanError(result, "Could not create pipeline cache"); auto obj = std::make_unique(device, pipelineCache); if (debugName) obj->SetDebugName(debugName); @@ -1331,7 +1331,7 @@ std::unique_ptr RenderPassBuilder::Create(VulkanDevice* device { VkRenderPass renderPass = 0; VkResult result = vkCreateRenderPass(device->device, &renderPassInfo, nullptr, &renderPass); - CheckVulkanError(result, "Could not create render pass"); + device->CheckVulkanError(result, "Could not create render pass"); auto obj = std::make_unique(device, renderPass); if (debugName) obj->SetDebugName(debugName); @@ -1474,7 +1474,7 @@ QueueSubmit& QueueSubmit::AddSignal(VulkanSemaphore* semaphore) void QueueSubmit::Execute(VulkanDevice* device, VkQueue queue, VulkanFence* fence) { VkResult result = vkQueueSubmit(device->GraphicsQueue, 1, &submitInfo, fence ? fence->fence : VK_NULL_HANDLE); - CheckVulkanError(result, "Could not submit command buffer"); + device->CheckVulkanError(result, "Could not submit command buffer"); } ///////////////////////////////////////////////////////////////////////////// @@ -1679,6 +1679,9 @@ VulkanDeviceBuilder::VulkanDeviceBuilder() OptionalExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME); OptionalExtension(VK_EXT_MEMORY_PRIORITY_EXTENSION_NAME); OptionalExtension(VK_AMD_DEVICE_COHERENT_MEMORY_EXTENSION_NAME); + + // Extensions desired for debugging + OptionalExtension(VK_EXT_DEVICE_FAULT_EXTENSION_NAME); } VulkanDeviceBuilder& VulkanDeviceBuilder::RequireExtension(const std::string& extensionName) diff --git a/libraries/ZVulkan/src/vulkandevice.cpp b/libraries/ZVulkan/src/vulkandevice.cpp index 62ea9e746..252f4e1f9 100644 --- a/libraries/ZVulkan/src/vulkandevice.cpp +++ b/libraries/ZVulkan/src/vulkandevice.cpp @@ -83,8 +83,7 @@ void VulkanDevice::CreateDevice() for (int index : neededFamilies) { - VkDeviceQueueCreateInfo queueCreateInfo = {}; - queueCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + VkDeviceQueueCreateInfo queueCreateInfo = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO }; queueCreateInfo.queueFamilyIndex = index; queueCreateInfo.queueCount = 1; queueCreateInfo.pQueuePriorities = &queuePriority; @@ -166,10 +165,50 @@ void VulkanDevice::SetObjectName(const char* name, uint64_t handle, VkObjectType { if (!Instance->DebugLayerActive) return; - VkDebugUtilsObjectNameInfoEXT info = {}; - info.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT; + VkDebugUtilsObjectNameInfoEXT info = { VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT }; info.objectHandle = handle; info.objectType = type; info.pObjectName = name; vkSetDebugUtilsObjectNameEXT(device, &info); } + +VulkanDeviceFaultInfo VulkanDevice::GetDeviceFaultInfo() +{ + if (!SupportsExtension(VK_EXT_DEVICE_FAULT_EXTENSION_NAME)) + return {}; + + VkDeviceFaultCountsEXT counts = {}; + VkResult result = vkGetDeviceFaultInfoEXT(device, &counts, nullptr); + if (result != VK_INCOMPLETE && result != VK_SUCCESS) + return {}; + + std::vector addressInfos(counts.addressInfoCount); + std::vector vendorInfos(counts.vendorInfoCount); + std::vector vendorBinaryData(counts.vendorBinarySize); + + VkDeviceFaultInfoEXT info = {}; + info.pAddressInfos = addressInfos.data(); + info.pVendorInfos = vendorInfos.data(); + info.pVendorBinaryData = vendorBinaryData.data(); + + result = vkGetDeviceFaultInfoEXT(device, &counts, &info); + if (result != VK_SUCCESS) + return {}; + + VulkanDeviceFaultInfo lostinfo; + lostinfo.description = info.description; + + /* + for (const VkDeviceFaultAddressInfoEXT& addressInfo : addressInfos) + { + // To do: does vk_mem_alloc have anything that helps us map an address to an allocation? + } + */ + + for (const VkDeviceFaultVendorInfoEXT& vendorInfo : vendorInfos) + { + lostinfo.vendorInfos.push_back(vendorInfo.description); + } + + return lostinfo; +} diff --git a/src/common/rendering/vulkan/commands/vk_commandbuffer.cpp b/src/common/rendering/vulkan/commands/vk_commandbuffer.cpp index 97ac7e60f..d7e84c81c 100644 --- a/src/common/rendering/vulkan/commands/vk_commandbuffer.cpp +++ b/src/common/rendering/vulkan/commands/vk_commandbuffer.cpp @@ -128,11 +128,11 @@ void VkCommandBufferManager::FlushCommands(VulkanCommandBuffer** commands, size_ do { result = vkWaitForFences(fb->GetDevice()->device, 1, &mSubmitFence[currentIndex]->fence, VK_TRUE, std::numeric_limits::max()); - CheckVulkanError(result, "Could not wait for commands"); + fb->GetDevice()->CheckVulkanError(result, "Could not wait for commands"); } while (result != VK_SUCCESS); result = vkResetFences(fb->GetDevice()->device, 1, &mSubmitFence[currentIndex]->fence); - CheckVulkanError(result, "Could not reset fence"); + fb->GetDevice()->CheckVulkanError(result, "Could not reset fence"); } QueueSubmit submit; @@ -220,12 +220,12 @@ void VkCommandBufferManager::WaitForCommands(bool finish, bool uploadOnly) if (numWaitFences > 0) { VkResult result = vkWaitForFences(fb->GetDevice()->device, numWaitFences, mSubmitWaitFences, VK_TRUE, std::numeric_limits::max()); - CheckVulkanError(result, "Could not wait for commands"); + fb->GetDevice()->CheckVulkanError(result, "Could not wait for commands"); if (result == VK_TIMEOUT) VulkanError("vkWaitForFences timed out! Broken display driver?"); result = vkResetFences(fb->GetDevice()->device, numWaitFences, mSubmitWaitFences); - CheckVulkanError(result, "Could not reset fences"); + fb->GetDevice()->CheckVulkanError(result, "Could not reset fences"); mNextSubmit = 0; } diff --git a/src/common/rendering/vulkan/vk_renderstate.cpp b/src/common/rendering/vulkan/vk_renderstate.cpp index 52a8e4e38..cca6eb7a4 100644 --- a/src/common/rendering/vulkan/vk_renderstate.cpp +++ b/src/common/rendering/vulkan/vk_renderstate.cpp @@ -1062,7 +1062,7 @@ void VkRenderState::GetQueryResults(int queryStart, int queryCount, TArray mQueryResultsBuffer.Resize(queryCount); VkResult result = vkGetQueryPoolResults(fb->GetDevice()->device, mRSBuffers->OcclusionQuery.QueryPool->pool, queryStart, queryCount, mQueryResultsBuffer.Size() * sizeof(uint32_t), mQueryResultsBuffer.Data(), sizeof(uint32_t), VK_QUERY_RESULT_WAIT_BIT); - CheckVulkanError(result, "Could not query occlusion query results"); + fb->GetDevice()->CheckVulkanError(result, "Could not query occlusion query results"); if (result == VK_NOT_READY) VulkanError("Occlusion query results returned VK_NOT_READY!");