diff --git a/src/common/rendering/hwrenderer/data/hw_renderstate.h b/src/common/rendering/hwrenderer/data/hw_renderstate.h index 8c753f044..7ac3662a2 100644 --- a/src/common/rendering/hwrenderer/data/hw_renderstate.h +++ b/src/common/rendering/hwrenderer/data/hw_renderstate.h @@ -731,6 +731,9 @@ public: // API-dependent render interface + // Worker threads + virtual void FlushCommands() { } + // Vertices virtual std::pair AllocVertices(unsigned int count) = 0; virtual void SetShadowData(const TArray& vertices, const TArray& indexes) = 0; diff --git a/src/common/rendering/vulkan/commands/vk_commandbuffer.cpp b/src/common/rendering/vulkan/commands/vk_commandbuffer.cpp index 4b1c7b5e5..bcb3aed83 100644 --- a/src/common/rendering/vulkan/commands/vk_commandbuffer.cpp +++ b/src/common/rendering/vulkan/commands/vk_commandbuffer.cpp @@ -71,24 +71,43 @@ VulkanCommandBuffer* VkCommandBufferManager::GetTransferCommands() { if (!mTransferCommands) { + std::unique_lock lock(mMutex); mTransferCommands = mCommandPool->createBuffer(); - mTransferCommands->SetDebugName("VulkanRenderDevice.mTransferCommands"); + mTransferCommands->SetDebugName("TransferCommands"); mTransferCommands->begin(); } return mTransferCommands.get(); } -VulkanCommandBuffer* VkCommandBufferManager::GetDrawCommands(int threadIndex) +VulkanCommandBuffer* VkCommandBufferManager::GetDrawCommands() { if (!mDrawCommands) { + std::unique_lock lock(mMutex); mDrawCommands = mCommandPool->createBuffer(); - mDrawCommands->SetDebugName("VulkanRenderDevice.mDrawCommands"); + mDrawCommands->SetDebugName("DrawCommands"); mDrawCommands->begin(); } return mDrawCommands.get(); } +std::unique_ptr VkCommandBufferManager::BeginThreadCommands() +{ + std::unique_lock lock(mMutex); + auto commands = mCommandPool->createBuffer(); + commands->SetDebugName("ThreadCommands"); + lock.unlock(); + commands->begin(); + return commands; +} + +void VkCommandBufferManager::EndThreadCommands(std::unique_ptr commands) +{ + commands->end(); + std::unique_lock lock(mMutex); + mThreadCommands.push_back(std::move(commands)); +} + void VkCommandBufferManager::BeginFrame() { if (mNextTimestampQuery > 0) @@ -134,28 +153,37 @@ void VkCommandBufferManager::FlushCommands(bool finish, bool lastsubmit, bool up if (!uploadOnly) fb->GetRenderState()->EndRenderPass(); - if ((!uploadOnly && mDrawCommands) || mTransferCommands) + std::unique_lock lock(mMutex); + + if (mTransferCommands) { - VulkanCommandBuffer* commands[2]; - size_t count = 0; + mTransferCommands->end(); + mFlushCommands.push_back(mTransferCommands.get()); + TransferDeleteList->Add(std::move(mTransferCommands)); + } - if (mTransferCommands) - { - mTransferCommands->end(); - commands[count++] = mTransferCommands.get(); - TransferDeleteList->Add(std::move(mTransferCommands)); - } - - if (!uploadOnly && mDrawCommands) + if (!uploadOnly) + { + if (mDrawCommands) { mDrawCommands->end(); - commands[count++] = mDrawCommands.get(); + mFlushCommands.push_back(mDrawCommands.get()); DrawDeleteList->Add(std::move(mDrawCommands)); } - FlushCommands(commands, count, finish, lastsubmit); + for (auto& buffer : mThreadCommands) + { + mFlushCommands.push_back(buffer.get()); + DrawDeleteList->Add(std::move(buffer)); + } + mThreadCommands.clear(); + } - current_rendered_commandbuffers += (int)count; + if (!mFlushCommands.empty()) + { + FlushCommands(mFlushCommands.data(), mFlushCommands.size(), finish, lastsubmit); + current_rendered_commandbuffers += (int)mFlushCommands.size(); + mFlushCommands.clear(); } } diff --git a/src/common/rendering/vulkan/commands/vk_commandbuffer.h b/src/common/rendering/vulkan/commands/vk_commandbuffer.h index 3386b1e45..561b5eb8c 100644 --- a/src/common/rendering/vulkan/commands/vk_commandbuffer.h +++ b/src/common/rendering/vulkan/commands/vk_commandbuffer.h @@ -15,7 +15,10 @@ public: void BeginFrame(); VulkanCommandBuffer* GetTransferCommands(); - VulkanCommandBuffer* GetDrawCommands(int threadIndex = 0); + VulkanCommandBuffer* GetDrawCommands(); + + std::unique_ptr BeginThreadCommands(); + void EndThreadCommands(std::unique_ptr commands); void FlushCommands(bool finish, bool lastsubmit = false, bool uploadOnly = false); @@ -67,6 +70,7 @@ private: std::unique_ptr mTransferCommands; std::unique_ptr mDrawCommands; + std::vector> mThreadCommands; enum { maxConcurrentSubmitCount = 8 }; std::unique_ptr mSubmitSemaphore[maxConcurrentSubmitCount]; @@ -86,4 +90,6 @@ private: int mNextTimestampQuery = 0; std::vector mGroupStack; std::vector timeElapsedQueries; + std::mutex mMutex; + std::vector mFlushCommands; }; diff --git a/src/common/rendering/vulkan/vk_renderstate.cpp b/src/common/rendering/vulkan/vk_renderstate.cpp index d3e716470..9acfb5718 100644 --- a/src/common/rendering/vulkan/vk_renderstate.cpp +++ b/src/common/rendering/vulkan/vk_renderstate.cpp @@ -188,7 +188,7 @@ void VkRenderState::Apply(int dt) drawcalls.Clock(); mApplyCount++; - if (mApplyCount >= vk_submit_size) + if (threadIndex == 0 && mApplyCount >= vk_submit_size) { fb->GetCommands()->FlushCommands(false); mApplyCount = 0; @@ -310,7 +310,16 @@ void VkRenderState::ApplyRenderPass(int dt) if (!inRenderPass) { - mCommandBuffer = fb->GetCommands()->GetDrawCommands(threadIndex); + if (threadIndex == 0) + { + mCommandBuffer = fb->GetCommands()->GetDrawCommands(); + } + else + { + mThreadCommandBuffer = fb->GetCommands()->BeginThreadCommands(); + mCommandBuffer = mThreadCommandBuffer.get(); + } + mScissorChanged = true; mViewportChanged = true; mStencilRefChanged = true; @@ -711,13 +720,18 @@ void VkRenderState::EndRenderPass() { mCommandBuffer->endRenderPass(); mCommandBuffer = nullptr; - mPipelineKey = {}; - - // Force rebind of all buffers - mLastViewpointOffset = 0xffffffff; - mLastVertexOffsets[0] = 0xffffffff; - mIndexBufferNeedsBind = true; } + + if (mThreadCommandBuffer) + { + fb->GetCommands()->EndThreadCommands(std::move(mThreadCommandBuffer)); + } + + // Force rebind of everything on next draw + mPipelineKey = {}; + mLastViewpointOffset = 0xffffffff; + mLastVertexOffsets[0] = 0xffffffff; + mIndexBufferNeedsBind = true; } void VkRenderState::EndFrame() diff --git a/src/common/rendering/vulkan/vk_renderstate.h b/src/common/rendering/vulkan/vk_renderstate.h index 0f9a6c279..a239bda6d 100644 --- a/src/common/rendering/vulkan/vk_renderstate.h +++ b/src/common/rendering/vulkan/vk_renderstate.h @@ -42,11 +42,6 @@ public: void EnableLineSmooth(bool on) override; void EnableDrawBuffers(int count, bool apply) override; - void BeginFrame(); - void SetRenderTarget(VkTextureImage *image, VulkanImageView *depthStencilView, int width, int height, VkFormat Format, VkSampleCountFlagBits samples); - void EndRenderPass(); - void EndFrame(); - // Buffers int SetViewpoint(const HWViewpointUniforms& vp) override; void SetViewpoint(int index) override; @@ -61,6 +56,14 @@ public: void UpdateShadowData(unsigned int index, const FFlatVertex* vertices, unsigned int count) override; void ResetVertices() override; + // Worker threads + void FlushCommands() override { EndRenderPass(); } + + void BeginFrame(); + void SetRenderTarget(VkTextureImage* image, VulkanImageView* depthStencilView, int width, int height, VkFormat Format, VkSampleCountFlagBits samples); + void EndRenderPass(); + void EndFrame(); + protected: void Apply(int dt); void ApplyRenderPass(int dt); @@ -144,6 +147,7 @@ protected: std::map> mRenderPassSetups; std::vector mPipelineLayouts; + std::unique_ptr mThreadCommandBuffer; }; class VkThreadRenderPassSetup