From 5fb2fa48c55f71dda56a8f53aec01b58438ac715 Mon Sep 17 00:00:00 2001
From: RaveYard <29225776+MrRaveYard@users.noreply.github.com>
Date: Thu, 10 Apr 2025 12:46:23 +0200
Subject: [PATCH] Add support for multiple pipeline work threads

---
 .../vulkan/pipelines/vk_renderpass.cpp        | 65 +++++++++++++++++--
 .../vulkan/pipelines/vk_renderpass.h          |  5 +-
 2 files changed, 62 insertions(+), 8 deletions(-)

diff --git a/src/common/rendering/vulkan/pipelines/vk_renderpass.cpp b/src/common/rendering/vulkan/pipelines/vk_renderpass.cpp
index 541ca427c..ef2b6be5e 100644
--- a/src/common/rendering/vulkan/pipelines/vk_renderpass.cpp
+++ b/src/common/rendering/vulkan/pipelines/vk_renderpass.cpp
@@ -39,6 +39,11 @@
 
 CVAR(Bool, gl_ubershaders, true, 0);
 CVAR(Bool, vk_debug_pipeline_creation, true, 0);
+EXTERN_CVAR(Bool, gl_multithread);
+CUSTOM_CVAR(Int, vk_pipeline_threads, 0, 0) // Todo: archive?
+{
+	if (self < 0) self = 0;
+}
 
 // Pipeline creation tracking and printing
 int Printf(const char* fmt, ...);
@@ -55,6 +60,29 @@ ADD_STAT(pipelines)
 	return out;
 }
 
+static unsigned MaxPipelineThreads()
+{
+	unsigned usedThreads = 2; // this thread + one extra
+	unsigned hwLimit = std::thread::hardware_concurrency();
+
+	if (gl_multithread)
+		usedThreads++;
+
+	return (usedThreads >= hwLimit) ? 1u : hwLimit - usedThreads; // max(1, hwLimit - usedThreads)
+}
+
+static unsigned CalculatePipelineThreadCountTarget()
+{
+	unsigned requested = static_cast<unsigned>(vk_pipeline_threads);
+
+	if (requested <= 0)
+	{
+		requested = std::thread::hardware_concurrency() / 2;
+	}
+
+	return clamp(requested, 1u, MaxPipelineThreads());
+}
+
 VkRenderPassManager::VkRenderPassManager(VulkanRenderDevice* fb) : fb(fb)
 {
 	FString path = M_GetCachePath(true);
@@ -83,15 +111,14 @@ VkRenderPassManager::VkRenderPassManager(VulkanRenderDevice* fb) : fb(fb)
 
 	PipelineCache = builder.Create(fb->GetDevice());
 
-	Worker.Thread = std::thread([this]() { WorkerThreadMain(); });
-
+	CreatePipelineWorkThreads();
 	CreateLightTilesPipeline();
 	CreateZMinMaxPipeline();
 }
 
 VkRenderPassManager::~VkRenderPassManager()
 {
-	StopWorkerThread();
+	StopWorkerThreads();
 
 	try
 	{
@@ -107,6 +134,12 @@ VkRenderPassManager::~VkRenderPassManager()
 
 void VkRenderPassManager::ProcessMainThreadTasks()
 {
+	if (CalculatePipelineThreadCountTarget() != Worker.Threads.size())
+	{
+		StopWorkerThreads();
+		CreatePipelineWorkThreads();
+	}
+
 	std::unique_lock lock(Worker.Mutex);
 	std::vector<std::function<void()>> tasks;
 	tasks.swap(Worker.MainTasks);
@@ -132,15 +165,18 @@ void VkRenderPassManager::RunOnMainThread(std::function<void()> task)
 	Worker.MainTasks.push_back(std::move(task));
 }
 
-void VkRenderPassManager::StopWorkerThread()
+void VkRenderPassManager::StopWorkerThreads()
 {
 	std::unique_lock lock(Worker.Mutex);
 	Worker.StopFlag = true;
 	lock.unlock();
 	Worker.CondVar.notify_all();
-	Worker.Thread.join();
+	for (auto& thread : Worker.Threads)
+	{
+		thread->join();
+	}
 	lock.lock();
-	Worker.MainTasks.clear();
+	Worker.Threads.clear();
 	Worker.WorkerTasks.clear();
 	Worker.StopFlag = false;
 }
@@ -254,6 +290,23 @@ VkPPRenderPassSetup* VkRenderPassManager::GetPPRenderPass(const VkPPRenderPassKe
 	return passSetup.get();
 }
 
+void VkRenderPassManager::CreatePipelineWorkThreads()
+{
+	unsigned threadCount = CalculatePipelineThreadCountTarget();
+
+	Worker.Threads.reserve(threadCount);
+
+	for (unsigned i = 0; i < threadCount; ++i)
+	{
+		Worker.Threads.emplace_back(new std::thread([this]() { WorkerThreadMain(); }));
+	}
+
+	if (vk_debug_pipeline_creation)
+	{
+		Printf("VkRenderPassManager: Created %d worker threads\n", threadCount);
+	}
+}
+
 void VkRenderPassManager::CreateLightTilesPipeline()
 {
 	LightTiles.Layout = PipelineLayoutBuilder()
diff --git a/src/common/rendering/vulkan/pipelines/vk_renderpass.h b/src/common/rendering/vulkan/pipelines/vk_renderpass.h
index 801d00857..719d8679f 100644
--- a/src/common/rendering/vulkan/pipelines/vk_renderpass.h
+++ b/src/common/rendering/vulkan/pipelines/vk_renderpass.h
@@ -171,10 +171,11 @@ public:
 	void RunOnMainThread(std::function<void()> task);
 
 private:
+	void CreatePipelineWorkThreads();
 	void CreateLightTilesPipeline();
 	void CreateZMinMaxPipeline();
 
-	void StopWorkerThread();
+	void StopWorkerThreads();
 	void WorkerThreadMain();
 
 	VulkanRenderDevice* fb = nullptr;
@@ -203,7 +204,7 @@ private:
 
 	struct
 	{
-		std::thread Thread;
+		std::vector<std::unique_ptr<std::thread>> Threads;
 		std::mutex Mutex;
 		std::condition_variable CondVar;
 		bool StopFlag = false;