Slice the BSP and process it on worker threads

src/rendering/hwrenderer/scene/hw_drawinfo.cpp

@@ -450,22 +450,13 @@ void HWDrawInfo::RenderPVS(bool drawpsprites, FRenderState& state)
 		uselevelmesh = true;
 
 		Bsp.Clock();
-		
-		// To do: do this over many threads
-		static HWVisibleSet VisibleSet;
-		VisibleSet.FindPVS(this);
 
-		// Merge results
-		SeenSectors.Clear();
-		SeenSides.Clear();
-		for (int sectorIndex : VisibleSet.SeenSectors.Get())
-			SeenSectors.Add(sectorIndex);
-		for (int sideIndex : VisibleSet.SeenSides.Get())
-			SeenSides.Add(sideIndex);
+		static HWVisibleSetThreads threads;
+		threads.FindPVS(this);
 
 		if (!gl_levelmesh) // Update sector heights for the non-levelmesh rendering of sectors
 		{
-			for (int sectorIndex : VisibleSet.SeenSectors.Get())
+			for (int sectorIndex : SeenSectors.Get())
 				CheckUpdate(state, &Level->sectors[sectorIndex]);
 		}
 

src/rendering/hwrenderer/scene/hw_visibleset.cpp

@@ -45,7 +45,7 @@ angle_t HWVisibleSet::FrustumAngle()
 	return a1;
 }
 
-void HWVisibleSet::FindPVS(HWDrawInfo* di)
+void HWVisibleSet::FindPVS(HWDrawInfo* di, int sliceIndex, int sliceCount)
 {
 	Level = di->Level;
 	Viewpoint = di->Viewpoint;
@@ -62,7 +62,14 @@ void HWVisibleSet::FindPVS(HWDrawInfo* di)
 
 	const auto& vp = Viewpoint;
 	angle_t a1 = FrustumAngle();
-	mClipper->SafeAddClipRangeRealAngles(vp.Angles.Yaw.BAMs() + a1, vp.Angles.Yaw.BAMs() - a1);
+
+	// Find the range for our slice
+	//uint64_t start = static_cast<uint64_t>(vp.Angles.Yaw.BAMs() + a1);
+	uint64_t end = static_cast<uint64_t>(vp.Angles.Yaw.BAMs() - a1);
+	uint64_t length = static_cast<uint64_t>(a1) * 2;
+	uint32_t sliceend = static_cast<uint32_t>(end + length * sliceIndex / sliceCount);
+	uint32_t slicestart = static_cast<uint32_t>(end + length * (sliceIndex + 1) / sliceCount);
+	mClipper->SafeAddClipRangeRealAngles(slicestart, sliceend);
 
 	drawctx.portalState.StartFrame();
 
@@ -601,3 +608,103 @@ void HWVisibleSet::RenderParticles(subsector_t* sub, sector_t* front)
 	SetupSprite.Unclock();
 #endif
 }
+
+/////////////////////////////////////////////////////////////////////////////
+
+HWVisibleSetThreads::HWVisibleSetThreads()
+{
+	SliceCount = std::max((int)(std::thread::hardware_concurrency() * 3 / 4), 1);
+	Slices.Resize(SliceCount);
+	WorkFlags.resize(SliceCount);
+
+	size_t threadCount = SliceCount - 1;
+	while (Threads.size() < threadCount)
+	{
+		int sliceIndex = Threads.size() + 1;
+		Threads.push_back(std::thread([=]() { WorkerMain(sliceIndex); }));
+	}
+}
+
+HWVisibleSetThreads::~HWVisibleSetThreads()
+{
+	std::unique_lock lock(Mutex);
+	StopFlag = true;
+	lock.unlock();
+	Condvar.notify_all();
+	for (auto& thread : Threads)
+		thread.join();
+}
+
+void HWVisibleSetThreads::FindPVS(HWDrawInfo* di)
+{
+	// Tell workers there's work to do!
+	std::unique_lock lock(Mutex);
+	DrawInfo = di;
+	for (int i = 0; i < SliceCount; i++)
+		WorkFlags[i] = true;
+	lock.unlock();
+	Condvar.notify_all();
+
+	// Process slice zero ourselves
+	Slices[0].FindPVS(DrawInfo, 0, SliceCount);
+
+	// Wait for all workers
+	lock.lock();
+	WorkFlags[0] = false;
+	while (true)
+	{
+		bool allDone = true;
+		for (int i = 0; i < SliceCount; i++)
+		{
+			if (WorkFlags[i])
+			{
+				allDone = false;
+				break;
+			}
+		}
+		if (allDone)
+			break;
+		Condvar.wait(lock);
+	}
+
+	// Merge results
+	di->SeenSectors.Clear();
+	di->SeenSides.Clear();
+	for (int i = 0; i < SliceCount; i++)
+	{
+		for (int sectorIndex : Slices[i].SeenSectors.Get())
+			di->SeenSectors.Add(sectorIndex);
+		for (int sideIndex : Slices[i].SeenSides.Get())
+			di->SeenSides.Add(sideIndex);
+	}
+}
+
+void HWVisibleSetThreads::WorkerMain(int sliceIndex)
+{
+	std::unique_lock lock(Mutex);
+	while (true)
+	{
+		if (StopFlag)
+			break;
+		if (WorkFlags[sliceIndex])
+		{
+			lock.unlock();
+			Slices[sliceIndex].FindPVS(DrawInfo, sliceIndex, SliceCount);
+			lock.lock();
+			WorkFlags[sliceIndex] = false;
+
+			bool allDone = true;
+			for (int i = 0; i < SliceCount; i++)
+			{
+				if (WorkFlags[i])
+				{
+					allDone = false;
+					break;
+				}
+			}
+			if (allDone)
+				Condvar.notify_all();
+		}
+		Condvar.wait(lock);
+	}
+}

src/rendering/hwrenderer/scene/hw_visibleset.h

@@ -4,6 +4,8 @@
 #include "r_utility.h"
 #include "hw_fakeflat.h"
 #include "hw_drawcontext.h"
+#include <mutex>
+#include <condition_variable>
 
 class Clipper;
 class HWPortal;
@@ -12,7 +14,7 @@ struct HWDrawInfo;
 class HWVisibleSet
 {
 public:
-	void FindPVS(HWDrawInfo* di);
+	void FindPVS(HWDrawInfo* di, int sliceIndex, int sliceCount);
 
 	struct VisList
 	{
@@ -91,3 +93,24 @@ private:
 
 	int rendered_lines = 0;
 };
+
+class HWVisibleSetThreads
+{
+public:
+	HWVisibleSetThreads();
+	~HWVisibleSetThreads();
+
+	void FindPVS(HWDrawInfo* di);
+
+private:
+	void WorkerMain(int sliceIndex);
+
+	int SliceCount = 1;
+	TArray<HWVisibleSet> Slices;
+	std::vector<std::thread> Threads;
+	std::mutex Mutex;
+	std::condition_variable Condvar;
+	std::vector<bool> WorkFlags;
+	bool StopFlag = false;
+	HWDrawInfo* DrawInfo = nullptr;
+};