OrdinaryMagician/vkdoom_m
OrdinaryMagician/vkdoom_m
1
Fork 0
Code Activity

Run the GLSL compiler on the worker thread

Browse source
This commit is contained in:
Magnus Norddahl 2025-04-11 23:17:43 +02:00
commit da0ff9ab04
7 changed files with 98 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

32
src/common/rendering/vulkan/pipelines/vk_renderpass.cpp
View file

@ -519,34 +519,38 @@ PipelineData* VkRenderPassSetup::GetSpecializedPipeline(const VkPipelineKey& key
{
// We haven't seen this before. Build it on the worker thread.
SpecializedPipelines[key] = {}; // Mark as seen
struct TaskData
{
std::unique_ptr<GraphicsPipelineBuilder> builder;
VkPipelineKey key;
std::unique_ptr<VulkanPipeline> pipeline;
UniformStructHolder uniforms;
};
auto data = std::make_shared<TaskData>();
// Compile the GLSL on the main thread (mainly due to the resource system)
UniformStructHolder uniforms;
data->builder = CreatePipeline(key, false, uniforms);
SpecializedPipelines[key].Uniforms = uniforms;
data->key = key;
// Schedule the pipeline building on a worker thread
VkPipelineKey k = key;
auto passManager = fb->GetRenderPassManager();
passManager->RunOnWorkerThread([=]() {
auto builder = CreatePipeline(data->key, false, data->uniforms);
cycle_t ct;
ct.ResetAndClock();
data->pipeline = data->builder->Create(fb->GetDevice());
data->pipeline = builder->Create(fb->GetDevice());
ct.Unclock();
const auto duration = ct.TimeMS();
passManager->RunOnMainThread([=]() {
auto& slot = SpecializedPipelines[k];
if (!slot.pipeline)
{
slot.pipeline = std::move(data->pipeline);
slot.Uniforms = std::move(data->uniforms);
}
pipeline_time += duration;
++pipeline_count;
@ -605,7 +609,7 @@ std::unique_ptr<VulkanPipeline> VkRenderPassSetup::CreateWithStats(GraphicsPipel
std::unique_ptr<GraphicsPipelineBuilder> VkRenderPassSetup::CreatePipeline(const VkPipelineKey& key, bool isUberShader, UniformStructHolder& Uniforms)
{
VkShaderProgram* program = fb->GetShaderManager()->Get(key.ShaderKey, isUberShader);
VkShaderProgram* program = fb->GetShaderManager()->GetGameShader(key.ShaderKey, isUberShader);
Uniforms.Clear();
Uniforms = program->Uniforms;
@ -718,19 +722,19 @@ void VkRenderPassSetup::PrecompileFragmentShaderLibrary(const VkPipelineKey& key
struct TaskData
{
std::unique_ptr<GraphicsPipelineBuilder> builder;
VkPipelineKey key;
std::unique_ptr<VulkanPipeline> pipeline;
};
auto data = std::make_shared<TaskData>();
data->builder = CreateFragmentShaderLibrary(key, isUberShader);
data->key = key;
auto passManager = fb->GetRenderPassManager();
passManager->RunOnWorkerThread([=]() {
cycle_t ct;
ct.ResetAndClock();
data->pipeline = data->builder->Create(fb->GetDevice());
data->pipeline = CreateFragmentShaderLibrary(data->key, isUberShader)->Create(fb->GetDevice());
ct.Unclock();
const auto duration = ct.TimeMS();
@ -762,7 +766,7 @@ VulkanPipeline* VkRenderPassSetup::GetFragmentShaderLibrary(const VkPipelineKey&
std::unique_ptr<VulkanPipeline> VkRenderPassSetup::LinkPipeline(const VkPipelineKey& key, bool isUberShader, UniformStructHolder& Uniforms)
{
VkShaderProgram* program = fb->GetShaderManager()->Get(key.ShaderKey, isUberShader);
VkShaderProgram* program = fb->GetShaderManager()->GetGameShader(key.ShaderKey, isUberShader);
Uniforms.Clear();
Uniforms = program->Uniforms;
@ -792,7 +796,7 @@ std::unique_ptr<VulkanPipeline> VkRenderPassSetup::CreateVertexInputLibrary(int
std::unique_ptr<VulkanPipeline> VkRenderPassSetup::CreateVertexShaderLibrary(const VkPipelineKey& key, bool isUberShader)
{
VkShaderProgram* program = fb->GetShaderManager()->Get(key.ShaderKey, isUberShader);
VkShaderProgram* program = fb->GetShaderManager()->GetGameShader(key.ShaderKey, isUberShader);
GraphicsPipelineBuilder builder;
builder.Cache(fb->GetRenderPassManager()->GetCache());
builder.Flags(VK_PIPELINE_CREATE_LIBRARY_BIT_KHR);
@ -807,7 +811,7 @@ std::unique_ptr<VulkanPipeline> VkRenderPassSetup::CreateVertexShaderLibrary(con
std::unique_ptr<GraphicsPipelineBuilder> VkRenderPassSetup::CreateFragmentShaderLibrary(const VkPipelineKey& key, bool isUberShader)
{
VkShaderProgram* program = fb->GetShaderManager()->Get(key.ShaderKey, isUberShader);
VkShaderProgram* program = fb->GetShaderManager()->GetGameShader(key.ShaderKey, isUberShader);
auto builder = std::make_unique<GraphicsPipelineBuilder>();
builder->Cache(fb->GetRenderPassManager()->GetCache());
builder->Flags(VK_PIPELINE_CREATE_LIBRARY_BIT_KHR);

32
src/common/rendering/vulkan/shaders/vk_shader.cpp
View file

@ -91,11 +91,15 @@ void VkShaderManager::Deinit()
RemoveVkPPShader(PPShaders.back());
}
VkShaderProgram* VkShaderManager::Get(const VkShaderKey& key, bool isUberShader)
VkShaderProgram* VkShaderManager::GetGameShader(const VkShaderKey& key, bool isUberShader)
{
VkShaderProgram& program = isUberShader ? generic[key.GeneralizedShaderKey()] : specialized[key];
if (program.frag)
return &program;
std::unique_lock lock(GameShaders.Mutex);
std::unique_ptr<VkShaderProgram>& program = isUberShader ? GameShaders.Generic[key.GeneralizedShaderKey()] : GameShaders.Specialized[key];
if (program)
return program.get();
program = std::make_unique<VkShaderProgram>();
const char* mainvp = "shaders/scene/vert_main.glsl";
const char* mainfp = "shaders/scene/frag_main.glsl";
@ -127,9 +131,9 @@ VkShaderProgram* VkShaderManager::Get(const VkShaderKey& key, bool isUberShader)
customKey.Layout.AlphaTest = false;
const auto& desc = effectshaders[key.SpecialEffect];
program.vert = LoadVertShader(desc.ShaderName, mainvp, nullptr, desc.defines, key, nullptr, isUberShader);
program->vert = LoadVertShader(desc.ShaderName, mainvp, nullptr, desc.defines, key, nullptr, isUberShader);
if (!key.NoFragmentShader)
program.frag = LoadFragShader(desc.ShaderName, desc.fp1, desc.fp2, desc.fp3, desc.fp4, desc.fp5, desc.defines, key, nullptr, isUberShader);
program->frag = LoadFragShader(desc.ShaderName, desc.fp1, desc.fp2, desc.fp3, desc.fp4, desc.fp5, desc.defines, key, nullptr, isUberShader);
}
else
{
@ -167,9 +171,9 @@ VkShaderProgram* VkShaderManager::Get(const VkShaderKey& key, bool isUberShader)
if (key.EffectState < FIRST_USER_SHADER)
{
const auto& desc = defaultshaders[key.EffectState];
program.vert = LoadVertShader(desc.ShaderName, mainvp, nullptr, desc.Defines, key, nullptr, isUberShader);
program->vert = LoadVertShader(desc.ShaderName, mainvp, nullptr, desc.Defines, key, nullptr, isUberShader);
if (!key.NoFragmentShader)
program.frag = LoadFragShader(desc.ShaderName, mainfp, desc.material_lump, desc.mateffect_lump, desc.lightmodel_lump_shared, desc.lightmodel_lump, desc.Defines, key, nullptr, isUberShader);
program->frag = LoadFragShader(desc.ShaderName, mainfp, desc.material_lump, desc.mateffect_lump, desc.lightmodel_lump_shared, desc.lightmodel_lump, desc.Defines, key, nullptr, isUberShader);
}
else
{
@ -177,14 +181,14 @@ VkShaderProgram* VkShaderManager::Get(const VkShaderKey& key, bool isUberShader)
const FString& name = ExtractFileBase(desc.shader.GetChars());
FString defines = defaultshaders[desc.shaderType].Defines + desc.defines;
program.vert = LoadVertShader(name, mainvp, desc.vertshader.IsEmpty() ? nullptr : desc.vertshader.GetChars(), defines.GetChars(), key, &desc, isUberShader);
program->vert = LoadVertShader(name, mainvp, desc.vertshader.IsEmpty() ? nullptr : desc.vertshader.GetChars(), defines.GetChars(), key, &desc, isUberShader);
if (!key.NoFragmentShader)
program.frag = LoadFragShader(name, mainfp, desc.shader.GetChars(), defaultshaders[desc.shaderType].mateffect_lump, defaultshaders[desc.shaderType].lightmodel_lump_shared, defaultshaders[desc.shaderType].lightmodel_lump, defines.GetChars(), key, &desc, isUberShader);
program->frag = LoadFragShader(name, mainfp, desc.shader.GetChars(), defaultshaders[desc.shaderType].mateffect_lump, defaultshaders[desc.shaderType].lightmodel_lump_shared, defaultshaders[desc.shaderType].lightmodel_lump, defines.GetChars(), key, &desc, isUberShader);
desc.Uniforms.WriteUniforms(program.Uniforms);
desc.Uniforms.WriteUniforms(program->Uniforms);
}
}
return &program;
return program.get();
}
enum class FieldCondition
@ -630,12 +634,12 @@ FString VkShaderManager::GetVersionBlock()
FString VkShaderManager::LoadPublicShaderLump(const char* lumpname)
{
return fb->GetShaderCache()->GetPublicFile(lumpname).Code;
return fb->GetShaderCache()->GetPublicFile(lumpname)->Code;
}
FString VkShaderManager::LoadPrivateShaderLump(const char* lumpname)
{
return fb->GetShaderCache()->GetPrivateFile(lumpname).Code;
return fb->GetShaderCache()->GetPrivateFile(lumpname)->Code;
}
VkPPShader* VkShaderManager::GetVkShader(PPShader* shader)

11
src/common/rendering/vulkan/shaders/vk_shader.h
View file

@ -4,6 +4,7 @@
#include <memory>
#include <list>
#include <map>
#include <mutex>
#include "vectors.h"
#include "matrix.h"
#include "name.h"
@ -168,7 +169,7 @@ public:
void Deinit();
VkShaderProgram* Get(const VkShaderKey& key, bool isUberShader);
VkShaderProgram* GetGameShader(const VkShaderKey& key, bool isUberShader);
bool CompileNextShader() { return true; }
@ -194,8 +195,12 @@ private:
VulkanRenderDevice* fb = nullptr;
std::map<uint64_t, VkShaderProgram> generic;
std::map<VkShaderKey, VkShaderProgram> specialized;
struct
{
std::mutex Mutex;
std::map<uint64_t, std::unique_ptr<VkShaderProgram>> Generic;
std::map<VkShaderKey, std::unique_ptr<VkShaderProgram>> Specialized;
} GameShaders;
std::list<VkPPShader*> PPShaders;

62
src/common/rendering/vulkan/shaders/vk_shadercache.cpp
View file

@ -30,6 +30,8 @@ std::vector<uint32_t> VkShaderCache::Compile(ShaderType type, const TArrayView<V
{
// Is this something we already compiled before?
std::unique_lock lock(CacheMutex);
FString checksum = CalcSha1(type, sources);
auto it = CodeCache.find(checksum);
if (it != CodeCache.end())
@ -43,8 +45,8 @@ std::vector<uint32_t> VkShaderCache::Compile(ShaderType type, const TArrayView<V
bool foundChanges = false;
for (const VkCachedInclude& includeinfo : cacheinfo.Includes)
{
const VkCachedShaderLump& lumpinfo = includeinfo.PrivateLump ? GetPrivateFile(includeinfo.LumpName) : GetPublicFile(includeinfo.LumpName);
if (lumpinfo.Checksum != includeinfo.Checksum)
const VkCachedShaderLump* lumpinfo = includeinfo.PrivateLump ? GetPrivateFile(includeinfo.LumpName) : GetPublicFile(includeinfo.LumpName);
if (lumpinfo->Checksum != includeinfo.Checksum)
{
foundChanges = true;
break;
@ -63,6 +65,8 @@ std::vector<uint32_t> VkShaderCache::Compile(ShaderType type, const TArrayView<V
}
}
lock.unlock();
// No match or out of date
// Compile it and store the dependencies:
@ -78,6 +82,8 @@ std::vector<uint32_t> VkShaderCache::Compile(ShaderType type, const TArrayView<V
cachedCompile.Code = compiler.Compile(fb->GetDevice());
cachedCompile.LastUsed = LaunchTime;
lock.lock();
auto& c = CodeCache[checksum];
c = std::move(cachedCompile);
return c.Code;
@ -88,12 +94,12 @@ ShaderIncludeResult VkShaderCache::OnInclude(VkCachedCompile& cachedCompile, FSt
if (depth > 8)
I_Error("Too much include recursion!");
const VkCachedShaderLump& file = system ? GetPrivateFile(headerName) : GetPublicFile(headerName);
const VkCachedShaderLump* file = system ? GetPrivateFile(headerName) : GetPublicFile(headerName);
VkCachedInclude cacheinfo;
cacheinfo.LumpName = headerName;
cacheinfo.PrivateLump = system;
cacheinfo.Checksum = file.Checksum;
cacheinfo.Checksum = file->Checksum;
cachedCompile.Includes.push_back(std::move(cacheinfo));
FString includeguardname;
@ -104,7 +110,7 @@ ShaderIncludeResult VkShaderCache::OnInclude(VkCachedCompile& cachedCompile, FSt
code << "#ifndef " << includeguardname.GetChars() << "\n";
code << "#define " << includeguardname.GetChars() << "\n";
code << "#line 1\n";
code << includeFilter(file.Code.GetChars()) << "\n";
code << includeFilter(file->Code.GetChars()) << "\n";
code << "#endif\n";
return ShaderIncludeResult(headerName.GetChars(), code.GetChars());
@ -129,34 +135,52 @@ FString VkShaderCache::CalcSha1(const FString& str)
return sha1.final();
}
const VkCachedShaderLump& VkShaderCache::GetPublicFile(const FString& lumpname)
const VkCachedShaderLump* VkShaderCache::GetPublicFile(const FString& lumpname)
{
std::unique_lock lock(FileMutex);
auto it = PublicFiles.find(lumpname);
if (it != PublicFiles.end())
return it->second;
return it->second.get();
VkCachedShaderLump cacheinfo;
cacheinfo.Code = LoadPublicShaderLump(lumpname.GetChars());
cacheinfo.Checksum = CalcSha1(cacheinfo.Code);
auto cacheinfo = std::make_unique<VkCachedShaderLump>();
cacheinfo->Code = LoadPublicShaderLump(lumpname.GetChars());
cacheinfo->Checksum = CalcSha1(cacheinfo->Code);
auto& result = PublicFiles[lumpname];
result = std::move(cacheinfo);
return result;
return result.get();
}
const VkCachedShaderLump& VkShaderCache::GetPrivateFile(const FString& lumpname)
const VkCachedShaderLump* VkShaderCache::GetPrivateFile(const FString& lumpname)
{
std::unique_lock lock(FileMutex);
auto it = PrivateFiles.find(lumpname);
if (it != PrivateFiles.end())
return it->second;
return it->second.get();
VkCachedShaderLump cacheinfo;
cacheinfo.Code = LoadPrivateShaderLump(lumpname.GetChars());
cacheinfo.Checksum = CalcSha1(cacheinfo.Code);
auto cacheinfo = std::make_unique<VkCachedShaderLump>();
cacheinfo->Code = LoadPrivateShaderLump(lumpname.GetChars());
cacheinfo->Checksum = CalcSha1(cacheinfo->Code);
auto& result = PrivateFiles[lumpname];
result = std::move(cacheinfo);
return result;
return result.get();
}
FString GetStringFromLumpThreadsafe(int lump)
{
auto reader = fileSystem.OpenFileReader(lump, FileSys::READER_NEW, FileSys::READERFLAG_SEEKABLE);
if (reader.GetBuffer())
{
return FString(reader.GetBuffer(), reader.GetLength());
}
else
{
auto data = reader.Read();
return FString(static_cast<const char*>(data.data()), data.size());
}
}
FString VkShaderCache::LoadPublicShaderLump(const char* lumpname)
@ -164,14 +188,14 @@ FString VkShaderCache::LoadPublicShaderLump(const char* lumpname)
int lump = fileSystem.CheckNumForFullName(lumpname, 0);
if (lump == -1) lump = fileSystem.CheckNumForFullName(lumpname);
if (lump == -1) I_Error("Unable to load '%s'", lumpname);
return GetStringFromLump(lump);
return GetStringFromLumpThreadsafe(lump);
}
FString VkShaderCache::LoadPrivateShaderLump(const char* lumpname)
{
int lump = fileSystem.CheckNumForFullName(lumpname, 0);
if (lump == -1) I_Error("Unable to load '%s'", lumpname);
return GetStringFromLump(lump);
return GetStringFromLumpThreadsafe(lump);
}
static uint8_t readUInt8(FileReader& fr) { uint8_t v = 0; fr.Read(&v, 1); return v; }

11
src/common/rendering/vulkan/shaders/vk_shadercache.h
View file

@ -50,8 +50,8 @@ public:
std::vector<uint32_t> Compile(ShaderType type, const TArrayView<VkShaderSource>& sources, const std::function<FString(FString)>& includeFilter = {});
const VkCachedShaderLump& GetPublicFile(const FString& lumpname);
const VkCachedShaderLump& GetPrivateFile(const FString& lumpname);
const VkCachedShaderLump* GetPublicFile(const FString& lumpname);
const VkCachedShaderLump* GetPrivateFile(const FString& lumpname);
private:
void Load();
@ -70,9 +70,12 @@ private:
VulkanRenderDevice* fb = nullptr;
std::map<FString, VkCachedShaderLump> PublicFiles;
std::map<FString, VkCachedShaderLump> PrivateFiles;
std::mutex CacheMutex;
std::map<FString, VkCachedCompile> CodeCache;
std::mutex FileMutex;
std::map<FString, std::unique_ptr<VkCachedShaderLump>> PublicFiles;
std::map<FString, std::unique_ptr<VkCachedShaderLump>> PrivateFiles;
};
class CachedGLSLCompiler

4
src/common/rendering/vulkan/vk_levelmesh.cpp
View file

@ -651,12 +651,12 @@ void VkLevelMesh::CreateViewerObjects()
FString VkLevelMesh::LoadPrivateShaderLump(const char* lumpname)
{
return fb->GetShaderCache()->GetPrivateFile(lumpname).Code;
return fb->GetShaderCache()->GetPrivateFile(lumpname)->Code;
}
FString VkLevelMesh::LoadPublicShaderLump(const char* lumpname)
{
return fb->GetShaderCache()->GetPublicFile(lumpname).Code;
return fb->GetShaderCache()->GetPublicFile(lumpname)->Code;
}
/////////////////////////////////////////////////////////////////////////////

4
src/common/rendering/vulkan/vk_lightmapper.cpp
View file

@ -610,12 +610,12 @@ int VkLightmapper::GetRaytracePipelineIndex()
FString VkLightmapper::LoadPrivateShaderLump(const char* lumpname)
{
return fb->GetShaderCache()->GetPrivateFile(lumpname).Code;
return fb->GetShaderCache()->GetPrivateFile(lumpname)->Code;
}
FString VkLightmapper::LoadPublicShaderLump(const char* lumpname)
{
return fb->GetShaderCache()->GetPublicFile(lumpname).Code;
return fb->GetShaderCache()->GetPublicFile(lumpname)->Code;
}
void VkLightmapper::CreateRaytracePipeline()