OrdinaryMagician/vkdoom_m
OrdinaryMagician/vkdoom_m
1
Fork 0
Code Activity

Cache all GLSL shader compilation

Browse source
This commit is contained in:
Magnus Norddahl 2025-02-26 01:47:07 +01:00
commit 640a2b2c41
6 changed files with 177 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

140
src/common/rendering/vulkan/shaders/vk_shadercache.cpp
View file

@ -5,10 +5,25 @@
#include "filesystem.h"
#include "engineerrors.h"
#include "cmdlib.h"
#include "i_specialpaths.h"
#include <zvulkan/vulkanbuilders.h>
#include <chrono>
VkShaderCache::VkShaderCache(VulkanRenderDevice* fb) : fb(fb)
{
FString path = M_GetCachePath(true);
CreatePath(path.GetChars());
CacheFilename = path + "/shadercache.zdsc";
using namespace std::chrono;
LaunchTime = (uint64_t)(duration_cast<seconds>(system_clock::now().time_since_epoch()).count());
Load();
}
VkShaderCache::~VkShaderCache()
{
Save();
}
std::vector<uint32_t> VkShaderCache::Compile(ShaderType type, const TArrayView<VkShaderSource>& sources, const std::function<FString(FString)>& includeFilter)
@ -23,7 +38,7 @@ std::vector<uint32_t> VkShaderCache::Compile(ShaderType type, const TArrayView<V
try
{
const VkCachedCompile& cacheinfo = it->second;
VkCachedCompile& cacheinfo = it->second;
bool foundChanges = false;
for (const VkCachedInclude& includeinfo : cacheinfo.Includes)
@ -37,7 +52,10 @@ std::vector<uint32_t> VkShaderCache::Compile(ShaderType type, const TArrayView<V
}
if (!foundChanges)
{
cacheinfo.LastUsed = LaunchTime;
return cacheinfo.Code;
}
}
catch (...)
{
@ -58,6 +76,7 @@ std::vector<uint32_t> VkShaderCache::Compile(ShaderType type, const TArrayView<V
for (const VkShaderSource& source : sources)
compiler.AddSource(source.Name, source.Code);
cachedCompile.Code = compiler.Compile(fb->GetDevice());
cachedCompile.LastUsed = LaunchTime;
auto& c = CodeCache[checksum];
c = std::move(cachedCompile);
@ -155,6 +174,125 @@ FString VkShaderCache::LoadPrivateShaderLump(const char* lumpname)
return GetStringFromLump(lump);
}
static uint8_t readUInt8(FileReader& fr) { uint8_t v = 0; fr.Read(&v, 1); return v; }
static uint16_t readUInt16(FileReader& fr) { uint16_t v = 0; fr.Read(&v, 2); return v; }
static uint32_t readUInt32(FileReader& fr) { uint32_t v = 0; fr.Read(&v, 4); return v; }
static uint64_t readUInt64(FileReader& fr) { uint64_t v = 0; fr.Read(&v, 8); return v; }
static FString readString(FileReader& fr, std::vector<char>& strbuffer)
{
size_t size = fr.ReadUInt32();
if (strbuffer.size() < size + 1)
strbuffer.resize(size + 1);
if (fr.Read(strbuffer.data(), size) != (FileReader::Size)size)
return {};
strbuffer[size] = 0;
return strbuffer.data();
}
static void writeUInt8(std::unique_ptr<FileWriter>& fw, uint8_t v) { fw->Write(&v, 1); }
static void writeUInt16(std::unique_ptr<FileWriter>& fw, uint16_t v) { fw->Write(&v, 2); }
static void writeUInt32(std::unique_ptr<FileWriter>& fw, uint32_t v) { fw->Write(&v, 4); }
static void writeUInt64(std::unique_ptr<FileWriter>& fw, uint64_t v) { fw->Write(&v, 8); }
static void writeString(std::unique_ptr<FileWriter>& fw, const FString& s)
{
writeUInt32(fw, s.Len());
fw->Write(s.GetChars(), s.Len());
}
void VkShaderCache::Load()
{
try
{
FileReader fr;
if (fr.OpenFile(CacheFilename.GetChars()))
{
char magic[11] = {};
fr.Read(magic, 11);
if (memcmp(magic, "shadercache", 11) != 0)
return;
uint32_t version = readUInt32(fr);
if (version != 1)
return;
std::vector<char> strbuffer;
uint32_t count = readUInt32(fr);
while (count > 0)
{
FString checksum = readString(fr, strbuffer);
VkCachedCompile cachedCompile;
cachedCompile.LastUsed = readUInt64(fr);
cachedCompile.Code.resize(readUInt32(fr));
if (fr.Read(cachedCompile.Code.data(), cachedCompile.Code.size() * sizeof(uint32_t)) != (FileReader::Size)cachedCompile.Code.size() * sizeof(uint32_t))
return;
uint32_t includecount = readUInt32(fr);
while (includecount > 0)
{
VkCachedInclude cachedInclude;
cachedInclude.LumpName = readString(fr, strbuffer);
cachedInclude.PrivateLump = readUInt8(fr) != 0;
cachedInclude.Checksum = readString(fr, strbuffer);
cachedCompile.Includes.push_back(std::move(cachedInclude));
includecount--;
}
// Drop entry if it hasn't been used for 30 days
if (cachedCompile.LastUsed + 30 * 24 * 60 * 60 > LaunchTime)
{
CodeCache[checksum] = std::move(cachedCompile);
}
count--;
}
}
}
catch (...)
{
}
}
void VkShaderCache::Save()
{
try
{
std::unique_ptr<FileWriter> fw(FileWriter::Open(CacheFilename.GetChars()));
if (!fw)
return;
fw->Write("shadercache", 11);
uint32_t version = 1;
writeUInt32(fw, version);
writeUInt32(fw, CodeCache.size());
for (const auto& it : CodeCache)
{
const FString& checksum = it.first;
const VkCachedCompile& cachedCompile = it.second;
writeString(fw, it.first);
writeUInt64(fw, cachedCompile.LastUsed);
writeUInt32(fw, (uint32_t)cachedCompile.Code.size());
fw->Write(cachedCompile.Code.data(), cachedCompile.Code.size() * sizeof(uint32_t));
writeUInt32(fw, cachedCompile.Includes.size());
for (const VkCachedInclude& cachedInclude : cachedCompile.Includes)
{
writeString(fw, cachedInclude.LumpName);
writeUInt8(fw, cachedInclude.PrivateLump ? 1 : 0);
writeString(fw, cachedInclude.Checksum);
}
}
}
catch (...)
{
}
}
/////////////////////////////////////////////////////////////////////////////
std::vector<uint32_t> CachedGLSLCompiler::Compile(VulkanRenderDevice* fb)

8
src/common/rendering/vulkan/shaders/vk_shadercache.h
View file

@ -37,6 +37,7 @@ public:
class VkCachedCompile
{
public:
uint64_t LastUsed = 0;
std::vector<uint32_t> Code;
std::vector<VkCachedInclude> Includes;
};
@ -45,6 +46,7 @@ class VkShaderCache
{
public:
VkShaderCache(VulkanRenderDevice* fb);
~VkShaderCache();
std::vector<uint32_t> Compile(ShaderType type, const TArrayView<VkShaderSource>& sources, const std::function<FString(FString)>& includeFilter = {});
@ -52,6 +54,9 @@ public:
const VkCachedShaderLump& GetPrivateFile(const FString& lumpname);
private:
void Load();
void Save();
FString CalcSha1(ShaderType type, const TArrayView<VkShaderSource>& sources);
FString CalcSha1(const FString& str);
@ -60,6 +65,9 @@ private:
static FString LoadPublicShaderLump(const char* lumpname);
static FString LoadPrivateShaderLump(const char* lumpname);
FString CacheFilename;
uint64_t LaunchTime = 0;
VulkanRenderDevice* fb = nullptr;
std::map<FString, VkCachedShaderLump> PublicFiles;

49
src/common/rendering/vulkan/vk_levelmesh.cpp
View file

@ -26,6 +26,7 @@
#include "vulkan/commands/vk_commandbuffer.h"
#include "vulkan/descriptorsets/vk_descriptorset.h"
#include "vulkan/pipelines/vk_renderpass.h"
#include "vulkan/shaders/vk_shadercache.h"
#include "hw_levelmesh.h"
#include "hw_material.h"
#include "texturemanager.h"
@ -603,28 +604,21 @@ void VkLevelMesh::CreateViewerObjects()
versionBlock += "#define USE_RAYQUERY\r\n";
}
auto onIncludeLocal = [](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); };
auto onIncludeSystem = [](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); };
Viewer.VertexShader = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Vertex)
.AddSource("versionblock", versionBlock)
.AddSource("vert_viewer.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_viewer.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("Viewer.VertexShader")
.Create("vertex", fb->GetDevice());
Viewer.FragmentShader = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Fragment)
.AddSource("versionblock", versionBlock)
.AddSource("frag_viewer.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_viewer.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("Viewer.FragmentShader")
.Create("vertex", fb->GetDevice());
@ -657,41 +651,12 @@ void VkLevelMesh::CreateViewerObjects()
FString VkLevelMesh::LoadPrivateShaderLump(const char* lumpname)
{
int lump = fileSystem.CheckNumForFullName(lumpname, 0);
if (lump == -1) I_Error("Unable to load '%s'", lumpname);
return GetStringFromLump(lump);
return fb->GetShaderCache()->GetPrivateFile(lumpname).Code;
}
FString VkLevelMesh::LoadPublicShaderLump(const char* lumpname)
{
int lump = fileSystem.CheckNumForFullName(lumpname, 0);
if (lump == -1) lump = fileSystem.CheckNumForFullName(lumpname);
if (lump == -1) I_Error("Unable to load '%s'", lumpname);
return GetStringFromLump(lump);
}
ShaderIncludeResult VkLevelMesh::OnInclude(FString headerName, FString includerName, size_t depth, bool system)
{
if (depth > 8)
I_Error("Too much include recursion!");
FString includeguardname;
includeguardname << "_HEADERGUARD_" << headerName.GetChars();
includeguardname.ReplaceChars("/\\.", '_');
FString code;
code << "#ifndef " << includeguardname.GetChars() << "\n";
code << "#define " << includeguardname.GetChars() << "\n";
code << "#line 1\n";
if (system)
code << LoadPrivateShaderLump(headerName.GetChars()).GetChars() << "\n";
else
code << LoadPublicShaderLump(headerName.GetChars()).GetChars() << "\n";
code << "#endif\n";
return ShaderIncludeResult(headerName.GetChars(), code.GetChars());
return fb->GetShaderCache()->GetPublicFile(lumpname).Code;
}
/////////////////////////////////////////////////////////////////////////////

5
src/common/rendering/vulkan/vk_levelmesh.h
View file

@ -118,9 +118,8 @@ private:
void CreateViewerObjects();
static FString LoadPrivateShaderLump(const char* lumpname);
static FString LoadPublicShaderLump(const char* lumpname);
static ShaderIncludeResult OnInclude(FString headerName, FString includerName, size_t depth, bool system);
FString LoadPrivateShaderLump(const char* lumpname);
FString LoadPublicShaderLump(const char* lumpname);
VulkanRenderDevice* fb = nullptr;

85
src/common/rendering/vulkan/vk_lightmapper.cpp
View file

@ -4,6 +4,7 @@
#include "vulkan/textures/vk_texture.h"
#include "vulkan/commands/vk_commandbuffer.h"
#include "vulkan/descriptorsets/vk_descriptorset.h"
#include "vulkan/shaders/vk_shadercache.h"
#include "vk_levelmesh.h"
#include "zvulkan/vulkanbuilders.h"
#include "filesystem.h"
@ -501,39 +502,30 @@ void VkLightmapper::CreateShaders()
traceprefix += "#define USE_RAYQUERY\r\n";
}
auto onIncludeLocal = [](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); };
auto onIncludeSystem = [](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); };
shaders.vertRaytrace = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Vertex)
.AddSource("VersionBlock", prefix)
.AddSource("vert_raytrace.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_raytrace.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("VkLightmapper.VertRaytrace")
.Create("VkLightmapper.VertRaytrace", fb->GetDevice());
shaders.vertScreenquad = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Vertex)
.AddSource("VersionBlock", prefix)
.AddSource("vert_screenquad.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_screenquad.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("VkLightmapper.VertScreenquad")
.Create("VkLightmapper.VertScreenquad", fb->GetDevice());
shaders.vertCopy = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Vertex)
.AddSource("VersionBlock", prefix)
.AddSource("vert_copy.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_copy.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("VkLightmapper.VertCopy")
.Create("VkLightmapper.VertCopy", fb->GetDevice());
@ -550,58 +542,48 @@ void VkLightmapper::CreateShaders()
defines += "#define USE_BOUNCE\n";
shaders.fragRaytrace[i] = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Fragment)
.AddSource("VersionBlock", defines)
.AddSource("frag_raytrace.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_raytrace.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("VkLightmapper.FragRaytrace")
.Create("VkLightmapper.FragRaytrace", fb->GetDevice());
}
shaders.fragResolve = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Fragment)
.AddSource("VersionBlock", prefix)
.AddSource("frag_resolve.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_resolve.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("VkLightmapper.FragResolve")
.Create("VkLightmapper.FragResolve", fb->GetDevice());
shaders.fragBlur[0] = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Fragment)
.AddSource("VersionBlock", prefix + "#define BLUR_HORIZONTAL\r\n")
.AddSource("frag_blur.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_blur.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("VkLightmapper.FragBlur")
.Create("VkLightmapper.FragBlur", fb->GetDevice());
shaders.fragBlur[1] = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Fragment)
.AddSource("VersionBlock", prefix + "#define BLUR_VERTICAL\r\n")
.AddSource("frag_blur.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_blur.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("VkLightmapper.FragBlur")
.Create("VkLightmapper.FragBlur", fb->GetDevice());
shaders.fragCopy = ShaderBuilder()
.Code(GLSLCompiler()
.Code(CachedGLSLCompiler()
.Type(ShaderType::Fragment)
.AddSource("VersionBlock", prefix)
.AddSource("frag_copy.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_copy.glsl").GetChars())
.OnIncludeLocal(onIncludeLocal)
.OnIncludeSystem(onIncludeSystem)
.Compile(fb->GetDevice()))
.Compile(fb))
.DebugName("VkLightmapper.FragCopy")
.Create("VkLightmapper.FragCopy", fb->GetDevice());
}
@ -628,41 +610,12 @@ int VkLightmapper::GetRaytracePipelineIndex()
FString VkLightmapper::LoadPrivateShaderLump(const char* lumpname)
{
int lump = fileSystem.CheckNumForFullName(lumpname, 0);
if (lump == -1) I_Error("Unable to load '%s'", lumpname);
return GetStringFromLump(lump);
return fb->GetShaderCache()->GetPrivateFile(lumpname).Code;
}
FString VkLightmapper::LoadPublicShaderLump(const char* lumpname)
{
int lump = fileSystem.CheckNumForFullName(lumpname, 0);
if (lump == -1) lump = fileSystem.CheckNumForFullName(lumpname);
if (lump == -1) I_Error("Unable to load '%s'", lumpname);
return GetStringFromLump(lump);
}
ShaderIncludeResult VkLightmapper::OnInclude(FString headerName, FString includerName, size_t depth, bool system)
{
if (depth > 8)
I_Error("Too much include recursion!");
FString includeguardname;
includeguardname << "_HEADERGUARD_" << headerName.GetChars();
includeguardname.ReplaceChars("/\\.", '_');
FString code;
code << "#ifndef " << includeguardname.GetChars() << "\n";
code << "#define " << includeguardname.GetChars() << "\n";
code << "#line 1\n";
if (system)
code << LoadPrivateShaderLump(headerName.GetChars()).GetChars() << "\n";
else
code << LoadPublicShaderLump(headerName.GetChars()).GetChars() << "\n";
code << "#endif\n";
return ShaderIncludeResult(headerName.GetChars(), code.GetChars());
return fb->GetShaderCache()->GetPublicFile(lumpname).Code;
}
void VkLightmapper::CreateRaytracePipeline()

5
src/common/rendering/vulkan/vk_lightmapper.h
View file

@ -135,9 +135,8 @@ private:
int GetRaytracePipelineIndex();
static FString LoadPrivateShaderLump(const char* lumpname);
static FString LoadPublicShaderLump(const char* lumpname);
static ShaderIncludeResult OnInclude(FString headerName, FString includerName, size_t depth, bool system);
FString LoadPrivateShaderLump(const char* lumpname);
FString LoadPublicShaderLump(const char* lumpname);
FVector3 SwapYZ(const FVector3& v) { return FVector3(v.X, v.Z, v.Y); }