From 41a345d3992ac13cded768d2563014e4bf3aa2f5 Mon Sep 17 00:00:00 2001 From: Magnus Norddahl Date: Tue, 25 Feb 2025 00:15:22 +0100 Subject: [PATCH] Update shader builder code and add initial work on a shader caching class --- src/CMakeLists.txt | 1 + src/common/rendering/vulkan/shaders/sha1.h | 308 ++++++++++++++++++ .../rendering/vulkan/shaders/vk_ppshader.cpp | 20 +- .../rendering/vulkan/shaders/vk_shader.cpp | 106 +++--- .../rendering/vulkan/shaders/vk_shader.h | 16 +- .../vulkan/shaders/vk_shadercache.cpp | 63 ++++ .../rendering/vulkan/shaders/vk_shadercache.h | 33 ++ src/common/rendering/vulkan/vk_levelmesh.cpp | 24 +- .../rendering/vulkan/vk_lightmapper.cpp | 96 +++--- .../rendering/vulkan/vk_lightprober.cpp | 12 +- 10 files changed, 561 insertions(+), 118 deletions(-) create mode 100644 src/common/rendering/vulkan/shaders/sha1.h create mode 100644 src/common/rendering/vulkan/shaders/vk_shadercache.cpp create mode 100644 src/common/rendering/vulkan/shaders/vk_shadercache.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b4aa5b2b3..1b2ac3af4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -752,6 +752,7 @@ set (VULKAN_SOURCES common/rendering/vulkan/pipelines/vk_pprenderpass.cpp common/rendering/vulkan/shaders/vk_shader.cpp common/rendering/vulkan/shaders/vk_ppshader.cpp + common/rendering/vulkan/shaders/vk_shadercache.cpp common/rendering/vulkan/textures/vk_hwtexture.cpp common/rendering/vulkan/textures/vk_pptexture.cpp common/rendering/vulkan/textures/vk_imagetransition.cpp diff --git a/src/common/rendering/vulkan/shaders/sha1.h b/src/common/rendering/vulkan/shaders/sha1.h new file mode 100644 index 000000000..0948665d8 --- /dev/null +++ b/src/common/rendering/vulkan/shaders/sha1.h @@ -0,0 +1,308 @@ + +#pragma once + +/* + sha1.hpp - source code of + + ============ + SHA-1 in C++ + ============ + + 100% Public Domain. + + Original C Code + -- Steve Reid + Small changes to fit into bglibs + -- Bruce Guenter + Translation to simpler C++ Code + -- Volker Diels-Grabsch + Safety fixes + -- Eugene Hopkinson + Header-only library + -- Zlatko Michailov +*/ + +#include +#include +#include +#include + +class SHA1 +{ +public: + SHA1(); + void update(const std::string& s); + void update(std::istream& is); + std::string final(); + +private: + uint32_t digest[5]; + std::string buffer; + uint64_t transforms; + + enum { BLOCK_INTS = 16 }; /* number of 32bit integers per SHA1 block */ + enum { BLOCK_BYTES = BLOCK_INTS * 4 }; + + inline static void reset(uint32_t digest[], std::string& buffer, uint64_t& transforms) + { + /* SHA1 initialization constants */ + digest[0] = 0x67452301; + digest[1] = 0xefcdab89; + digest[2] = 0x98badcfe; + digest[3] = 0x10325476; + digest[4] = 0xc3d2e1f0; + + /* Reset counters */ + buffer = ""; + transforms = 0; + } + + + inline static uint32_t rol(const uint32_t value, const size_t bits) + { + return (value << bits) | (value >> (32 - bits)); + } + + + inline static uint32_t blk(const uint32_t block[BLOCK_INTS], const size_t i) + { + return rol(block[(i + 13) & 15] ^ block[(i + 8) & 15] ^ block[(i + 2) & 15] ^ block[i], 1); + } + + + /* + * (R0+R1), R2, R3, R4 are the different operations used in SHA1 + */ + + inline static void R0(const uint32_t block[BLOCK_INTS], const uint32_t v, uint32_t& w, const uint32_t x, const uint32_t y, uint32_t& z, const size_t i) + { + z += ((w & (x ^ y)) ^ y) + block[i] + 0x5a827999 + rol(v, 5); + w = rol(w, 30); + } + + + inline static void R1(uint32_t block[BLOCK_INTS], const uint32_t v, uint32_t& w, const uint32_t x, const uint32_t y, uint32_t& z, const size_t i) + { + block[i] = blk(block, i); + z += ((w & (x ^ y)) ^ y) + block[i] + 0x5a827999 + rol(v, 5); + w = rol(w, 30); + } + + + inline static void R2(uint32_t block[BLOCK_INTS], const uint32_t v, uint32_t& w, const uint32_t x, const uint32_t y, uint32_t& z, const size_t i) + { + block[i] = blk(block, i); + z += (w ^ x ^ y) + block[i] + 0x6ed9eba1 + rol(v, 5); + w = rol(w, 30); + } + + + inline static void R3(uint32_t block[BLOCK_INTS], const uint32_t v, uint32_t& w, const uint32_t x, const uint32_t y, uint32_t& z, const size_t i) + { + block[i] = blk(block, i); + z += (((w | x) & y) | (w & x)) + block[i] + 0x8f1bbcdc + rol(v, 5); + w = rol(w, 30); + } + + + inline static void R4(uint32_t block[BLOCK_INTS], const uint32_t v, uint32_t& w, const uint32_t x, const uint32_t y, uint32_t& z, const size_t i) + { + block[i] = blk(block, i); + z += (w ^ x ^ y) + block[i] + 0xca62c1d6 + rol(v, 5); + w = rol(w, 30); + } + + + /* + * Hash a single 512-bit block. This is the core of the algorithm. + */ + + inline static void transform(uint32_t digest[], uint32_t block[BLOCK_INTS], uint64_t& transforms) + { + /* Copy digest[] to working vars */ + uint32_t a = digest[0]; + uint32_t b = digest[1]; + uint32_t c = digest[2]; + uint32_t d = digest[3]; + uint32_t e = digest[4]; + + /* 4 rounds of 20 operations each. Loop unrolled. */ + R0(block, a, b, c, d, e, 0); + R0(block, e, a, b, c, d, 1); + R0(block, d, e, a, b, c, 2); + R0(block, c, d, e, a, b, 3); + R0(block, b, c, d, e, a, 4); + R0(block, a, b, c, d, e, 5); + R0(block, e, a, b, c, d, 6); + R0(block, d, e, a, b, c, 7); + R0(block, c, d, e, a, b, 8); + R0(block, b, c, d, e, a, 9); + R0(block, a, b, c, d, e, 10); + R0(block, e, a, b, c, d, 11); + R0(block, d, e, a, b, c, 12); + R0(block, c, d, e, a, b, 13); + R0(block, b, c, d, e, a, 14); + R0(block, a, b, c, d, e, 15); + R1(block, e, a, b, c, d, 0); + R1(block, d, e, a, b, c, 1); + R1(block, c, d, e, a, b, 2); + R1(block, b, c, d, e, a, 3); + R2(block, a, b, c, d, e, 4); + R2(block, e, a, b, c, d, 5); + R2(block, d, e, a, b, c, 6); + R2(block, c, d, e, a, b, 7); + R2(block, b, c, d, e, a, 8); + R2(block, a, b, c, d, e, 9); + R2(block, e, a, b, c, d, 10); + R2(block, d, e, a, b, c, 11); + R2(block, c, d, e, a, b, 12); + R2(block, b, c, d, e, a, 13); + R2(block, a, b, c, d, e, 14); + R2(block, e, a, b, c, d, 15); + R2(block, d, e, a, b, c, 0); + R2(block, c, d, e, a, b, 1); + R2(block, b, c, d, e, a, 2); + R2(block, a, b, c, d, e, 3); + R2(block, e, a, b, c, d, 4); + R2(block, d, e, a, b, c, 5); + R2(block, c, d, e, a, b, 6); + R2(block, b, c, d, e, a, 7); + R3(block, a, b, c, d, e, 8); + R3(block, e, a, b, c, d, 9); + R3(block, d, e, a, b, c, 10); + R3(block, c, d, e, a, b, 11); + R3(block, b, c, d, e, a, 12); + R3(block, a, b, c, d, e, 13); + R3(block, e, a, b, c, d, 14); + R3(block, d, e, a, b, c, 15); + R3(block, c, d, e, a, b, 0); + R3(block, b, c, d, e, a, 1); + R3(block, a, b, c, d, e, 2); + R3(block, e, a, b, c, d, 3); + R3(block, d, e, a, b, c, 4); + R3(block, c, d, e, a, b, 5); + R3(block, b, c, d, e, a, 6); + R3(block, a, b, c, d, e, 7); + R3(block, e, a, b, c, d, 8); + R3(block, d, e, a, b, c, 9); + R3(block, c, d, e, a, b, 10); + R3(block, b, c, d, e, a, 11); + R4(block, a, b, c, d, e, 12); + R4(block, e, a, b, c, d, 13); + R4(block, d, e, a, b, c, 14); + R4(block, c, d, e, a, b, 15); + R4(block, b, c, d, e, a, 0); + R4(block, a, b, c, d, e, 1); + R4(block, e, a, b, c, d, 2); + R4(block, d, e, a, b, c, 3); + R4(block, c, d, e, a, b, 4); + R4(block, b, c, d, e, a, 5); + R4(block, a, b, c, d, e, 6); + R4(block, e, a, b, c, d, 7); + R4(block, d, e, a, b, c, 8); + R4(block, c, d, e, a, b, 9); + R4(block, b, c, d, e, a, 10); + R4(block, a, b, c, d, e, 11); + R4(block, e, a, b, c, d, 12); + R4(block, d, e, a, b, c, 13); + R4(block, c, d, e, a, b, 14); + R4(block, b, c, d, e, a, 15); + + /* Add the working vars back into digest[] */ + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + + /* Count the number of transformations */ + transforms++; + } + + + inline static void buffer_to_block(const std::string& buffer, uint32_t block[BLOCK_INTS]) + { + /* Convert the std::string (byte buffer) to a uint32_t array (MSB) */ + for (size_t i = 0; i < BLOCK_INTS; i++) + { + block[i] = (buffer[4 * i + 3] & 0xff) + | (buffer[4 * i + 2] & 0xff) << 8 + | (buffer[4 * i + 1] & 0xff) << 16 + | (buffer[4 * i + 0] & 0xff) << 24; + } + } +}; + +inline SHA1::SHA1() +{ + reset(digest, buffer, transforms); +} + +inline void SHA1::update(const std::string &s) +{ + std::istringstream is(s); + update(is); +} + +inline void SHA1::update(std::istream &is) +{ + while (true) + { + char sbuf[BLOCK_BYTES]; + is.read(sbuf, BLOCK_BYTES - buffer.size()); + buffer.append(sbuf, (std::size_t)is.gcount()); + if (buffer.size() != BLOCK_BYTES) + { + return; + } + uint32_t block[BLOCK_INTS]; + buffer_to_block(buffer, block); + transform(digest, block, transforms); + buffer.clear(); + } +} + +// Add padding and return the message digest. +inline std::string SHA1::final() +{ + /* Total number of hashed bits */ + uint64_t total_bits = (transforms*BLOCK_BYTES + buffer.size()) * 8; + + /* Padding */ + buffer += (char)0x80; + size_t orig_size = buffer.size(); + while (buffer.size() < BLOCK_BYTES) + { + buffer += (char)0x00; + } + + uint32_t block[BLOCK_INTS]; + buffer_to_block(buffer, block); + + if (orig_size > BLOCK_BYTES - 8) + { + transform(digest, block, transforms); + for (size_t i = 0; i < BLOCK_INTS - 2; i++) + { + block[i] = 0; + } + } + + /* Append total_bits, split this uint64_t into two uint32_t */ + block[BLOCK_INTS - 1] = (uint32_t)total_bits; + block[BLOCK_INTS - 2] = (uint32_t)(total_bits >> 32); + transform(digest, block, transforms); + + /* Hex std::string */ + std::ostringstream result; + for (size_t i = 0; i < sizeof(digest) / sizeof(digest[0]); i++) + { + result << std::hex << std::setfill('0') << std::setw(8); + result << digest[i]; + } + + /* Reset for next run */ + reset(digest, buffer, transforms); + + return result.str(); +} diff --git a/src/common/rendering/vulkan/shaders/vk_ppshader.cpp b/src/common/rendering/vulkan/shaders/vk_ppshader.cpp index 3fa7cbd90..ec2107782 100644 --- a/src/common/rendering/vulkan/shaders/vk_ppshader.cpp +++ b/src/common/rendering/vulkan/shaders/vk_ppshader.cpp @@ -36,18 +36,22 @@ VkPPShader::VkPPShader(VulkanRenderDevice* fb, PPShader *shader) : fb(fb) prolog += shader->Defines; VertexShader = ShaderBuilder() - .Type(ShaderType::Vertex) - .AddSource(shader->VertexShader.GetChars(), LoadShaderCode(shader->VertexShader, "", shader->Version).GetChars()) - .OnIncludeLocal([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); }) - .OnIncludeSystem([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }) + .Code(GLSLCompiler() + .Type(ShaderType::Vertex) + .AddSource(shader->VertexShader.GetChars(), LoadShaderCode(shader->VertexShader, "", shader->Version).GetChars()) + .OnIncludeLocal([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); }) + .OnIncludeSystem([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }) + .Compile(fb->GetDevice())) .DebugName(shader->VertexShader.GetChars()) .Create(shader->VertexShader.GetChars(), fb->GetDevice()); FragmentShader = ShaderBuilder() - .Type(ShaderType::Fragment) - .AddSource(shader->FragmentShader.GetChars(), LoadShaderCode(shader->FragmentShader, prolog, shader->Version).GetChars()) - .OnIncludeLocal([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); }) - .OnIncludeSystem([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource(shader->FragmentShader.GetChars(), LoadShaderCode(shader->FragmentShader, prolog, shader->Version).GetChars()) + .OnIncludeLocal([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); }) + .OnIncludeSystem([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }) + .Compile(fb->GetDevice())) .DebugName(shader->FragmentShader.GetChars()) .Create(shader->FragmentShader.GetChars(), fb->GetDevice()); diff --git a/src/common/rendering/vulkan/shaders/vk_shader.cpp b/src/common/rendering/vulkan/shaders/vk_shader.cpp index ccf77b21e..e1678629c 100644 --- a/src/common/rendering/vulkan/shaders/vk_shader.cpp +++ b/src/common/rendering/vulkan/shaders/vk_shader.cpp @@ -22,6 +22,7 @@ #include "vk_shader.h" #include "vk_ppshader.h" +#include "vk_shadercache.h" #include "vulkan/vk_renderdevice.h" #include "vulkan/pipelines/vk_renderpass.h" #include @@ -33,40 +34,52 @@ VkShaderManager::VkShaderManager(VulkanRenderDevice* fb) : fb(fb) { + ShaderCache = std::make_unique(fb); + ZMinMax.vert = ShaderBuilder() - .Type(ShaderType::Vertex) + .Code(GLSLCompiler() + .Type(ShaderType::Vertex) + .AddSource("VersionBlock", GetVersionBlock().GetChars()) + .AddSource("shaders/scene/vert_zminmax.glsl", LoadPrivateShaderLump("shaders/scene/vert_zminmax.glsl").GetChars()) + .Compile(fb->GetDevice())) .DebugName("ZMinMax.vert") - .AddSource("VersionBlock", GetVersionBlock().GetChars()) - .AddSource("shaders/scene/vert_zminmax.glsl", LoadPrivateShaderLump("shaders/scene/vert_zminmax.glsl").GetChars()) .Create("ZMinMax.vert", fb->GetDevice()); ZMinMax.frag[0] = ShaderBuilder() - .Type(ShaderType::Fragment) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("VersionBlock", GetVersionBlock().GetChars()) + .AddSource("shaders/scene/frag_zminmax0.glsl", LoadPrivateShaderLump("shaders/scene/frag_zminmax0.glsl").GetChars()) + .Compile(fb->GetDevice())) .DebugName("ZMinMax0.frag") - .AddSource("VersionBlock", GetVersionBlock().GetChars()) - .AddSource("shaders/scene/frag_zminmax0.glsl", LoadPrivateShaderLump("shaders/scene/frag_zminmax0.glsl").GetChars()) .Create("ZMinMax0.frag", fb->GetDevice()); ZMinMax.frag[1] = ShaderBuilder() - .Type(ShaderType::Fragment) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("VersionBlock", GetVersionBlock().GetChars()) + .AddSource("DefinesBlock", "#define MULTISAMPLE\n") + .AddSource("shaders/scene/frag_zminmax0.glsl", LoadPrivateShaderLump("shaders/scene/frag_zminmax0.glsl").GetChars()) + .Compile(fb->GetDevice())) .DebugName("ZMinMax0.frag") - .AddSource("VersionBlock", GetVersionBlock().GetChars()) - .AddSource("DefinesBlock", "#define MULTISAMPLE\n") - .AddSource("shaders/scene/frag_zminmax0.glsl", LoadPrivateShaderLump("shaders/scene/frag_zminmax0.glsl").GetChars()) .Create("ZMinMax0.frag", fb->GetDevice()); ZMinMax.frag[2] = ShaderBuilder() - .Type(ShaderType::Fragment) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("VersionBlock", GetVersionBlock().GetChars()) + .AddSource("shaders/scene/frag_zminmax1.glsl", LoadPrivateShaderLump("shaders/scene/frag_zminmax1.glsl").GetChars()) + .Compile(fb->GetDevice())) .DebugName("ZMinMax1.frag") - .AddSource("VersionBlock", GetVersionBlock().GetChars()) - .AddSource("shaders/scene/frag_zminmax1.glsl", LoadPrivateShaderLump("shaders/scene/frag_zminmax1.glsl").GetChars()) .Create("ZMinMax1.frag", fb->GetDevice()); LightTiles = ShaderBuilder() - .Type(ShaderType::Compute) + .Code(GLSLCompiler() + .Type(ShaderType::Compute) + .AddSource("VersionBlock", GetVersionBlock().GetChars()) + .AddSource("shaders/scene/comp_lighttiles.glsl", LoadPrivateShaderLump("shaders/scene/comp_lighttiles.glsl").GetChars()) + .Compile(fb->GetDevice())) .DebugName("LightTiles.comp") - .AddSource("VersionBlock", GetVersionBlock().GetChars()) - .AddSource("shaders/scene/comp_lighttiles.glsl", LoadPrivateShaderLump("shaders/scene/comp_lighttiles.glsl").GetChars()) .Create("LightTiles.comp", fb->GetDevice()); } @@ -569,15 +582,17 @@ std::unique_ptr VkShaderManager::LoadVertShader(FString shadername } return ShaderBuilder() - .Type(ShaderType::Vertex) + .Code(GLSLCompiler() + .Type(ShaderType::Vertex) + .AddSource("VersionBlock", GetVersionBlock().GetChars()) + .AddSource("DefinesBlock", definesBlock.GetChars()) + .AddSource("LayoutBlock", layoutBlock.GetChars()) + .AddSource("shaders/scene/layout_shared.glsl", LoadPrivateShaderLump("shaders/scene/layout_shared.glsl").GetChars()) + .AddSource(vert_lump_custom ? vert_lump_custom : vert_lump, codeBlock.GetChars()) + .OnIncludeLocal([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); }) + .OnIncludeSystem([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }) + .Compile(fb->GetDevice())) .DebugName(shadername.GetChars()) - .AddSource("VersionBlock", GetVersionBlock().GetChars()) - .AddSource("DefinesBlock", definesBlock.GetChars()) - .AddSource("LayoutBlock", layoutBlock.GetChars()) - .AddSource("shaders/scene/layout_shared.glsl", LoadPrivateShaderLump("shaders/scene/layout_shared.glsl").GetChars()) - .AddSource(vert_lump_custom ? vert_lump_custom : vert_lump, codeBlock.GetChars()) - .OnIncludeLocal([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); }) - .OnIncludeSystem([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }) .Create(shadername.GetChars(), fb->GetDevice()); } @@ -668,19 +683,21 @@ std::unique_ptr VkShaderManager::LoadFragShader(FString shadername } return ShaderBuilder() - .Type(ShaderType::Fragment) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("VersionBlock", GetVersionBlock().GetChars()) + .AddSource("DefinesBlock", definesBlock.GetChars()) + .AddSource("LayoutBlock", layoutBlock.GetChars()) + .AddSource("shaders/scene/layout_shared.glsl", LoadPrivateShaderLump("shaders/scene/layout_shared.glsl").GetChars()) + .AddSource("shaders/scene/includes.glsl", LoadPrivateShaderLump("shaders/scene/includes.glsl").GetChars()) + .AddSource(mateffectname.GetChars(), mateffectBlock.GetChars()) + .AddSource(materialname.GetChars(), materialBlock.GetChars()) + .AddSource(lightname.GetChars(), lightBlock.GetChars()) + .AddSource(frag_lump, codeBlock.GetChars()) + .OnIncludeLocal([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); }) + .OnIncludeSystem([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }) + .Compile(fb->GetDevice())) .DebugName(shadername.GetChars()) - .AddSource("VersionBlock", GetVersionBlock().GetChars()) - .AddSource("DefinesBlock", definesBlock.GetChars()) - .AddSource("LayoutBlock", layoutBlock.GetChars()) - .AddSource("shaders/scene/layout_shared.glsl", LoadPrivateShaderLump("shaders/scene/layout_shared.glsl").GetChars()) - .AddSource("shaders/scene/includes.glsl", LoadPrivateShaderLump("shaders/scene/includes.glsl").GetChars()) - .AddSource(mateffectname.GetChars(), mateffectBlock.GetChars()) - .AddSource(materialname.GetChars(), materialBlock.GetChars()) - .AddSource(lightname.GetChars(), lightBlock.GetChars()) - .AddSource(frag_lump, codeBlock.GetChars()) - .OnIncludeLocal([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, false); }) - .OnIncludeSystem([=](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }) .Create(shadername.GetChars(), fb->GetDevice()); } @@ -732,28 +749,24 @@ ShaderIncludeResult VkShaderManager::OnInclude(FString headerName, FString inclu return ShaderIncludeResult(headerName.GetChars(), code.GetChars()); } -FString VkShaderManager::LoadPublicShaderLump(const char *lumpname, bool isUberShader) +FString VkShaderManager::LoadPublicShaderLump(const char* lumpname) { int lump = fileSystem.CheckNumForFullName(lumpname, 0); if (lump == -1) lump = fileSystem.CheckNumForFullName(lumpname); if (lump == -1) I_Error("Unable to load '%s'", lumpname); - - return LoadShaderLump(lump, isUberShader); + return GetStringFromLump(lump); } -FString VkShaderManager::LoadPrivateShaderLump(const char *lumpname, bool isUberShader) +FString VkShaderManager::LoadPrivateShaderLump(const char* lumpname) { int lump = fileSystem.CheckNumForFullName(lumpname, 0); if (lump == -1) I_Error("Unable to load '%s'", lumpname); - - return LoadShaderLump(lump, isUberShader); + return GetStringFromLump(lump); } -FString VkShaderManager::LoadShaderLump(int lumpnum, bool isUberShader) +FString VkShaderManager::SubstituteDefines(FString str, bool isUberShader) { - FString str = GetStringFromLump(lumpnum); - - if(isUberShader) + if (isUberShader) { str.Substitute("#uifdef", "if"); str.Substitute("#uelifdef", "else if"); @@ -771,7 +784,6 @@ FString VkShaderManager::LoadShaderLump(int lumpnum, bool isUberShader) str.Substitute("#uelse", "#else"); str.Substitute("#uendif", "#endif"); } - return str; } diff --git a/src/common/rendering/vulkan/shaders/vk_shader.h b/src/common/rendering/vulkan/shaders/vk_shader.h index 900616a72..d9fe61aea 100644 --- a/src/common/rendering/vulkan/shaders/vk_shader.h +++ b/src/common/rendering/vulkan/shaders/vk_shader.h @@ -2,22 +2,20 @@ #pragma once #include - +#include +#include #include "vectors.h" #include "matrix.h" #include "name.h" #include "hw_renderstate.h" -#include -#include - #include "hw_dynlightdata.h" - #include "hwrenderer/postprocessing/hw_useruniforms.h" class ShaderIncludeResult; class VulkanRenderDevice; class VulkanDevice; class VulkanShader; +class VkShaderCache; class VkPPShader; class PPShader; @@ -167,16 +165,18 @@ private: ShaderIncludeResult OnInclude(FString headerName, FString includerName, size_t depth, bool system); FString GetVersionBlock(); - FString LoadPublicShaderLump(const char *lumpname, bool isUberShader = false); - FString LoadPrivateShaderLump(const char *lumpname, bool isUberShader = false); + FString LoadPublicShaderLump(const char *lumpname); + FString LoadPrivateShaderLump(const char *lumpname); - FString LoadShaderLump(int lumpnum, bool isUberShader); + static FString SubstituteDefines(FString code, bool isUberShader); void BuildLayoutBlock(FString &definesBlock, bool isFrag, const VkShaderKey& key, const UserShaderDesc *shader, bool isUberShader = false); void BuildDefinesBlock(FString &definesBlock, const char *defines, bool isFrag, const VkShaderKey& key, const UserShaderDesc *shader, bool isUberShader = false); VulkanRenderDevice* fb = nullptr; + std::unique_ptr ShaderCache; + std::map programs; std::list PPShaders; diff --git a/src/common/rendering/vulkan/shaders/vk_shadercache.cpp b/src/common/rendering/vulkan/shaders/vk_shadercache.cpp new file mode 100644 index 000000000..eef1c6f0e --- /dev/null +++ b/src/common/rendering/vulkan/shaders/vk_shadercache.cpp @@ -0,0 +1,63 @@ + +#include "vk_shadercache.h" +#include "sha1.h" +#include "filesystem.h" +#include "engineerrors.h" +#include "cmdlib.h" +#include + +VkShaderCache::VkShaderCache(VulkanRenderDevice* fb) : fb(fb) +{ +} + +const VkShaderSourceFile& VkShaderCache::GetPublicFile(const FString& lumpname) +{ + auto it = PublicFiles.find(lumpname); + if (it != PublicFiles.end()) + return it->second; + + FString code = LoadPublicShaderLump(lumpname.GetChars()); + FString sha1 = CalcSha1(code); + + VkShaderSourceFile& file = PublicFiles[lumpname]; + file.Sourcecode = std::move(code); + file.Sha1 = std::move(sha1); + return file; +} + +const VkShaderSourceFile& VkShaderCache::GetPrivateFile(const FString& lumpname) +{ + auto it = PrivateFiles.find(lumpname); + if (it != PrivateFiles.end()) + return it->second; + + FString code = LoadPrivateShaderLump(lumpname.GetChars()); + FString sha1 = CalcSha1(code); + + VkShaderSourceFile& file = PrivateFiles[lumpname]; + file.Sourcecode = std::move(code); + file.Sha1 = std::move(sha1); + return file; +} + +FString VkShaderCache::LoadPublicShaderLump(const char* lumpname) +{ + int lump = fileSystem.CheckNumForFullName(lumpname, 0); + if (lump == -1) lump = fileSystem.CheckNumForFullName(lumpname); + if (lump == -1) I_Error("Unable to load '%s'", lumpname); + return GetStringFromLump(lump); +} + +FString VkShaderCache::LoadPrivateShaderLump(const char* lumpname) +{ + int lump = fileSystem.CheckNumForFullName(lumpname, 0); + if (lump == -1) I_Error("Unable to load '%s'", lumpname); + return GetStringFromLump(lump); +} + +FString VkShaderCache::CalcSha1(const FString& str) +{ + SHA1 sha1; + sha1.update(str.GetChars()); + return sha1.final(); +} diff --git a/src/common/rendering/vulkan/shaders/vk_shadercache.h b/src/common/rendering/vulkan/shaders/vk_shadercache.h new file mode 100644 index 000000000..50f7acfaf --- /dev/null +++ b/src/common/rendering/vulkan/shaders/vk_shadercache.h @@ -0,0 +1,33 @@ +#pragma once + +#include "common/utility/zstring.h" +#include +#include + +class VulkanRenderDevice; + +class VkShaderSourceFile +{ +public: + FString Sha1; + FString Sourcecode; +}; + +class VkShaderCache +{ +public: + VkShaderCache(VulkanRenderDevice* fb); + +private: + const VkShaderSourceFile& GetPublicFile(const FString& lumpname); + const VkShaderSourceFile& GetPrivateFile(const FString& lumpname); + + static FString LoadPublicShaderLump(const char* lumpname); + static FString LoadPrivateShaderLump(const char* lumpname); + static FString CalcSha1(const FString& str); + + VulkanRenderDevice* fb; + + std::map PublicFiles; + std::map PrivateFiles; +}; diff --git a/src/common/rendering/vulkan/vk_levelmesh.cpp b/src/common/rendering/vulkan/vk_levelmesh.cpp index 566b6d94f..223f4b4d2 100644 --- a/src/common/rendering/vulkan/vk_levelmesh.cpp +++ b/src/common/rendering/vulkan/vk_levelmesh.cpp @@ -607,20 +607,24 @@ void VkLevelMesh::CreateViewerObjects() auto onIncludeSystem = [](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }; Viewer.VertexShader = ShaderBuilder() - .Type(ShaderType::Vertex) - .AddSource("versionblock", versionBlock) - .AddSource("vert_viewer.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_viewer.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Vertex) + .AddSource("versionblock", versionBlock) + .AddSource("vert_viewer.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_viewer.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("Viewer.VertexShader") .Create("vertex", fb->GetDevice()); Viewer.FragmentShader = ShaderBuilder() - .Type(ShaderType::Fragment) - .AddSource("versionblock", versionBlock) - .AddSource("frag_viewer.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_viewer.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("versionblock", versionBlock) + .AddSource("frag_viewer.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_viewer.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("Viewer.FragmentShader") .Create("vertex", fb->GetDevice()); diff --git a/src/common/rendering/vulkan/vk_lightmapper.cpp b/src/common/rendering/vulkan/vk_lightmapper.cpp index 7e5fc9994..53d20ac8a 100644 --- a/src/common/rendering/vulkan/vk_lightmapper.cpp +++ b/src/common/rendering/vulkan/vk_lightmapper.cpp @@ -505,29 +505,35 @@ void VkLightmapper::CreateShaders() auto onIncludeSystem = [](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }; shaders.vertRaytrace = ShaderBuilder() - .Type(ShaderType::Vertex) - .AddSource("VersionBlock", prefix) - .AddSource("vert_raytrace.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_raytrace.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Vertex) + .AddSource("VersionBlock", prefix) + .AddSource("vert_raytrace.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_raytrace.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("VkLightmapper.VertRaytrace") .Create("VkLightmapper.VertRaytrace", fb->GetDevice()); shaders.vertScreenquad = ShaderBuilder() - .Type(ShaderType::Vertex) - .AddSource("VersionBlock", prefix) - .AddSource("vert_screenquad.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_screenquad.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Vertex) + .AddSource("VersionBlock", prefix) + .AddSource("vert_screenquad.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_screenquad.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("VkLightmapper.VertScreenquad") .Create("VkLightmapper.VertScreenquad", fb->GetDevice()); shaders.vertCopy = ShaderBuilder() - .Type(ShaderType::Vertex) - .AddSource("VersionBlock", prefix) - .AddSource("vert_copy.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_copy.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Vertex) + .AddSource("VersionBlock", prefix) + .AddSource("vert_copy.glsl", LoadPrivateShaderLump("shaders/lightmap/vert_copy.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("VkLightmapper.VertCopy") .Create("VkLightmapper.VertCopy", fb->GetDevice()); @@ -544,48 +550,58 @@ void VkLightmapper::CreateShaders() defines += "#define USE_BOUNCE\n"; shaders.fragRaytrace[i] = ShaderBuilder() - .Type(ShaderType::Fragment) - .AddSource("VersionBlock", defines) - .AddSource("frag_raytrace.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_raytrace.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("VersionBlock", defines) + .AddSource("frag_raytrace.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_raytrace.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("VkLightmapper.FragRaytrace") .Create("VkLightmapper.FragRaytrace", fb->GetDevice()); } shaders.fragResolve = ShaderBuilder() - .Type(ShaderType::Fragment) - .AddSource("VersionBlock", prefix) - .AddSource("frag_resolve.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_resolve.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("VersionBlock", prefix) + .AddSource("frag_resolve.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_resolve.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("VkLightmapper.FragResolve") .Create("VkLightmapper.FragResolve", fb->GetDevice()); shaders.fragBlur[0] = ShaderBuilder() - .Type(ShaderType::Fragment) - .AddSource("VersionBlock", prefix + "#define BLUR_HORIZONTAL\r\n") - .AddSource("frag_blur.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_blur.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("VersionBlock", prefix + "#define BLUR_HORIZONTAL\r\n") + .AddSource("frag_blur.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_blur.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("VkLightmapper.FragBlur") .Create("VkLightmapper.FragBlur", fb->GetDevice()); shaders.fragBlur[1] = ShaderBuilder() - .Type(ShaderType::Fragment) - .AddSource("VersionBlock", prefix + "#define BLUR_VERTICAL\r\n") - .AddSource("frag_blur.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_blur.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("VersionBlock", prefix + "#define BLUR_VERTICAL\r\n") + .AddSource("frag_blur.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_blur.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("VkLightmapper.FragBlur") .Create("VkLightmapper.FragBlur", fb->GetDevice()); shaders.fragCopy = ShaderBuilder() - .Type(ShaderType::Fragment) - .AddSource("VersionBlock", prefix) - .AddSource("frag_copy.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_copy.glsl").GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Fragment) + .AddSource("VersionBlock", prefix) + .AddSource("frag_copy.glsl", LoadPrivateShaderLump("shaders/lightmap/frag_copy.glsl").GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName("VkLightmapper.FragCopy") .Create("VkLightmapper.FragCopy", fb->GetDevice()); } diff --git a/src/common/rendering/vulkan/vk_lightprober.cpp b/src/common/rendering/vulkan/vk_lightprober.cpp index cae04b0c0..b142ba1af 100644 --- a/src/common/rendering/vulkan/vk_lightprober.cpp +++ b/src/common/rendering/vulkan/vk_lightprober.cpp @@ -519,11 +519,13 @@ std::unique_ptr VkLightprober::CompileShader(const std::string& na auto onIncludeSystem = [](std::string headerName, std::string includerName, size_t depth) { return OnInclude(headerName.c_str(), includerName.c_str(), depth, true); }; return ShaderBuilder() - .Type(ShaderType::Compute) - .AddSource("VersionBlock", prefix) - .AddSource(name, LoadPrivateShaderLump(filename.c_str()).GetChars()) - .OnIncludeLocal(onIncludeLocal) - .OnIncludeSystem(onIncludeSystem) + .Code(GLSLCompiler() + .Type(ShaderType::Compute) + .AddSource("VersionBlock", prefix) + .AddSource(name, LoadPrivateShaderLump(filename.c_str()).GetChars()) + .OnIncludeLocal(onIncludeLocal) + .OnIncludeSystem(onIncludeSystem) + .Compile(fb->GetDevice())) .DebugName(debugName) .Create(debugName, fb->GetDevice()); }