From eb134d3fbf9ebca9967d74f85b55a7f7f8cecb76 Mon Sep 17 00:00:00 2001
From: RaveYard <29225776+MrRaveYard@users.noreply.github.com>
Date: Wed, 20 Sep 2023 22:44:05 +0200
Subject: [PATCH] Add color blending for lights

---
 .../rendering/hwrenderer/data/hw_levelmesh.h  |  1 +
 .../vulkan/accelstructs/vk_raytrace.cpp       |  2 +
 .../vulkan/accelstructs/vk_raytrace.h         |  2 +-
 src/rendering/hwrenderer/doom_levelmesh.cpp   |  1 +
 .../shaders/lightmap/frag_raytrace.glsl       | 48 ++++++++++++++++---
 5 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/src/common/rendering/hwrenderer/data/hw_levelmesh.h b/src/common/rendering/hwrenderer/data/hw_levelmesh.h
index 1d017dcef..0d8152b5f 100644
--- a/src/common/rendering/hwrenderer/data/hw_levelmesh.h
+++ b/src/common/rendering/hwrenderer/data/hw_levelmesh.h
@@ -66,6 +66,7 @@ struct LevelMeshSurface
 	//
 
 	FTextureID texture = FNullTextureID();
+	float alpha = 1.0;
 	
 	int portalIndex = 0;
 	int sectorGroup = 0;
diff --git a/src/common/rendering/vulkan/accelstructs/vk_raytrace.cpp b/src/common/rendering/vulkan/accelstructs/vk_raytrace.cpp
index 6a13c7af5..1a3cfc71e 100644
--- a/src/common/rendering/vulkan/accelstructs/vk_raytrace.cpp
+++ b/src/common/rendering/vulkan/accelstructs/vk_raytrace.cpp
@@ -137,6 +137,8 @@ void VkRaytrace::CreateBuffers()
 			info.TextureIndex = -1;
 		}
 
+		info.Alpha = surface->alpha;
+
 		surfaceInfo.Push(info);
 	}
 
diff --git a/src/common/rendering/vulkan/accelstructs/vk_raytrace.h b/src/common/rendering/vulkan/accelstructs/vk_raytrace.h
index e80f45e2e..b3c13e03e 100644
--- a/src/common/rendering/vulkan/accelstructs/vk_raytrace.h
+++ b/src/common/rendering/vulkan/accelstructs/vk_raytrace.h
@@ -34,7 +34,7 @@ struct SurfaceInfo
 	float SamplingDistance;
 	uint32_t PortalIndex;
 	int32_t TextureIndex;
-	float Padding;
+	float Alpha;
 };
 
 struct SurfaceVertex
diff --git a/src/rendering/hwrenderer/doom_levelmesh.cpp b/src/rendering/hwrenderer/doom_levelmesh.cpp
index cc3d68706..526e9d107 100644
--- a/src/rendering/hwrenderer/doom_levelmesh.cpp
+++ b/src/rendering/hwrenderer/doom_levelmesh.cpp
@@ -845,6 +845,7 @@ void DoomLevelMesh::CreateSideSurfaces(FLevelLocals &doomMap, side_t *side)
 		surf.ControlSector = nullptr;
 		surf.sectorGroup = sectorGroup[front->Index()];
 		surf.texture = texture;
+		surf.alpha = float(side->linedef->alpha);
 
 		Surfaces.Push(surf);
 	}
diff --git a/wadsrc/static/shaders/lightmap/frag_raytrace.glsl b/wadsrc/static/shaders/lightmap/frag_raytrace.glsl
index 3c40414b2..5b095814a 100644
--- a/wadsrc/static/shaders/lightmap/frag_raytrace.glsl
+++ b/wadsrc/static/shaders/lightmap/frag_raytrace.glsl
@@ -48,7 +48,7 @@ struct SurfaceInfo
 	float SamplingDistance;
 	uint PortalIndex;
 	int TextureIndex;
-	float Padding2;
+	float Alpha;
 };
 
 struct PortalInfo
@@ -112,6 +112,7 @@ int TraceFirstHitTriangle(vec3 origin, float tmin, vec3 dir, float tmax);
 int TraceFirstHitTriangleT(vec3 origin, float tmin, vec3 dir, float tmax, out float t);
 
 vec3 primitiveWeights;
+vec4 rayColor;
 
 void main()
 {
@@ -158,9 +159,10 @@ vec3 TraceLight(vec3 origin, vec3 normal, LightInfo light)
 		float attenuation = distAttenuation * angleAttenuation * spotAttenuation;
 		if (attenuation > 0.0)
 		{
+			rayColor = vec4(light.Color.rgb, 1.0);
 			if(TracePoint(origin, light.Origin, minDistance, dir, dist))
 			{
-				incoming.rgb += light.Color * (attenuation * light.Intensity);
+				incoming.rgb += (rayColor.rgb * rayColor.w) * (attenuation * light.Intensity);
 			}
 		}
 	}
@@ -174,11 +176,13 @@ vec3 TraceSunLight(vec3 origin)
 	vec3 incoming = vec3(0.0);
 	const float dist = 32768.0;
 
+	rayColor = vec4(SunColor.rgb * SunIntensity, 1.0);
+
 	int primitiveID = TraceFirstHitTriangle(origin, minDistance, SunDir, dist);
 	if (primitiveID != -1)
 	{
 		SurfaceInfo surface = surfaces[surfaceIndices[primitiveID]];
-		incoming.rgb += SunColor * SunIntensity * surface.Sky;
+		incoming.rgb = rayColor.rgb * rayColor.w * surface.Sky;
 	}
 	return incoming;
 }
@@ -505,6 +509,17 @@ int TraceFirstHitTriangleNoPortal(vec3 origin, float tmin, vec3 dir, float tmax,
 
 #endif
 
+vec4 alphaBlend(vec4 a, vec4 b)
+{
+	float na = a.w + b.w * (1.0 - a.w);
+	return vec4((a.xyz * a.w + b.xyz * b.w * (1.0 - a.w)) / na, na);
+}
+
+vec4 blend(vec4 a, vec4 b)
+{
+	return alphaBlend(a, b);
+}
+
 int TraceFirstHitTriangleT(vec3 origin, float tmin, vec3 dir, float tmax, out float t)
 {
 	int primitiveID;
@@ -529,10 +544,15 @@ int TraceFirstHitTriangleT(vec3 origin, float tmin, vec3 dir, float tmax, out fl
 				break;
 			}
 
-			if (texture(textures[surface.TextureIndex], uv).w > 0.9)
+			vec4 color = texture(textures[surface.TextureIndex], uv);
+			color.w *= surface.Alpha;
+
+			if (color.w > 0.999 || rayColor.rgb == vec3(0))
 			{
 				break;
 			}
+
+			rayColor = blend(color, rayColor) * (1.0 - color.w);
 		}
 
 		// Portal was hit: Apply transformation onto the ray
@@ -576,9 +596,25 @@ bool TracePoint(vec3 origin, vec3 target, float tmin, vec3 dir, float tmax)
 
 		SurfaceInfo surface = surfaces[surfaceIndices[primitiveID]];
 
-		if(surface.PortalIndex == 0)
+		if (surface.PortalIndex == 0)
 		{
-			break;
+			int index = primitiveID * 3;
+			vec2 uv = vertices[elements[index + 0]].uv * primitiveWeights.x + vertices[elements[index + 1]].uv * primitiveWeights.y + vertices[elements[index + 2]].uv * primitiveWeights.z;
+
+			if (surface.TextureIndex < 0)
+			{
+				break;
+			}
+
+			vec4 color = texture(textures[surface.TextureIndex], uv);
+			color.w *= surface.Alpha;
+
+			if (color.w > 0.999 || rayColor.rgb == vec3(0))
+			{
+				break;
+			}
+
+			rayColor = blend(color, rayColor) * (1.0 - color.w);
 		}
 
 		if(dot(surface.Normal, dir) >= 0.0)