fix normals for sprite lighting

src/common/rendering/hwrenderer/data/hw_renderstate.h

@@ -140,6 +140,7 @@ protected:
 	uint8_t mBrightmapEnabled : 1;
 	uint8_t mWireframe : 2;
 	uint8_t mShadeVertex : 1;
+	uint8_t mLightNoNormals : 1;
 
 	FVector4 mWireframeColor;
 	FVector4 uObjectColor;
@@ -184,6 +185,7 @@ public:
 
 	void Reset()
 	{
+		mLightNoNormals = 0;
 		mShadeVertex = 0;
 		mWireframe = 0;
 		mWireframeColor = toFVector4(PalEntry(0xffffffff));
@@ -272,6 +274,11 @@ public:
 		mShadeVertex = value;
 	}
 
+	void SetLightNoNormals(bool value)
+	{
+		mLightNoNormals = value;
+	}
+
 	void SetWireframeColor(FVector4 color)
 	{
 		mWireframeColor = color;

src/common/rendering/vulkan/shaders/vk_shader.cpp

@@ -422,9 +422,9 @@ void VkShaderManager::BuildDefinesBlock(FString &definesBlock, const char *defin
 
 
 	if (key.ShadeVertex) definesBlock << "#define SHADE_VERTEX\n";
+	if (key.LightNoNormals) definesBlock << "#define LIGHT_NONORMALS\n";
 
 	definesBlock << ((key.Simple2D) ? "#define uFogEnabled -3\n" : "#define uFogEnabled 0\n");
-
 }
 
 std::unique_ptr<VulkanShader> VkShaderManager::LoadVertShader(FString shadername, const char *vert_lump, const char *vert_lump_custom, const char *defines, const VkShaderKey& key, const UserShaderDesc *shader)

src/common/rendering/vulkan/shaders/vk_shader.h

@@ -109,7 +109,8 @@ public:
 			uint64_t UseRaytracePrecise : 1; // USE_RAYTRACE_PRECISE
 			uint64_t ShadowmapFilter : 4; // SHADOWMAP_FILTER
 			uint64_t ShadeVertex : 1; // SHADE_VERTEX
-			uint64_t Unused : 29;
+			uint64_t LightNoNormals : 1; // LIGHT_NONORMALS
+			uint64_t Unused : 28;
 		};
 		uint64_t AsQWORD = 0;
 	};

src/common/rendering/vulkan/vk_renderstate.cpp

@@ -330,6 +330,7 @@ void VkRenderState::ApplyRenderPass(int dt)
 	}
 
 	pipelineKey.ShaderKey.ShadeVertex = mShadeVertex;
+	pipelineKey.ShaderKey.LightNoNormals = mLightNoNormals;
 
 	pipelineKey.ShaderKey.UseShadowmap = gl_light_shadows == 1;
 	pipelineKey.ShaderKey.UseRaytrace = gl_light_shadows >= 2;

src/rendering/hwrenderer/scene/hw_spritelight.cpp

@@ -280,7 +280,7 @@ void HWDrawInfo::GetDynSpriteLightList(AActor *self, FDynLightData &modellightda
 
 		ActorTraceStaticLight staticLight(self);
 
-		if (gl_spritelight > 0 || ActorTraceStaticLight::TraceSunVisibility(x, y, z, (self ? &self->StaticLightsTraceCache : nullptr), (self ? staticLight.ActorMoved : false)))
+		if ((level.lightmaps && gl_spritelight > 0) || ActorTraceStaticLight::TraceSunVisibility(x, y, z, (self ? &self->StaticLightsTraceCache : nullptr), (self ? staticLight.ActorMoved : false)))
 		{
 			AddSunLightToList(modellightdata, x, y, z, self->Level->SunDirection, self->Level->SunColor * self->Level->SunIntensity, gl_spritelight > 0);
 		}

src/rendering/hwrenderer/scene/hw_sprites.cpp

@@ -305,6 +305,7 @@ void HWSprite::DrawSprite(HWDrawInfo *di, FRenderState &state, bool translucent)
 				state.SetDepthBias(-1, -128);
 			}
 
+			state.SetLightNoNormals(true);
 			state.SetLightIndex(dynlightindex);
 			state.Draw(DT_TriangleStrip, vertexindex, 4);
 			state.SetLightIndex(-1);
@@ -318,6 +319,7 @@ void HWSprite::DrawSprite(HWDrawInfo *di, FRenderState &state, bool translucent)
 				state.Draw(DT_TriangleStrip, vertexindex, 4);
 				state.SetTextureMode(TM_NORMAL);
 			}
+			state.SetLightNoNormals(false);
 		}
 		else
 		{

wadsrc/static/shaders/scene/light_trace.glsl

@@ -46,10 +46,16 @@ float traceHit(vec3 origin, vec3 direction, float dist)
 
 float traceShadow(vec3 lightpos, float softShadowRadius)
 {
-	vec3 origin = pixelpos.xyz + vWorldNormal.xyz;
 	vec3 target = lightpos.xyz + 0.01; // nudge light position slightly as Doom maps tend to have their lights perfectly aligned with planes
-
+#ifdef LIGHT_NONORMALS
+	vec3 origin = pixelpos.xyz;
 	vec3 direction = normalize(target - origin);
+	origin -= direction;
+#else
+	vec3 origin = pixelpos.xyz + vWorldNormal.xyz;
+	vec3 direction = normalize(target - origin);
+#endif
+
 	float dist = distance(origin, target);
 
 #if SHADOWMAP_FILTER == 0

wadsrc/static/shaders/scene/lightmodel_normal.glsl

@@ -5,10 +5,13 @@
 		
 		if (light.radius < lightdistance)
 			return vec3(0.0); // Early out lights touching surface but not this fragment
-
+		
 		vec3 lightdir = normalize(light.pos.xyz - pixelpos.xyz);
-		float dotprod = dot(normal, lightdir);
-		if (dotprod < -0.0001) return vec3(0.0);	// light hits from the backside. This can happen with full sector light lists and must be rejected for all cases. Note that this can cause precision issues.
+		
+		#ifndef LIGHT_NONORMALS
+			float dotprod = dot(normal, lightdir);
+			if (dotprod < -0.0001) return vec3(0.0);	// light hits from the backside. This can happen with full sector light lists and must be rejected for all cases. Note that this can cause precision issues.
+		#endif
 		
 		float attenuation = distanceAttenuation(lightdistance, light.radius, light.strength, light.linearity);
 
@@ -16,13 +19,15 @@
 		{
 			attenuation *= spotLightAttenuation(light.pos.xyz, light.spotDir.xyz, light.spotInnerAngle, light.spotOuterAngle);
 		}
-
-		if ((light.flags & LIGHTINFO_ATTENUATED) != 0)
-		{
-			attenuation *= clamp(dotprod, 0.0, 1.0);
-		}
 		
-
+		#ifndef LIGHT_NONORMALS
+			if ((light.flags & LIGHTINFO_ATTENUATED) != 0)
+			{
+				attenuation *= clamp(dotprod, 0.0, 1.0);
+			}
+		#endif
+		
+		
 		if (attenuation > 0.0) // Skip shadow map test if possible
 		{
 			// light.radius >= 1000000.0 is sunlight(?), skip attenuation

wadsrc/static/shaders/scene/vert_main.glsl

@@ -14,28 +14,29 @@
 		
 		if (light.radius < lightdistance)
 			return vec3(0.0); // Early out lights touching surface but not this fragment
-
+		
 		vec3 lightdir = normalize(light.pos.xyz - pixelpos.xyz);
-		float dotprod = dot(vWorldNormal.xyz, lightdir);
 		
 		float attenuation = distanceAttenuation(lightdistance, light.radius, light.strength, light.linearity);
-
+		
 		if ((light.flags & LIGHTINFO_SPOT) != 0)
 		{
 			attenuation *= spotLightAttenuation(light.pos.xyz, light.spotDir.xyz, light.spotInnerAngle, light.spotOuterAngle);
 		}
-
-		if ((light.flags & LIGHTINFO_ATTENUATED) != 0)
-		{
-			attenuation *= clamp(dotprod, 0.0, 1.0);
-		}
 		
-
+		#ifndef LIGHT_NONORMALS
+			if ((light.flags & LIGHTINFO_ATTENUATED) != 0)
+			{
+				float dotprod = dot(vWorldNormal.xyz, lightdir);
+				attenuation *= clamp(dotprod, 0.0, 1.0);
+			}
+		#endif
+		
 		if (attenuation > 0.0) // Skip shadow map test if possible
 		{
 			#ifdef USE_RAYTRACE
 				// light.radius >= 1000000.0 is sunlight(?), skip attenuation
-				if(light.radius < 1000000.0 && (light.flags & LIGHTINFO_SHADOWMAPPED) != 0)
+				if((light.flags & LIGHTINFO_SHADOWMAPPED) != 0)
 				{
 					attenuation *= traceShadow(light.pos.xyz, light.softShadowRadius);
 				}
@@ -47,7 +48,7 @@
 		{
 			return vec3(0.0);
 		}
-
+		
 		return vec3(0.0);
 	}