Fix AO not appearing and some other tracing adjustments

wadsrc/static/shaders/lightmap/frag_raytrace.glsl

@@ -26,7 +26,7 @@ void main()
 #endif
 
 	vec3 normal = surfaces[SurfaceIndex].Normal;
-	vec3 origin = worldpos + normal * 0.1;
+	vec3 origin = worldpos + normal * 0.01;
 
 #if defined(USE_SUNLIGHT)
 	vec3 incoming = TraceSunLight(origin, normal, SurfaceIndex);

wadsrc/static/shaders/lightmap/trace_ambient_occlusion.glsl

@@ -2,9 +2,11 @@
 vec2 Hammersley(uint i, uint N);
 float RadicalInverse_VdC(uint bits);
 
+float TraceAORay(vec3 origin, float tmin, vec3 dir, float tmax);
+
 float TraceAmbientOcclusion(vec3 origin, vec3 normal)
 {
-	const float minDistance = 0.05;
+	const float minDistance = 0.01;
 	const float aoDistance = 100;
 	const int SampleCount = 128;
 
@@ -19,30 +21,44 @@ float TraceAmbientOcclusion(vec3 origin, vec3 normal)
 		vec2 Xi = Hammersley(i, SampleCount);
 		vec3 H = normalize(vec3(Xi.x * 2.0f - 1.0f, Xi.y * 2.0f - 1.0f, 1.5 - length(Xi)));
 		vec3 L = H.x * tangent + H.y * bitangent + H.z * N;
-
-		TraceResult result = TraceFirstHit(origin, minDistance, L, aoDistance);
-
-		// Ignore surfaces with textures, skies or portals
-		if (result.primitiveIndex != -1)
-		{
-			SurfaceInfo surface = GetSurface(result.primitiveIndex);
-			if (surface.Sky == 0.0 && surface.TextureIndex == 0 && surface.PortalIndex == 0)
-			{
-				ambience += clamp(result.t / aoDistance, 0.0, 1.0);
-			}
-			else
-			{
-				ambience += 1.0;
-			}
-		}
-		else
-		{
-			ambience += 1.0;
-		}
+		ambience += clamp(TraceAORay(origin, minDistance, L, aoDistance) / aoDistance, 0.0, 1.0);
 	}
 	return ambience / float(SampleCount);
 }
 
+float TraceAORay(vec3 origin, float tmin, vec3 dir, float tmax)
+{
+	float tcur = 0.0;
+	for (int i = 0; i < 3; i++)
+	{
+		TraceResult result = TraceFirstHit(origin, tmin, dir, tmax - tcur);
+		if (result.primitiveIndex == -1)
+			return tmax;
+
+		SurfaceInfo surface = GetSurface(result.primitiveIndex);
+
+		// Stop if hit sky portal
+		if (surface.Sky > 0.0)
+			return tmax;
+
+		// Stop if opaque surface
+		if (surface.PortalIndex == 0 /*surface.TextureIndex == 0*/)
+		{
+			return tcur + result.t;
+		}
+
+		// Move to surface hit point
+		origin += dir * result.t;
+		tcur += result.t;
+		if (tcur >= tmax)
+			return tmax;
+
+		// Move through the portal, if any
+		TransformRay(surface.PortalIndex, origin, dir);
+	}
+	return tmax;
+}
+
 vec2 Hammersley(uint i, uint N)
 {
 	return vec2(float(i) / float(N), RadicalInverse_VdC(i));

wadsrc/static/shaders/lightmap/trace_light.glsl

@@ -81,7 +81,7 @@ vec4 TracePointLightRay(vec3 origin, vec3 lightpos, float tmin, vec4 rayColor)
 		origin += dir * result.t;
 		tmax -= result.t;
 
-		// Move through the portal
+		// Move through the portal, if any
 		TransformRay(surface.PortalIndex, origin, dir);
 	}
 	return vec4(0.0);

wadsrc/static/shaders/lightmap/trace_sunlight.glsl

@@ -69,7 +69,7 @@ vec4 TraceSunRay(vec3 origin, float tmin, vec3 dir, float tmax, vec4 rayColor)
 		if (tmax <= tmin)
 			return vec4(0.0);
 
-		// Move through the portal
+		// Move through the portal, if any
 		TransformRay(surface.PortalIndex, origin, dir);
 	}
 	return vec4(0.0);