Added FXAA post-processing

Implementation of Fast Approximate Anti-Aliasing is based on nVidia sample:
https://github.com/NVIDIAGameWorks/GraphicsSamples/tree/master/samples/es3-kepler/FXAA

src/CMakeLists.txt

@@ -1131,6 +1131,7 @@ set( FASTMATH_SOURCES
 	gl/shaders/gl_colormapshader.cpp
 	gl/shaders/gl_tonemapshader.cpp
 	gl/shaders/gl_lensshader.cpp
+	gl/shaders/gl_fxaashader.cpp
 	gl/system/gl_interface.cpp
 	gl/system/gl_framebuffer.cpp
 	gl/system/gl_debug.cpp

src/gl/renderer/gl_postprocess.cpp

@@ -60,6 +60,7 @@
 #include "gl/shaders/gl_tonemapshader.h"
 #include "gl/shaders/gl_colormapshader.h"
 #include "gl/shaders/gl_lensshader.h"
+#include "gl/shaders/gl_fxaashader.h"
 #include "gl/shaders/gl_presentshader.h"
 #include "gl/renderer/gl_2ddrawer.h"
 #include "gl/stereo3d/gl_stereo3d.h"
@@ -98,6 +99,14 @@ CVAR(Float, gl_lens_k, -0.12f, CVAR_ARCHIVE | CVAR_GLOBALCONFIG)
 CVAR(Float, gl_lens_kcube, 0.1f, CVAR_ARCHIVE | CVAR_GLOBALCONFIG)
 CVAR(Float, gl_lens_chromatic, 1.12f, CVAR_ARCHIVE | CVAR_GLOBALCONFIG)
 
+CUSTOM_CVAR(Int, gl_fxaa, 0, CVAR_ARCHIVE | CVAR_GLOBALCONFIG)
+{
+	if (self < 0 || self >= FFXAAShader::Count)
+	{
+		self = 0;
+	}
+}
+
 EXTERN_CVAR(Float, vid_brightness)
 EXTERN_CVAR(Float, vid_contrast)
 
@@ -447,6 +456,51 @@ void FGLRenderer::LensDistortScene()
 	FGLDebug::PopGroup();
 }
 
+//-----------------------------------------------------------------------------
+//
+// Apply FXAA and place the result in the HUD/2D texture
+//
+//-----------------------------------------------------------------------------
+
+void FGLRenderer::ApplyFXAA()
+{
+	if (0 == gl_fxaa)
+	{
+		return;
+	}
+
+	FGLDebug::PushGroup("ApplyFXAA");
+
+	const GLfloat rpcRes[2] =
+	{
+		1.0f / mBuffers->GetWidth(),
+		1.0f / mBuffers->GetHeight()
+	};
+
+	FGLPostProcessState savedState;
+
+	mBuffers->BindNextFB();
+	mBuffers->BindCurrentTexture(0);
+	mFXAALumaShader->Bind();
+	mFXAALumaShader->InputTexture.Set(0);
+	RenderScreenQuad();
+	mBuffers->NextTexture();
+
+	mBuffers->BindNextFB();
+	mBuffers->BindCurrentTexture(0);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+	mFXAAShader->Bind();
+	mFXAAShader->InputTexture.Set(0);
+	mFXAAShader->ReciprocalResolution.Set(rpcRes);
+	RenderScreenQuad();
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+	mBuffers->NextTexture();
+
+	FGLDebug::PopGroup();
+}
+
 //-----------------------------------------------------------------------------
 //
 // Copies the rendered screen to its final destination

src/gl/renderer/gl_renderer.cpp

@@ -56,6 +56,7 @@
 #include "gl/shaders/gl_tonemapshader.h"
 #include "gl/shaders/gl_colormapshader.h"
 #include "gl/shaders/gl_lensshader.h"
+#include "gl/shaders/gl_fxaashader.h"
 #include "gl/shaders/gl_presentshader.h"
 #include "gl/shaders/gl_present3dRowshader.h"
 #include "gl/stereo3d/gl_stereo3d.h"
@@ -115,6 +116,8 @@ FGLRenderer::FGLRenderer(OpenGLFrameBuffer *fb)
 	mTonemapPalette = nullptr;
 	mColormapShader = nullptr;
 	mLensShader = nullptr;
+	mFXAAShader = nullptr;
+	mFXAALumaShader = nullptr;
 }
 
 void gl_LoadModels();
@@ -133,6 +136,8 @@ void FGLRenderer::Initialize(int width, int height)
 	mColormapShader = new FColormapShader();
 	mTonemapPalette = nullptr;
 	mLensShader = new FLensShader();
+	mFXAAShader = new FFXAAShader;
+	mFXAALumaShader = new FFXAALumaShader;
 	mPresentShader = new FPresentShader();
 	mPresent3dRowShader = new FPresent3DRowShader();
 	m2DDrawer = new F2DDrawer;
@@ -198,6 +203,8 @@ FGLRenderer::~FGLRenderer()
 	if (mTonemapPalette) delete mTonemapPalette;
 	if (mColormapShader) delete mColormapShader;
 	if (mLensShader) delete mLensShader;
+	delete mFXAAShader;
+	delete mFXAALumaShader;
 }
 
 //==========================================================================

src/gl/renderer/gl_renderer.h

@@ -28,6 +28,8 @@ class FBlurShader;
 class FTonemapShader;
 class FColormapShader;
 class FLensShader;
+class FFXAALumaShader;
+class FFXAAShader;
 class FPresentShader;
 class FPresent3DRowShader;
 class F2DDrawer;
@@ -104,6 +106,8 @@ public:
 	FColormapShader *mColormapShader;
 	FHardwareTexture *mTonemapPalette;
 	FLensShader *mLensShader;
+	FFXAALumaShader *mFXAALumaShader;
+	FFXAAShader *mFXAAShader;
 	FPresentShader *mPresentShader;
 	FPresent3DRowShader *mPresent3dRowShader;
 
@@ -180,6 +184,7 @@ public:
 	void BindTonemapPalette(int texunit);
 	void ClearTonemapPalette();
 	void LensDistortScene();
+	void ApplyFXAA();
 	void CopyToBackbuffer(const GL_IRECT *bounds, bool applyGamma);
 	void DrawPresentTexture(const GL_IRECT &box, bool applyGamma);
 	void Flush();

src/gl/scene/gl_scene.cpp

@@ -832,6 +832,7 @@ sector_t * FGLRenderer::RenderViewpoint (AActor * camera, GL_IRECT * bounds, flo
 			TonemapScene();
 			ColormapScene();
 			LensDistortScene();
+			ApplyFXAA();
 
 			// This should be done after postprocessing, not before.
 			mBuffers->BindCurrentFB();

src/gl/shaders/gl_fxaashader.cpp

@@ -0,0 +1,99 @@
+//
+//---------------------------------------------------------------------------
+//
+// Copyright(C) 2016 Alexey Lysiuk
+// All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with this program.  If not, see http://www.gnu.org/licenses/
+//
+//--------------------------------------------------------------------------
+//
+
+//
+// Fast approXimate Anti-Aliasing (FXAA) post-processing
+//
+
+#include "gl/system/gl_system.h"
+#include "gl/shaders/gl_fxaashader.h"
+
+EXTERN_CVAR(Int, gl_fxaa)
+
+void FFXAALumaShader::Bind()
+{
+	if (!mShader)
+	{
+		mShader.Compile(FShaderProgram::Vertex, "shaders/glsl/screenquad.vp", "", 330);
+		mShader.Compile(FShaderProgram::Fragment, "shaders/glsl/fxaa.fp", "#define FXAA_LUMA_PASS\n", 330);
+		mShader.SetFragDataLocation(0, "FragColor");
+		mShader.Link("shaders/glsl/fxaa");
+		mShader.SetAttribLocation(0, "PositionInProjection");
+		InputTexture.Init(mShader, "InputTexture");
+	}
+
+	mShader.Bind();
+}
+
+static int GetMaxVersion()
+{
+	return gl.glslversion >= 4.f ? 400 : 330;
+}
+
+static FString GetDefines()
+{
+	int quality;
+
+	switch (gl_fxaa)
+	{
+	default:
+	case FFXAAShader::Low:     quality = 10; break;
+	case FFXAAShader::Medium:  quality = 12; break;
+	case FFXAAShader::High:    quality = 29; break;
+	case FFXAAShader::Extreme: quality = 39; break;
+	}
+
+	const int gatherAlpha = GetMaxVersion() >= 400 ? 1 : 0;
+
+	// TODO: enable FXAA_GATHER4_ALPHA on OpenGL earlier than 4.0
+	// when GL_ARB_gpu_shader5/GL_NV_gpu_shader5 extensions are supported
+
+	FString result;
+	result.Format(
+		"#define FXAA_QUALITY__PRESET %i\n"
+		"#define FXAA_GATHER4_ALPHA %i\n",
+		quality, gatherAlpha);
+
+	return result;
+}
+
+void FFXAAShader::Bind()
+{
+	assert(gl_fxaa > 0 && gl_fxaa < Count);
+	FShaderProgram &shader = mShaders[gl_fxaa];
+
+	if (!shader)
+	{
+		const FString defines = GetDefines();
+		const int maxVersion = GetMaxVersion();
+
+		shader.Compile(FShaderProgram::Vertex, "shaders/glsl/screenquad.vp", "", 330);
+		shader.Compile(FShaderProgram::Fragment, "shaders/glsl/fxaa.fp", defines, maxVersion);
+		shader.SetFragDataLocation(0, "FragColor");
+		shader.Link("shaders/glsl/fxaa");
+		shader.SetAttribLocation(0, "PositionInProjection");
+		InputTexture.Init(shader, "InputTexture");
+		ReciprocalResolution.Init(shader, "ReciprocalResolution");
+	}
+
+	shader.Bind();
+}

src/gl/shaders/gl_fxaashader.h

@@ -0,0 +1,66 @@
+// 
+//---------------------------------------------------------------------------
+//
+// Copyright(C) 2016 Alexey Lysiuk
+// All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with this program.  If not, see http://www.gnu.org/licenses/
+//
+//--------------------------------------------------------------------------
+//
+
+//
+// Fast approXimate Anti-Aliasing (FXAA) post-processing
+//
+
+#ifndef __GL_FXAASHADER_H__
+#define __GL_FXAASHADER_H__
+
+#include "gl_shaderprogram.h"
+
+class FFXAALumaShader
+{
+public:
+	void Bind();
+
+	FBufferedUniform1i InputTexture;
+
+private:
+	FShaderProgram mShader;
+};
+
+
+class FFXAAShader
+{
+public:
+	enum Quality
+	{
+		None,
+		Low,
+		Medium,
+		High,
+		Extreme,
+		Count
+	};
+
+	void Bind();
+
+	FBufferedUniform1i InputTexture;
+	FBufferedUniform2f ReciprocalResolution;
+
+private:
+	FShaderProgram mShaders[Count];
+};
+
+#endif // __GL_FXAASHADER_H__