Enable realtime lights for vertex-lit surfaces

Use the lightMapping GLSL shader instead of `generic` to render BSP
surfaces that are vertex-lit due to an explicit `rgbGen vertex`. This
makes them use the same code path as BSP surfaces where vertex lighting
is automatically selected due to the absence of a lightmap.

So now it is possible to render realtime lights on explicitly vertex-lit
surfaces, though they are still not as bright as they should be due to
#1415.

In the following commit, this will let us remove some ugly code used for
applying the overbright factor to the `generic` shader.

Author: slipher

src/engine/renderer/Material.cpp

@@ -680,7 +680,8 @@ void MaterialSystem::GenerateWorldCommandBuffer( std::vector<MaterialSurface>& s
 			surfaceCommand.drawCommand.baseInstance |= surface.texDataDynamic[stage]
 				? ( surface.texDataIDs[stage] + texData.size() ) << TEX_BUNDLE_BITS
 				: surface.texDataIDs[stage] << TEX_BUNDLE_BITS;
-			surfaceCommand.drawCommand.baseInstance |= ( HasLightMap( &surface ) ? GetLightMapNum( &surface ) : 255 ) << LIGHTMAP_BITS;
+			surfaceCommand.drawCommand.baseInstance |=
+				( !pStage->forceVertexLighting && HasLightMap( &surface ) ? GetLightMapNum( &surface ) : 255 ) << LIGHTMAP_BITS;
 			surfaceCommands[cmdID] = surfaceCommand;
 
 			material->drawCommandCount++;

src/engine/renderer/ShadeCommon.h

@@ -121,7 +121,12 @@ template<typename Obj> void SetLightDeluxeMode(
 	lightMode = lightMode_t::FULLBRIGHT;
 	deluxeMode = deluxeMode_t::NONE;
 
-	if ( hasExplicitelyDisabledLightMap( stage->shader ) && !isExplicitelyVertexLitSurface( stage->shader ) )
+	if ( stage->forceVertexLighting )
+	{
+		lightMode = lightMode_t::VERTEX;
+		deluxeMode = deluxeMode_t::NONE;
+	}
+	else if ( hasExplicitelyDisabledLightMap( stage->shader ) && !isExplicitelyVertexLitSurface( stage->shader ) )
 	{
 		// Use fullbright on “surfaceparm nolightmap” materials.
 	}

src/engine/renderer/tr_bsp.cpp

@@ -871,7 +871,7 @@ static shader_t* ShaderForShaderNum( int shaderNum ) {
 
 	dshader_t* dsh = &s_worldData.shaders[shaderNum];
 
-	shader_t* shader = R_FindShader( dsh->shader, RSF_3D );
+	shader_t* shader = R_FindShader( dsh->shader, RSF_3D | RSF_BSP );
 
 	// If the shader had errors, just use default shader
 	if ( shader->defaultShader ) {

src/engine/renderer/tr_local.h

@@ -964,6 +964,8 @@ enum
 	// TODO(0.56): move to the public RegisterShaderFlags_t interface
 #define RSF_3D ( BIT( 30 ) )
 
+#define RSF_BSP ( BIT ( 29 ) )
+
 	using stageRenderer_t = void(*)(shaderStage_t *);
 	using stageShaderBuildMarker_t = void(*)(const shaderStage_t*);
 	using surfaceDataUpdater_t = void(*)(uint32_t*, shaderStage_t*, bool, bool, bool);
@@ -1033,6 +1035,8 @@ enum
 		bool        highQuality;
 		bool        forceHighQuality;
 
+		bool forceVertexLighting;
+
 		bool        hasDepthFade; // for soft particles
 		float           depthFadeValue;
 

src/engine/renderer/tr_shader.cpp

@@ -5202,6 +5202,14 @@ static void FinishStages()
 				lightStageFound = true;
 				break;
 
+			case stageType_t::ST_COLORMAP:
+				if ( stage->rgbGen == colorGen_t::CGEN_VERTEX && shader.registerFlags & RSF_BSP )
+				{
+					// vertex colors used as lighting detected. Enable overbright and realtime lights
+					stage->type = stageType_t::ST_DIFFUSEMAP;
+					stage->forceVertexLighting = true; // use vertex lighting even if there is a lightmap
+				}
+
 			default:
 				break;
 		}
@@ -6453,6 +6461,7 @@ class ListShadersCmd : public Cmd::StaticCmd
 				shader->registerFlags & RSF_FORCE_LIGHTMAP ? 'L' : '_',
 				shader->registerFlags & RSF_SPRITE ? 'S' : '_',
 				shader->registerFlags & RSF_3D ? '3' : '_',
+				shader->registerFlags & RSF_BSP ? 'B' : '_',
 			};
 
 			if ( !shaderSortName.count( (shaderSort_t) shader->sort ) )