ignore transparent improvements

include/render/fx_renderer/fx_renderer.h

@@ -186,6 +186,9 @@ struct fx_renderer {
 		struct blur_shader blur1;
 		struct blur_shader blur2;
 		struct blur_effects_shader blur_effects;
+		struct discard_transparent_shader discard_transparent_rgba;
+		struct discard_transparent_shader discard_transparent_rgbx;
+		struct discard_transparent_shader discard_transparent_ext;
 	} shaders;
 
 	struct wl_list buffers; // fx_framebuffer.link

include/render/fx_renderer/shaders.h

@@ -117,8 +117,6 @@ struct tex_shader {
 	GLint radius_top_right;
 	GLint radius_bottom_left;
 	GLint radius_bottom_right;
-
-	GLint discard_transparent;
 };
 
 bool link_tex_program(struct tex_shader *shader, GLint client_version, enum fx_tex_shader_source source);
@@ -170,4 +168,15 @@ struct blur_effects_shader {
 
 bool link_blur_effects_program(struct blur_effects_shader *shader, GLint client_version);
 
+struct discard_transparent_shader {
+	GLuint program;
+	GLint proj;
+	GLint tex_proj;
+	GLint tex;
+	GLint pos_attrib;
+};
+
+bool link_discard_transparent_program(struct discard_transparent_shader *shader,
+		GLint client_version, enum fx_tex_shader_source source);
+
 #endif

include/scenefx/render/pass.h

@@ -57,7 +57,6 @@ struct fx_render_texture_options {
 	const struct wlr_box *clip_box; // Used to clip csd. Ignored if NULL
 	enum corner_location corners;
 	int corner_radius;
-	bool discard_transparent;
 };
 
 struct fx_render_rect_options {

render/fx_renderer/fx_pass.c

@@ -16,6 +16,7 @@
 #include "render/pass.h"
 #include "scenefx/render/fx_renderer/fx_renderer.h"
 #include "scenefx/render/fx_renderer/fx_effect_framebuffers.h"
+#include "scenefx/render/pass.h"
 #include "scenefx/types/fx/corner_location.h"
 #include "scenefx/types/fx/blur_data.h"
 #include "util/matrix.h"
@@ -27,7 +28,6 @@ struct fx_render_texture_options fx_render_texture_options_default(
 	struct fx_render_texture_options options = {
 		.corner_radius = 0,
 		.corners = CORNER_LOCATION_NONE,
-		.discard_transparent = false,
 		.clip_box = NULL,
 	};
 	memcpy(&options.base, base, sizeof(*base));
@@ -341,8 +341,7 @@ void fx_render_pass_add_texture(struct fx_gles_render_pass *pass,
 
 	bool has_alpha = texture->has_alpha
 		|| alpha < 1.0
-		|| fx_options->corner_radius > 0
-		|| fx_options->discard_transparent;
+		|| fx_options->corner_radius > 0;
 	setup_blending(!has_alpha ? WLR_RENDER_BLEND_MODE_NONE : options->blend_mode);
 
 	glUseProgram(shader->program);
@@ -367,7 +366,6 @@ void fx_render_pass_add_texture(struct fx_gles_render_pass *pass,
 	glUniform1f(shader->alpha, alpha);
 	glUniform2f(shader->size, clip_box->width, clip_box->height);
 	glUniform2f(shader->position, clip_box->x, clip_box->y);
-	glUniform1f(shader->discard_transparent, fx_options->discard_transparent);
 	glUniform1f(shader->radius_top_left, (CORNER_LOCATION_TOP_LEFT & corners) == CORNER_LOCATION_TOP_LEFT ?
 			fx_options->corner_radius : 0);
 	glUniform1f(shader->radius_top_right, (CORNER_LOCATION_TOP_RIGHT & corners) == CORNER_LOCATION_TOP_RIGHT ?
@@ -901,6 +899,90 @@ static struct fx_framebuffer *get_main_buffer_blur(struct fx_gles_render_pass *p
 	return fx_options->current_buffer;
 }
 
+static void fx_render_pass_add_discard_transparent(struct fx_gles_render_pass *pass,
+		struct wlr_render_texture_options *options) {
+	struct fx_renderer *renderer = pass->buffer->renderer;
+	struct fx_texture *texture = fx_get_texture(options->texture);
+
+	struct discard_transparent_shader *shader = NULL;
+
+	switch (texture->target) {
+	case GL_TEXTURE_2D:
+		if (texture->has_alpha) {
+			shader = &renderer->shaders.discard_transparent_rgba;
+		} else {
+			shader = &renderer->shaders.discard_transparent_rgbx;
+		}
+		break;
+	case GL_TEXTURE_EXTERNAL_OES:
+		// EGL_EXT_image_dma_buf_import_modifiers requires
+		// GL_OES_EGL_image_external
+		assert(renderer->exts.OES_egl_image_external);
+		shader = &renderer->shaders.discard_transparent_ext;
+		break;
+	default:
+		abort();
+	}
+
+	struct wlr_box dst_box;
+	struct wlr_fbox src_fbox;
+	wlr_render_texture_options_get_src_box(options, &src_fbox);
+	wlr_render_texture_options_get_dst_box(options, &dst_box);
+
+	src_fbox.x /= options->texture->width;
+	src_fbox.y /= options->texture->height;
+	src_fbox.width /= options->texture->width;
+	src_fbox.height /= options->texture->height;
+
+	push_fx_debug(renderer);
+
+	if (options->wait_timeline != NULL) {
+		int sync_file_fd =
+			wlr_drm_syncobj_timeline_export_sync_file(options->wait_timeline, options->wait_point);
+		if (sync_file_fd < 0) {
+			return;
+		}
+
+		EGLSyncKHR sync = wlr_egl_create_sync(renderer->egl, sync_file_fd);
+		close(sync_file_fd);
+		if (sync == EGL_NO_SYNC_KHR) {
+			return;
+		}
+
+		bool ok = wlr_egl_wait_sync(renderer->egl, sync);
+		wlr_egl_destroy_sync(renderer->egl, sync);
+		if (!ok) {
+			return;
+		}
+	}
+
+	glUseProgram(shader->program);
+
+	glActiveTexture(GL_TEXTURE0);
+	glBindTexture(texture->target, texture->tex);
+
+	switch (options->filter_mode) {
+	case WLR_SCALE_FILTER_BILINEAR:
+		glTexParameteri(texture->target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+		glTexParameteri(texture->target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+		break;
+	case WLR_SCALE_FILTER_NEAREST:
+		glTexParameteri(texture->target, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+		glTexParameteri(texture->target, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+		break;
+	}
+
+	glUniform1i(shader->tex, 0);
+
+	set_proj_matrix(shader->proj, pass->projection_matrix, &dst_box);
+	set_tex_matrix(shader->tex_proj, options->transform, &src_fbox);
+
+	render(&dst_box, options->clip, shader->pos_attrib);
+
+	glBindTexture(texture->target, 0);
+	pop_fx_debug(renderer);
+}
+
 void fx_render_pass_add_blur(struct fx_gles_render_pass *pass,
 		struct fx_render_blur_pass_options *fx_options) {
 	if (pass->buffer->renderer->basic_renderer) {
@@ -945,14 +1027,13 @@ void fx_render_pass_add_blur(struct fx_gles_render_pass *pass,
 	struct fx_texture *blur_texture = fx_get_texture(wlr_texture);
 	blur_texture->has_alpha = true;
 
+	bool should_ignore_transparent = fx_options->ignore_transparent &&
+			tex_options->base.texture;
+
 	// Get a stencil of the window ignoring transparent regions
-	if (fx_options->ignore_transparent && fx_options->tex_options.base.texture) {
+	if (should_ignore_transparent) {
 		stencil_mask_init();
-
-		struct fx_render_texture_options tex_options = fx_options->tex_options;
-		tex_options.discard_transparent = true;
-		fx_render_pass_add_texture(pass, &tex_options);
-
+		fx_render_pass_add_discard_transparent(pass, &tex_options->base);
 		stencil_mask_close(true);
 	}
 
@@ -970,7 +1051,7 @@ void fx_render_pass_add_blur(struct fx_gles_render_pass *pass,
 	wlr_texture_destroy(&blur_texture->wlr_texture);
 
 	// Finish stenciling
-	if (fx_options->ignore_transparent && fx_options->tex_options.base.texture) {
+	if (should_ignore_transparent) {
 		stencil_mask_fini();
 	}
 

render/fx_renderer/fx_renderer.c

@@ -368,6 +368,23 @@ static bool link_shaders(struct fx_renderer *renderer) {
 		goto error;
 	}
 
+	// discard transparent shaders
+	if (!link_discard_transparent_program(&renderer->shaders.discard_transparent_rgba,
+			(GLint) client_version, SHADER_SOURCE_TEXTURE_RGBA)) {
+		wlr_log(WLR_ERROR, "Could not link discard_transparent_rgba shader");
+		goto error;
+	}
+	if (!link_discard_transparent_program(&renderer->shaders.discard_transparent_rgbx,
+			(GLint) client_version, SHADER_SOURCE_TEXTURE_RGBX)) {
+		wlr_log(WLR_ERROR, "Could not link discard_transparent_rgbx shader");
+		goto error;
+	}
+	if (!link_discard_transparent_program(&renderer->shaders.discard_transparent_ext,
+			(GLint) client_version, SHADER_SOURCE_TEXTURE_EXTERNAL)) {
+		wlr_log(WLR_ERROR, "Could not link discard_transparent_ext shader");
+		goto error;
+	}
+
 	return true;
 
 error:
@@ -382,6 +399,9 @@ static bool link_shaders(struct fx_renderer *renderer) {
 	glDeleteProgram(renderer->shaders.blur1.program);
 	glDeleteProgram(renderer->shaders.blur2.program);
 	glDeleteProgram(renderer->shaders.blur_effects.program);
+	glDeleteProgram(renderer->shaders.discard_transparent_rgba.program);
+	glDeleteProgram(renderer->shaders.discard_transparent_rgbx.program);
+	glDeleteProgram(renderer->shaders.discard_transparent_ext.program);
 
 	return false;
 }

render/fx_renderer/gles2/shaders/discard_transparent.frag

@@ -0,0 +1,42 @@
+#define SOURCE %d
+
+#define SOURCE_TEXTURE_RGBA 1
+#define SOURCE_TEXTURE_RGBX 2
+#define SOURCE_TEXTURE_EXTERNAL 3
+
+#if !defined(SOURCE)
+#error "Missing shader preamble"
+#endif
+
+#if SOURCE == SOURCE_TEXTURE_EXTERNAL
+#extension GL_OES_EGL_image_external : require
+#endif
+
+#ifdef GL_FRAGMENT_PRECISION_HIGH
+precision highp float;
+#else
+precision mediump float;
+#endif
+
+varying vec2 v_texcoord;
+
+#if SOURCE == SOURCE_TEXTURE_EXTERNAL
+uniform samplerExternalOES tex;
+#elif SOURCE == SOURCE_TEXTURE_RGBA || SOURCE == SOURCE_TEXTURE_RGBX
+uniform sampler2D tex;
+#endif
+
+vec4 sample_texture() {
+#if SOURCE == SOURCE_TEXTURE_RGBA || SOURCE == SOURCE_TEXTURE_EXTERNAL
+	return texture2D(tex, v_texcoord);
+#elif SOURCE == SOURCE_TEXTURE_RGBX
+	return vec4(texture2D(tex, v_texcoord).rgb, 1.0);
+#endif
+}
+
+void main() {
+	if (sample_texture().a == 0.0) {
+		discard;
+		return;
+	}
+}

render/fx_renderer/gles2/shaders/meson.build

@@ -13,6 +13,7 @@ shaders = [
     'blur1.frag',
     'blur2.frag',
     'blur_effects.frag',
+    'discard_transparent.frag',
 ]
 
 foreach name : shaders

render/fx_renderer/gles2/shaders/tex.frag

@@ -35,8 +35,6 @@ uniform float radius_top_right;
 uniform float radius_bottom_left;
 uniform float radius_bottom_right;
 
-uniform bool discard_transparent;
-
 vec4 sample_texture() {
 #if SOURCE == SOURCE_TEXTURE_RGBA || SOURCE == SOURCE_TEXTURE_EXTERNAL
 	return texture2D(tex, v_texcoord);
@@ -57,9 +55,4 @@ void main() {
         radius_bottom_right
     );
 	gl_FragColor = mix(sample_texture() * alpha, vec4(0.0), corner_alpha);
-
-	if (discard_transparent && gl_FragColor.a == 0.0) {
-		discard;
-		return;
-	}
 }

render/fx_renderer/gles3/shaders/discard_transparent.frag

@@ -0,0 +1,40 @@
+#version 300 es
+
+#define SOURCE %d
+
+#define SOURCE_TEXTURE_RGBA 1
+#define SOURCE_TEXTURE_RGBX 2
+#define SOURCE_TEXTURE_EXTERNAL 3
+
+#if !defined(SOURCE)
+#error "Missing shader preamble"
+#endif
+
+#if SOURCE == SOURCE_TEXTURE_EXTERNAL
+#extension GL_OES_EGL_image_external : require
+#endif
+
+precision highp float;
+
+in vec2 v_texcoord;
+
+#if SOURCE == SOURCE_TEXTURE_EXTERNAL
+uniform samplerExternalOES tex;
+#elif SOURCE == SOURCE_TEXTURE_RGBA || SOURCE == SOURCE_TEXTURE_RGBX
+uniform sampler2D tex;
+#endif
+
+vec4 sample_texture() {
+#if SOURCE == SOURCE_TEXTURE_RGBA || SOURCE == SOURCE_TEXTURE_EXTERNAL
+	return texture2D(tex, v_texcoord);
+#elif SOURCE == SOURCE_TEXTURE_RGBX
+	return vec4(texture2D(tex, v_texcoord).rgb, 1.0);
+#endif
+}
+
+void main() {
+	if (sample_texture().a == 0.0) {
+		discard;
+		return;
+	}
+}

render/fx_renderer/gles3/shaders/meson.build

@@ -13,6 +13,7 @@ shaders = [
     'blur1.frag',
     'blur2.frag',
     'blur_effects.frag',
+	'discard_transparent.frag'
 ]
 
 foreach name : shaders

render/fx_renderer/gles3/shaders/tex.frag

@@ -33,8 +33,6 @@ uniform float radius_top_right;
 uniform float radius_bottom_left;
 uniform float radius_bottom_right;
 
-uniform bool discard_transparent;
-
 out vec4 fragColor;
 
 vec4 sample_texture() {
@@ -57,9 +55,4 @@ void main() {
         radius_bottom_right
     );
 	fragColor = mix(sample_texture() * alpha, vec4(0.0), corner_alpha);
-
-	if (discard_transparent && fragColor.a == 0.0) {
-		discard;
-		return;
-	}
 }