Fix T84053: Mask overlay in image editor not working

The mask overlay wasn't part of the overlay engine. The reasoning nehind
this was that more editors used the mask overlay and most of them used
old drawing code. This patch adds the mask overlay drawing to the draw
overlay engine. This code path will only be used by the image editor
VSE, Compositor and Movie Clip editor will still use the previous
method.

During this patch some alternatives have been researched:
1. `ED_mask_draw_region`: this would lead to different code paths when
   drawing in the image editor, and some hacks to retrieve the correct
   framebuffer.
2. Add mask drawing to image engine: Would lead to incorrect color
   management when viewing the mask.
3. Add mask drawing to image engine and overlay engine: Would lead to
   duplicated code.
4. Add mask drawing to overlay engine and for combined overlay select
   the correct framebuffer.

Option 4 was chosen as the exception (switching framebuffers) can be
done without hacks. The code stays clean.

source/blender/draw/CMakeLists.txt

@@ -379,7 +379,8 @@ data_to_c_simple(engines/overlay/shaders/edit_uv_verts_vert.glsl SRC)
 data_to_c_simple(engines/overlay/shaders/edit_uv_verts_frag.glsl SRC)
 data_to_c_simple(engines/overlay/shaders/edit_uv_faces_vert.glsl SRC)
 data_to_c_simple(engines/overlay/shaders/edit_uv_face_dots_vert.glsl SRC)
-data_to_c_simple(engines/overlay/shaders/edit_uv_stencil_image_vert.glsl SRC)
+data_to_c_simple(engines/overlay/shaders/edit_uv_image_vert.glsl SRC)
+data_to_c_simple(engines/overlay/shaders/edit_uv_image_mask_frag.glsl SRC)
 data_to_c_simple(engines/overlay/shaders/edit_uv_stretching_vert.glsl SRC)
 data_to_c_simple(engines/overlay/shaders/edit_uv_tiled_image_borders_vert.glsl SRC)
 data_to_c_simple(engines/overlay/shaders/extra_frag.glsl SRC)

source/blender/draw/engines/overlay/overlay_edit_uv.c

@@ -26,11 +26,14 @@
 
 #include "BKE_editmesh.h"
 #include "BKE_image.h"
+#include "BKE_mask.h"
 #include "BKE_paint.h"
 
 #include "DNA_brush_types.h"
 #include "DNA_mesh_types.h"
 
+#include "DEG_depsgraph_query.h"
+
 #include "ED_image.h"
 
 #include "IMB_imbuf_types.h"
@@ -70,6 +73,27 @@ static OVERLAY_UVLineStyle edit_uv_line_style_from_space_image(const SpaceImage
   }
 }
 
+/* TODO(jbakker): the GPU texture should be cached with the mask. */
+static GPUTexture *edit_uv_mask_texture(
+    Mask *mask, const int width, const int height_, const float aspx, const float aspy)
+{
+  const int height = (float)height_ * (aspy / aspx);
+  MaskRasterHandle *handle;
+  float *buffer = MEM_mallocN(sizeof(float) * height * width, __func__);
+
+  /* Initialize rasterization handle. */
+  handle = BKE_maskrasterize_handle_new();
+  BKE_maskrasterize_handle_init(handle, mask, width, height, true, true, true);
+
+  BKE_maskrasterize_buffer(handle, width, height, buffer);
+
+  /* Free memory. */
+  BKE_maskrasterize_handle_free(handle);
+  GPUTexture *texture = GPU_texture_create_2d(mask->id.name, width, height, 1, GPU_R16F, buffer);
+  MEM_freeN(buffer);
+  return texture;
+}
+
 /* -------------------------------------------------------------------- */
 /** \name Internal API
  * \{ */
@@ -93,6 +117,7 @@ void OVERLAY_edit_uv_init(OVERLAY_Data *vedata)
   const bool has_edit_object = (draw_ctx->object_edit) != NULL;
   const bool is_paint_mode = sima->mode == SI_MODE_PAINT;
   const bool is_view_mode = sima->mode == SI_MODE_VIEW;
+  const bool is_mask_mode = sima->mode == SI_MODE_MASK;
   const bool is_edit_mode = draw_ctx->object_mode == OB_MODE_EDIT;
   const bool do_uv_overlay = is_image_type && is_uv_editor && has_edit_object;
   const bool show_modified_uvs = sima->flag & SI_DRAWSHADOW;
@@ -118,6 +143,14 @@ void OVERLAY_edit_uv_init(OVERLAY_Data *vedata)
                                       (is_view_mode && do_tex_paint_shadows &&
                                        ((draw_ctx->object_mode & (OB_MODE_TEXTURE_PAINT)) != 0)) ||
                                       (do_uv_overlay && (show_modified_uvs)));
+
+  pd->edit_uv.do_mask_overlay = show_overlays && is_mask_mode && (sima->mask_info.mask != NULL) &&
+                                ((sima->mask_info.draw_flag & MASK_DRAWFLAG_OVERLAY) != 0);
+  pd->edit_uv.mask_overlay_mode = sima->mask_info.overlay_mode;
+  pd->edit_uv.mask = sima->mask_info.mask ? (Mask *)DEG_get_evaluated_id(
+                                                draw_ctx->depsgraph, &sima->mask_info.mask->id) :
+                                            NULL;
+
   pd->edit_uv.do_uv_stretching_overlay = show_overlays && do_uvstretching_overlay;
   pd->edit_uv.uv_opacity = sima->uv_opacity;
   pd->edit_uv.do_tiled_image_overlay = show_overlays && is_image_type && is_tiled_image;
@@ -132,7 +165,12 @@ void OVERLAY_edit_uv_init(OVERLAY_Data *vedata)
   pd->edit_uv.total_area_ratio = 0.0f;
   pd->edit_uv.total_area_ratio_inv = 0.0f;
 
-  ED_space_image_get_uv_aspect(sima, &pd->edit_uv.aspect[0], &pd->edit_uv.aspect[1]);
+  /* During engine init phase the sima isn't locked and we are able to retrieve the needed data.
+   * During cache_init the image engine locks the sima and makes it imposible to retrieve the data.
+   */
+  ED_space_image_get_uv_aspect(sima, &pd->edit_uv.uv_aspect[0], &pd->edit_uv.uv_aspect[1]);
+  ED_space_image_get_size(sima, &pd->edit_uv.image_size[0], &pd->edit_uv.image_size[1]);
+  ED_space_image_get_aspect(sima, &pd->edit_uv.image_aspect[0], &pd->edit_uv.image_aspect[1]);
 }
 
 void OVERLAY_edit_uv_cache_init(OVERLAY_Data *vedata)
@@ -225,7 +263,7 @@ void OVERLAY_edit_uv_cache_init(OVERLAY_Data *vedata)
       GPUShader *sh = OVERLAY_shader_edit_uv_stretching_angle_get();
       pd->edit_uv_stretching_grp = DRW_shgroup_create(sh, psl->edit_uv_stretching_ps);
       DRW_shgroup_uniform_block(pd->edit_uv_stretching_grp, "globalsBlock", G_draw.block_ubo);
-      DRW_shgroup_uniform_vec2_copy(pd->edit_uv_stretching_grp, "aspect", pd->edit_uv.aspect);
+      DRW_shgroup_uniform_vec2_copy(pd->edit_uv_stretching_grp, "aspect", pd->edit_uv.uv_aspect);
     }
     else /* SI_UVDT_STRETCH_AREA */ {
       GPUShader *sh = OVERLAY_shader_edit_uv_stretching_area_get();
@@ -334,6 +372,26 @@ void OVERLAY_edit_uv_cache_init(OVERLAY_Data *vedata)
     pd->edit_uv.stencil_ibuf = NULL;
     pd->edit_uv.stencil_image = NULL;
   }
+
+  if (pd->edit_uv.do_mask_overlay) {
+    const bool is_combined_overlay = pd->edit_uv.mask_overlay_mode == MASK_OVERLAY_COMBINED;
+    DRWState state = DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_ALWAYS;
+    state |= is_combined_overlay ? DRW_STATE_BLEND_MUL : DRW_STATE_BLEND_ALPHA;
+    DRW_PASS_CREATE(psl->edit_uv_mask_ps, state);
+
+    GPUShader *sh = OVERLAY_shader_edit_uv_mask_image();
+    GPUBatch *geom = DRW_cache_quad_get();
+    DRWShadingGroup *grp = DRW_shgroup_create(sh, psl->edit_uv_mask_ps);
+    GPUTexture *mask_texture = edit_uv_mask_texture(pd->edit_uv.mask,
+                                                    pd->edit_uv.image_size[0],
+                                                    pd->edit_uv.image_size[1],
+                                                    pd->edit_uv.image_aspect[1],
+                                                    pd->edit_uv.image_aspect[1]);
+    pd->edit_uv.mask_texture = mask_texture;
+    DRW_shgroup_uniform_texture(grp, "imgTexture", mask_texture);
+    DRW_shgroup_uniform_vec4_copy(grp, "color", (float[4]){1.0f, 1.0f, 1.0f, 1.0f});
+    DRW_shgroup_call_obmat(grp, geom, NULL);
+  }
 }
 
 void OVERLAY_edit_uv_cache_populate(OVERLAY_Data *vedata, Object *ob)
@@ -436,6 +494,8 @@ static void OVERLAY_edit_uv_draw_finish(OVERLAY_Data *vedata)
     pd->edit_uv.stencil_image = NULL;
     pd->edit_uv.stencil_ibuf = NULL;
   }
+
+  DRW_TEXTURE_FREE_SAFE(pd->edit_uv.mask_texture);
 }
 
 void OVERLAY_edit_uv_draw(OVERLAY_Data *vedata)
@@ -447,6 +507,21 @@ void OVERLAY_edit_uv_draw(OVERLAY_Data *vedata)
   if (pd->edit_uv.do_tiled_image_border_overlay) {
     DRW_draw_pass(psl->edit_uv_tiled_image_borders_ps);
   }
+  if (pd->edit_uv.do_mask_overlay) {
+    /* Combined overlay renders in the default framebuffer and modifies the image in SRS.
+     * The alpha overlay renders in the overlay framebuffer. */
+    const bool is_combined_overlay = pd->edit_uv.mask_overlay_mode == MASK_OVERLAY_COMBINED;
+    GPUFrameBuffer *previous_framebuffer = NULL;
+    if (is_combined_overlay) {
+      DefaultFramebufferList *dfbl = DRW_viewport_framebuffer_list_get();
+      previous_framebuffer = GPU_framebuffer_active_get();
+      GPU_framebuffer_bind(dfbl->default_fb);
+    }
+    DRW_draw_pass(psl->edit_uv_mask_ps);
+    if (previous_framebuffer) {
+      GPU_framebuffer_bind(previous_framebuffer);
+    }
+  }
 
   if (pd->edit_uv.do_uv_stretching_overlay) {
     edit_uv_stretching_update_ratios(vedata);

source/blender/draw/engines/overlay/overlay_private.h

@@ -34,9 +34,9 @@ extern "C" {
 #  define USE_GEOM_SHADER_WORKAROUND 0
 #endif
 
-/* Needed for eSpaceImage_UVDT_Stretch */
+/* Needed for eSpaceImage_UVDT_Stretch and eMaskOverlayMode */
+#include "DNA_mask_types.h"
 #include "DNA_space_types.h"
-
 /* Forward declarations */
 struct ImBuf;
 
@@ -99,6 +99,7 @@ typedef struct OVERLAY_PassList {
   DRWPass *edit_uv_stretching_ps;
   DRWPass *edit_uv_tiled_image_borders_ps;
   DRWPass *edit_uv_stencil_ps;
+  DRWPass *edit_uv_mask_ps;
   DRWPass *extra_ps[2];
   DRWPass *extra_blend_ps;
   DRWPass *extra_centers_ps;
@@ -368,18 +369,23 @@ typedef struct OVERLAY_PrivateData {
     bool do_tiled_image_overlay;
     bool do_tiled_image_border_overlay;
     bool do_stencil_overlay;
+    bool do_mask_overlay;
 
     bool do_faces;
     bool do_face_dots;
 
     float uv_opacity;
+
+    int image_size[2];
+    float image_aspect[2];
+
     /* edge drawing */
     OVERLAY_UVLineStyle line_style;
     float dash_length;
     int do_smooth_wire;
 
     /* stretching overlay */
-    float aspect[2];
+    float uv_aspect[2];
     eSpaceImage_UVDT_Stretch draw_type;
     ListBase totals;
     float total_area_ratio;
@@ -389,6 +395,11 @@ typedef struct OVERLAY_PrivateData {
     struct Image *stencil_image;
     struct ImBuf *stencil_ibuf;
     void *stencil_lock;
+
+    /* mask overlay */
+    Mask *mask;
+    eMaskOverlayMode mask_overlay_mode;
+    GPUTexture *mask_texture;
   } edit_uv;
   struct {
     bool transparent;
@@ -690,6 +701,7 @@ GPUShader *OVERLAY_shader_edit_uv_stretching_area_get(void);
 GPUShader *OVERLAY_shader_edit_uv_stretching_angle_get(void);
 GPUShader *OVERLAY_shader_edit_uv_tiled_image_borders_get(void);
 GPUShader *OVERLAY_shader_edit_uv_stencil_image(void);
+GPUShader *OVERLAY_shader_edit_uv_mask_image(void);
 GPUShader *OVERLAY_shader_extra(bool is_select);
 GPUShader *OVERLAY_shader_extra_groundline(void);
 GPUShader *OVERLAY_shader_extra_wire(bool use_object, bool is_select);

source/blender/draw/engines/overlay/overlay_shader.c

@@ -78,7 +78,7 @@ extern char datatoc_edit_uv_edges_frag_glsl[];
 extern char datatoc_edit_uv_faces_vert_glsl[];
 extern char datatoc_edit_uv_face_dots_vert_glsl[];
 extern char datatoc_edit_uv_stretching_vert_glsl[];
-extern char datatoc_edit_uv_stencil_image_vert_glsl[];
+extern char datatoc_edit_uv_image_vert_glsl[];
 extern char datatoc_edit_uv_tiled_image_borders_vert_glsl[];
 extern char datatoc_extra_frag_glsl[];
 extern char datatoc_extra_vert_glsl[];
@@ -94,6 +94,7 @@ extern char datatoc_facing_vert_glsl[];
 extern char datatoc_grid_frag_glsl[];
 extern char datatoc_grid_vert_glsl[];
 extern char datatoc_image_frag_glsl[];
+extern char datatoc_edit_uv_image_mask_frag_glsl[];
 extern char datatoc_image_vert_glsl[];
 extern char datatoc_motion_path_line_vert_glsl[];
 extern char datatoc_motion_path_line_geom_glsl[];
@@ -185,6 +186,7 @@ typedef struct OVERLAY_Shaders {
   GPUShader *edit_uv_stretching_area;
   GPUShader *edit_uv_tiled_image_borders;
   GPUShader *edit_uv_stencil_image;
+  GPUShader *edit_uv_mask_image;
   GPUShader *extra;
   GPUShader *extra_select;
   GPUShader *extra_groundline;
@@ -1070,11 +1072,25 @@ GPUShader *OVERLAY_shader_edit_uv_stencil_image(void)
   OVERLAY_Shaders *sh_data = &e_data.sh_data[0];
   if (!sh_data->edit_uv_stencil_image) {
     sh_data->edit_uv_stencil_image = DRW_shader_create_with_shaderlib(
-        datatoc_edit_uv_stencil_image_vert_glsl, NULL, datatoc_image_frag_glsl, e_data.lib, NULL);
+        datatoc_edit_uv_image_vert_glsl, NULL, datatoc_image_frag_glsl, e_data.lib, NULL);
   }
   return sh_data->edit_uv_stencil_image;
 }
 
+GPUShader *OVERLAY_shader_edit_uv_mask_image(void)
+{
+  OVERLAY_Shaders *sh_data = &e_data.sh_data[0];
+  if (!sh_data->edit_uv_mask_image) {
+    sh_data->edit_uv_mask_image = DRW_shader_create_with_shaderlib(
+        datatoc_edit_uv_image_vert_glsl,
+        NULL,
+        datatoc_edit_uv_image_mask_frag_glsl,
+        e_data.lib,
+        NULL);
+  }
+  return sh_data->edit_uv_mask_image;
+}
+
 GPUShader *OVERLAY_shader_image(void)
 {
   const DRWContextState *draw_ctx = DRW_context_state_get();

source/blender/draw/engines/overlay/shaders/edit_uv_image_mask_frag.glsl

@@ -0,0 +1,14 @@
+#pragma BLENDER_REQUIRE(common_colormanagement_lib.glsl)
+
+uniform sampler2D imgTexture;
+uniform vec4 color;
+
+in vec2 uvs;
+out vec4 fragColor;
+
+void main()
+{
+  vec2 uvs_clamped = clamp(uvs, 0.0, 1.0);
+  float mask_value = texture_read_as_linearrgb(imgTexture, true, uvs_clamped).r;
+  fragColor = vec4(color.rgb * mask_value, color.a);
+}

source/blender/draw/tests/shaders_test.cc

@@ -241,6 +241,7 @@ TEST_F(DrawTest, overlay_glsl_shaders)
     EXPECT_NE(OVERLAY_shader_edit_uv_stretching_angle_get(), nullptr);
     EXPECT_NE(OVERLAY_shader_edit_uv_tiled_image_borders_get(), nullptr);
     EXPECT_NE(OVERLAY_shader_edit_uv_stencil_image(), nullptr);
+    EXPECT_NE(OVERLAY_shader_edit_uv_mask_image(), nullptr);
     EXPECT_NE(OVERLAY_shader_extra(false), nullptr);
     EXPECT_NE(OVERLAY_shader_extra(true), nullptr);
     EXPECT_NE(OVERLAY_shader_extra_groundline(), nullptr);

source/blender/editors/space_image/space_image.c

@@ -705,7 +705,8 @@ static void image_main_region_draw(const bContext *C, ARegion *region)
     ED_mask_draw_region(depsgraph,
                         mask,
                         region,
-                        sima->mask_info.draw_flag,
+                        /* Mask overlay is drawn by image/overlay engine. */
+                        sima->mask_info.draw_flag & ~MASK_DRAWFLAG_OVERLAY,
                         sima->mask_info.draw_type,
                         sima->mask_info.overlay_mode,
                         width,