Added a new cavity automasking mode for sculpt colors painting.

In theory it should also work with other sculpt tools (the
automasking code is fairly general) though it doesn't seem
to do much.

release/scripts/startup/bl_ui/properties_paint_common.py

@@ -925,6 +925,9 @@ def brush_settings_advanced(layout, context, brush, popover=False):
         # topology automasking
         col.prop(brush, "use_automasking_topology", text="Topology")
 
+        col.prop(brush, "concave_mask_factor", text="Cavity Factor");
+        col.prop(brush, "invert_automasking_concavity", text="Invert Cavity Mask");
+
         # face masks automasking
         col.prop(brush, "use_automasking_face_sets", text="Face Sets")
 

source/blender/editors/sculpt_paint/sculpt_automasking.c

@@ -100,6 +100,13 @@ bool SCULPT_is_automasking_enabled(const Sculpt *sd, const SculptSession *ss, co
   if (SCULPT_is_automasking_mode_enabled(sd, br, BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS)) {
     return true;
   }
+  if (SCULPT_is_automasking_mode_enabled(sd, br, BRUSH_AUTOMASKING_CONCAVITY)) {
+    return true;
+  }
+  if (br->concave_mask_factor > 0.0f) {
+    return true;
+  }
+
   return false;
 }
 
@@ -131,14 +138,38 @@ float SCULPT_automasking_factor_get(AutomaskingCache *automasking,
                                     SculptSession *ss,
                                     SculptVertRef vert)
 {
+  float mask = 1.0f;
+  bool do_concave;
+
   if (!automasking) {
-    return 1.0f;
+    return mask;
   }
+
+  do_concave = ss->cache->brush->concave_mask_factor > 0.0f ||
+               (automasking->settings.flags & BRUSH_AUTOMASKING_CONCAVITY);
+
   /* If the cache is initialized with valid info, use the cache. This is used when the
    * automasking information can't be computed in real time per vertex and needs to be
    * initialized for the whole mesh when the stroke starts. */
   if (automasking->factor) {
-    return automasking->factor[BKE_pbvh_vertex_index_to_table(ss->pbvh, vert)];
+    mask = automasking->factor[BKE_pbvh_vertex_index_to_table(ss->pbvh, vert)];
+  }
+
+  if (do_concave) {
+    if (!automasking->factor) {
+      mask = SCULPT_calc_concavity(ss, vert);
+    }
+
+    if (automasking->settings.flags & BRUSH_AUTOMASKING_INVERT_CONCAVITY) {
+      mask = 1.0 - mask;
+    }
+
+    mask = pow(mask*1.5f, (1.0f + automasking->settings.concave_factor) * 8.0);
+    CLAMP(mask, 0.0f, 1.0f);
+  }
+
+  if (automasking->factor) {
+    return mask;
   }
 
   if (automasking->settings.flags & BRUSH_AUTOMASKING_FACE_SETS) {
@@ -159,7 +190,7 @@ float SCULPT_automasking_factor_get(AutomaskingCache *automasking,
     }
   }
 
-  return 1.0f;
+  return mask;
 }
 
 void SCULPT_automasking_cache_free(AutomaskingCache *automasking)
@@ -339,7 +370,65 @@ static void SCULPT_automasking_cache_settings_update(AutomaskingCache *automaski
                                                      Brush *brush)
 {
   automasking->settings.flags = sculpt_automasking_mode_effective_bits(sd, brush);
+
+  if (brush->concave_mask_factor != 0.0f) {
+    automasking->settings.flags |= BRUSH_AUTOMASKING_CONCAVITY;
+  }
+
   automasking->settings.initial_face_set = SCULPT_active_face_set_get(ss);
+  automasking->settings.concave_factor = brush->concave_mask_factor;
+}
+
+float SCULPT_calc_concavity(SculptSession *ss, SculptVertRef vref)
+{
+  SculptVertexNeighborIter ni;
+  float co[3], tot = 0.0, elen = 0.0;
+  const float *vco = SCULPT_vertex_co_get(ss, vref);
+
+  zero_v3(co);
+
+  SCULPT_VERTEX_NEIGHBORS_ITER_BEGIN (ss, vref, ni) {
+    const float *vco2 = SCULPT_vertex_co_get(ss, ni.vertex);
+
+    elen += len_v3v3(vco, vco2);
+    add_v3_v3(co, vco2);
+    tot += 1.0f;
+  }
+  SCULPT_VERTEX_NEIGHBORS_ITER_END(ni);
+
+  if (!tot) {
+    return 0.5f;
+  }
+
+  elen /= tot;
+  mul_v3_fl(co, 1.0 / tot);
+  sub_v3_v3(co, vco);
+  mul_v3_fl(co, -1.0 / elen);
+
+  float no[3];
+  SCULPT_vertex_normal_get(ss, vref, no);
+
+  float f = dot_v3v3(co, no) * 0.5 + 0.5;
+  return 1.0 - f;
+}
+
+static void SCULPT_concavity_automasking_init(Object *ob, Brush *brush, float *factor)
+{
+  SculptSession *ss = ob->sculpt;
+
+  if (!ss) {
+    return;
+  }
+
+  const int totvert = SCULPT_vertex_count_get(ss);
+
+  for (int i = 0; i < totvert; i++) {
+    SculptVertRef vref = BKE_pbvh_table_index_to_vertex(ss->pbvh, i);
+    float f = SCULPT_calc_concavity(ss, vref);
+
+    factor[i] *= f;
+  }
+  // BKE_pbvh_vertex_iter_begin
 }
 
 AutomaskingCache *SCULPT_automasking_cache_init(Sculpt *sd, Brush *brush, Object *ob)
@@ -387,6 +476,10 @@ AutomaskingCache *SCULPT_automasking_cache_init(Sculpt *sd, Brush *brush, Object
     SCULPT_boundary_automasking_init(
         ob, AUTOMASK_INIT_BOUNDARY_FACE_SETS, boundary_propagation_steps, automasking->factor);
   }
+  if (SCULPT_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_CONCAVITY)) {
+    SCULPT_vertex_random_access_ensure(ss);
+    SCULPT_concavity_automasking_init(ob, brush, automasking->factor);
+  }
 
   return automasking;
 }

source/blender/editors/sculpt_paint/sculpt_intern.h

@@ -897,6 +897,7 @@ typedef struct AutomaskingSettings {
   /* Flags from eAutomasking_flag. */
   int flags;
   int initial_face_set;
+  float concave_factor;
 } AutomaskingSettings;
 
 typedef struct AutomaskingCache {
@@ -1228,3 +1229,4 @@ bool SCULPT_ensure_dyntopo_node_undo(struct Object *ob,
                                      int extraType);
 
 void SCULPT_update_flat_vcol_shading(struct Object *ob, struct Scene *scene);
+float SCULPT_calc_concavity(SculptSession *ss, SculptVertRef vref);

source/blender/editors/sculpt_paint/sculpt_paint_color.c

@@ -122,6 +122,7 @@ static void do_paint_brush_task_cb_ex(void *__restrict userdata,
   PBVHVertexIter vd;
   PBVHColorBufferNode *color_buffer;
 
+  
   SculptOrigVertData orig_data;
   SCULPT_orig_vert_data_init(&orig_data, data->ob, data->nodes[n], SCULPT_UNDO_COLOR);
   orig_data.datatype = SCULPT_UNDO_COLOR;

source/blender/makesdna/DNA_brush_enums.h

@@ -324,6 +324,8 @@ typedef enum eAutomasking_flag {
   BRUSH_AUTOMASKING_FACE_SETS = (1 << 1),
   BRUSH_AUTOMASKING_BOUNDARY_EDGES = (1 << 2),
   BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS = (1 << 3),
+  BRUSH_AUTOMASKING_CONCAVITY = (1 << 4),
+  BRUSH_AUTOMASKING_INVERT_CONCAVITY = (1<<5)
 } eAutomasking_flag;
 
 typedef enum ePaintBrush_flag {
@@ -513,6 +515,7 @@ typedef enum eBrushUVSculptTool {
         /* These brushes could handle dynamic topology, \ \
          * but user feedback indicates it's better not to */ \
         SCULPT_TOOL_SMOOTH, \
+        SCULPT_TOOL_VCOL_BOUNDARY, \
         SCULPT_TOOL_MASK) == 0)
 
 #define SCULPT_TOOL_HAS_TOPOLOGY_RAKE(t) \

source/blender/makesdna/DNA_brush_types.h

@@ -367,7 +367,7 @@ typedef struct Brush {
   float mask_stencil_pos[2];
   float mask_stencil_dimension[2];
 
-  int _pad11;
+  float concave_mask_factor;
   struct BrushGpencilSettings *gpencil_settings;
 } Brush;
 

source/blender/makesrna/intern/rna_brush.c

@@ -2842,6 +2842,15 @@ static void rna_def_brush(BlenderRNA *brna)
       prop, "Autosmooth", "Amount of smoothing to automatically apply to each stroke");
   RNA_def_property_update(prop, 0, "rna_Brush_update");
 
+  prop = RNA_def_property(srna, "concave_mask_factor", PROP_FLOAT, PROP_FACTOR);
+  RNA_def_property_float_sdna(prop, NULL, "concave_mask_factor");
+  RNA_def_property_float_default(prop, 0);
+  RNA_def_property_range(prop, 0.0f, 1.0f);
+  RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.001, 3);
+  RNA_def_property_ui_text(
+      prop, "Cavity Mask", "Mask to concave areas");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
   prop = RNA_def_property(srna, "topology_rake_factor", PROP_FLOAT, PROP_FACTOR);
   RNA_def_property_float_sdna(prop, NULL, "topology_rake_factor");
   RNA_def_property_float_default(prop, 0);
@@ -2999,6 +3008,16 @@ static void rna_def_brush(BlenderRNA *brna)
       "When locked keep using the plane origin of surface where stroke was initiated");
   RNA_def_property_update(prop, 0, "rna_Brush_update");
 
+  //note that concavity flag is derived from brush->concave_mask_factor being nonzero,
+  //so we just expose the invert concave flag here
+  prop = RNA_def_property(srna, "invert_automasking_concavity", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(
+      prop, NULL, "automasking_flags", BRUSH_AUTOMASKING_INVERT_CONCAVITY);
+  RNA_def_property_ui_text(prop,
+                           "Invert Cavity Mask",
+                           "Invert mask to expose convex instead of concave areas");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
   prop = RNA_def_property(srna, "use_automasking_topology", PROP_BOOLEAN, PROP_NONE);
   RNA_def_property_boolean_sdna(prop, NULL, "automasking_flags", BRUSH_AUTOMASKING_TOPOLOGY);
   RNA_def_property_ui_text(prop,