Sculpt: Edge Automasking

This automasking option protects the open boundary edges of the mesh from the brush deformation. This is needed to sculpt cloths and it works nicely with the cloth brush.
It has a Propagation Steps property that controls the falloff of the mask from the edge.

Limitations:
- The automask is recalculated at the beginning of each stroke, creating a little bit of lag in high poly meshes, but it is not necessary. This can be fixed in the future by caching the edge distances, increasing a little bit the complexity of the code.
- The boundary vertex detection in meshes is not ideal and it fails with triangulated geometry, but it is the same as in the smooth brush. After fixing this, we should refactor the smooth brush to use the API and let the automasking option manually control the affected vertices.
- It does not work in Multires (it needs to be implemented in the API). The smooth brush in Multires is also not making boundary vertices.
- The falloff has a visible line artifact on grid patterns. We can smooth the final automasking factors several iterations, but it will make the initialization much slower. This can also be added in the future if we decided to cache the distances.

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D6705

release/scripts/startup/bl_ui/properties_paint_common.py

@@ -813,6 +813,11 @@ def brush_settings_advanced(layout, context, brush, popover=False):
 
         # face masks automasking
         layout.prop(brush, "use_automasking_face_sets")
+        
+        # boundary edges automasking
+        layout.prop(brush, "use_automasking_boundary_edges")
+        layout.prop(brush, "automasking_boundary_edges_propagation_steps")
+
 
         # sculpt plane settings
         if capabilities.has_sculpt_plane:

source/blender/blenloader/intern/versioning_280.c

@@ -4811,5 +4811,13 @@ void blo_do_versions_280(FileData *fd, Library *UNUSED(lib), Main *bmain)
         }
       }
     }
+
+    /* Boundary Edges Automasking. */
+    if (!DNA_struct_elem_find(
+            fd->filesdna, "Brush", "int", "automasking_boundary_edges_propagation_steps")) {
+      for (Brush *br = bmain->brushes.first; br; br = br->id.next) {
+        br->automasking_boundary_edges_propagation_steps = 1;
+      }
+    }
   }
 }

source/blender/editors/sculpt_paint/sculpt.c

@@ -618,6 +618,42 @@ void SCULPT_vertex_neighbors_get(SculptSession *ss,
   }
 }
 
+static bool sculpt_vertex_is_boundary(SculptSession *ss, const int index)
+{
+  switch (BKE_pbvh_type(ss->pbvh)) {
+    case PBVH_FACES: {
+      const MeshElemMap *vert_map = &ss->pmap[index];
+
+      if (vert_map->count <= 1) {
+        return false;
+      }
+
+      for (int i = 0; i < vert_map->count; i++) {
+        const MPoly *p = &ss->mpoly[vert_map->indices[i]];
+        unsigned f_adj_v[2];
+        if (poly_get_adj_loops_from_vert(p, ss->mloop, index, f_adj_v) != -1) {
+          int j;
+          for (j = 0; j < ARRAY_SIZE(f_adj_v); j += 1) {
+            if (!(vert_map->count != 2 || ss->pmap[f_adj_v[j]].count <= 2)) {
+              return false;
+            }
+          }
+        }
+      }
+      return true;
+    }
+    case PBVH_BMESH: {
+      BMVert *v = BM_vert_at_index(ss->bm, index);
+      return BM_vert_is_boundary(v);
+    }
+
+    case PBVH_GRIDS:
+      return true;
+  }
+
+  return true;
+}
+
 /* Utils */
 bool SCULPT_check_vertex_pivot_symmetry(const float vco[3], const float pco[3], const char symm)
 {
@@ -1476,6 +1512,9 @@ static bool sculpt_automasking_enabled(SculptSession *ss, const Brush *br)
   if (br->automasking_flags & BRUSH_AUTOMASKING_FACE_SETS) {
     return true;
   }
+  if (br->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
+    return true;
+  }
   return false;
 }
 
@@ -1543,6 +1582,11 @@ static float *sculpt_topology_automasking_init(Sculpt *sd, Object *ob, float *au
     return NULL;
   }
 
+  const int totvert = SCULPT_vertex_count_get(ss);
+  for (int i = 0; i < totvert; i++) {
+    ss->cache->automask[i] = 0.0f;
+  }
+
   /* Flood fill automask to connected vertices. Limited to vertices inside
    * the brush radius if the tool requires it. */
   SculptFloodFill flood;
@@ -1579,25 +1623,83 @@ static float *sculpt_face_sets_automasking_init(Sculpt *sd, Object *ob, float *a
   int tot_vert = SCULPT_vertex_count_get(ss);
   int active_face_set = SCULPT_vertex_face_set_get(ss, SCULPT_active_vertex_get(ss));
   for (int i = 0; i < tot_vert; i++) {
-    if (SCULPT_vertex_has_face_set(ss, i, active_face_set)) {
-      automask_factor[i] = 1;
-    }
-    else {
-      automask_factor[i] = 0;
+    if (!SCULPT_vertex_has_face_set(ss, i, active_face_set)) {
+      automask_factor[i] *= 0.0f;
     }
   }
 
   return automask_factor;
 }
 
+#define EDGE_DISTANCE_INF -1
+
+static float *sculpt_boundary_edges_automasking_init(Sculpt *sd,
+                                                     Object *ob,
+                                                     float *automask_factor)
+{
+  SculptSession *ss = ob->sculpt;
+  Brush *brush = BKE_paint_brush(&sd->paint);
+  const int propagation_steps = brush->automasking_boundary_edges_propagation_steps;
+
+  if (!sculpt_automasking_enabled(ss, brush)) {
+    return NULL;
+  }
+
+  if (BKE_pbvh_type(ss->pbvh) == PBVH_FACES && !ss->pmap) {
+    BLI_assert(!"Boundary Edges masking: pmap missing");
+    return NULL;
+  }
+
+  const int totvert = SCULPT_vertex_count_get(ss);
+  int *edge_distance = MEM_callocN(sizeof(int) * totvert, "automask_factor");
+
+  for (int i = 0; i < totvert; i++) {
+    edge_distance[i] = EDGE_DISTANCE_INF;
+    if (!sculpt_vertex_is_boundary(ss, i)) {
+      edge_distance[i] = 0;
+    }
+  }
+
+  for (int propagation_it = 0; propagation_it < propagation_steps; propagation_it++) {
+    for (int i = 0; i < totvert; i++) {
+      if (edge_distance[i] == EDGE_DISTANCE_INF) {
+        SculptVertexNeighborIter ni;
+        sculpt_vertex_neighbors_iter_begin(ss, i, ni)
+        {
+          if (edge_distance[ni.index] == propagation_it) {
+            edge_distance[i] = propagation_it + 1;
+          }
+        }
+        sculpt_vertex_neighbors_iter_end(ni);
+      }
+    }
+  }
+
+  for (int i = 0; i < totvert; i++) {
+    if (edge_distance[i] != EDGE_DISTANCE_INF) {
+      const float p = 1.0f - ((float)edge_distance[i] / (float)propagation_steps);
+      const float edge_boundary_automask = 3.0f * p * p - 2.0f * p * p * p;
+      automask_factor[i] *= (1.0f - edge_boundary_automask);
+    }
+  }
+
+  MEM_SAFE_FREE(edge_distance);
+  return automask_factor;
+}
+
 static void sculpt_automasking_init(Sculpt *sd, Object *ob)
 {
   SculptSession *ss = ob->sculpt;
   Brush *brush = BKE_paint_brush(&sd->paint);
+  const int totvert = SCULPT_vertex_count_get(ss);
 
   ss->cache->automask = MEM_callocN(sizeof(float) * SCULPT_vertex_count_get(ss),
                                     "automask_factor");
 
+  for (int i = 0; i < totvert; i++) {
+    ss->cache->automask[i] = 1.0f;
+  }
+
   if (brush->automasking_flags & BRUSH_AUTOMASKING_TOPOLOGY) {
     SCULPT_vertex_random_access_init(ss);
     sculpt_topology_automasking_init(sd, ob, ss->cache->automask);
@@ -1606,6 +1708,11 @@ static void sculpt_automasking_init(Sculpt *sd, Object *ob)
     SCULPT_vertex_random_access_init(ss);
     sculpt_face_sets_automasking_init(sd, ob, ss->cache->automask);
   }
+
+  if (brush->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
+    SCULPT_vertex_random_access_init(ss);
+    sculpt_boundary_edges_automasking_init(sd, ob, ss->cache->automask);
+  }
 }
 
 /* ===== Sculpting =====

source/blender/makesdna/DNA_brush_defaults.h

@@ -103,6 +103,7 @@
     .pose_smooth_iterations = 4, \
     .pose_ik_segments = 1, \
     .hardness = 0.0f, \
+    .automasking_boundary_edges_propagation_steps = 1, \
  \
     /* A kernel radius of 1 has almost no effect (T63233). */ \
     .blur_kernel_radius = 2, \

source/blender/makesdna/DNA_brush_types.h

@@ -321,6 +321,7 @@ typedef enum eGP_Sculpt_Mode_Flag {
 typedef enum eAutomasking_flag {
   BRUSH_AUTOMASKING_TOPOLOGY = (1 << 0),
   BRUSH_AUTOMASKING_FACE_SETS = (1 << 1),
+  BRUSH_AUTOMASKING_BOUNDARY_EDGES = (1 << 2),
 } eAutomasking_flag;
 
 typedef struct Brush {
@@ -426,7 +427,7 @@ typedef struct Brush {
   char gpencil_sculpt_tool;
   /** Active grease pencil weight tool. */
   char gpencil_weight_tool;
-  char _pad1_[6];
+  char _pad1[6];
 
   float autosmooth_factor;
 
@@ -446,7 +447,9 @@ typedef struct Brush {
   int curve_preset;
   float hardness;
 
+  /* automasking */
   int automasking_flags;
+  int automasking_boundary_edges_propagation_steps;
 
   /* Factor that controls the shape of the brush tip by rounding the corners of a square. */
   /* 0.0 value produces a square, 1.0 produces a circle. */
@@ -497,7 +500,6 @@ typedef struct Brush {
   float mask_stencil_pos[2];
   float mask_stencil_dimension[2];
 
-  char _pad6[4];
   struct BrushGpencilSettings *gpencil_settings;
 
 } Brush;

source/blender/makesrna/intern/rna_brush.c

@@ -2169,6 +2169,17 @@ static void rna_def_brush(BlenderRNA *brna)
       prop, "Hardness", "How close the brush falloff starts from the edge of the brush");
   RNA_def_property_update(prop, 0, "rna_Brush_update");
 
+  prop = RNA_def_property(
+      srna, "automasking_boundary_edges_propagation_steps", PROP_INT, PROP_UNSIGNED);
+  RNA_def_property_int_sdna(prop, NULL, "automasking_boundary_edges_propagation_steps");
+  RNA_def_property_range(prop, 1, 20);
+  RNA_def_property_ui_range(prop, 1, 20, 1, 3);
+  RNA_def_property_ui_text(prop,
+                           "Propagation Steps",
+                           "Distance where boundary edge automaking is going to protect vertices "
+                           "from the fully masked edge");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
   prop = RNA_def_property(srna, "auto_smooth_factor", PROP_FLOAT, PROP_FACTOR);
   RNA_def_property_float_sdna(prop, NULL, "autosmooth_factor");
   RNA_def_property_float_default(prop, 0);
@@ -2309,6 +2320,11 @@ static void rna_def_brush(BlenderRNA *brna)
                            "Affect only vertices that share Face Sets with the active vertex");
   RNA_def_property_update(prop, 0, "rna_Brush_update");
 
+  prop = RNA_def_property(srna, "use_automasking_boundary_edges", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "automasking_flags", BRUSH_AUTOMASKING_BOUNDARY_EDGES);
+  RNA_def_property_ui_text(prop, "Edges Automasking", "Do not affect non manifold boundary edges");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
   prop = RNA_def_property(srna, "use_scene_spacing", PROP_ENUM, PROP_NONE);
   RNA_def_property_enum_bitflag_sdna(prop, NULL, "flag");
   RNA_def_property_enum_items(prop, brush_spacing_unit_items);