Scultp: Face Set boundary automasking

With this brush option it is possible to mask the boundary vertices of
all face sets. This is especially useful in the cloth brush, where face
sets can be used to simulate seams between different patches of cloth
and produce different patterns and effects.

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D7230

release/scripts/startup/bl_ui/properties_paint_common.py

@@ -823,8 +823,9 @@ def brush_settings_advanced(layout, context, brush, popover=False):
         # face masks automasking
         layout.prop(brush, "use_automasking_face_sets")
         
-        # boundary edges automasking
+        # boundary edges/face sets automasking
         layout.prop(brush, "use_automasking_boundary_edges")
+        layout.prop(brush, "use_automasking_boundary_face_sets")
         layout.prop(brush, "automasking_boundary_edges_propagation_steps")
 
 

source/blender/editors/sculpt_paint/sculpt.c

@@ -1601,6 +1601,9 @@ static bool sculpt_automasking_enabled(SculptSession *ss, const Brush *br)
   if (br->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
     return true;
   }
+  if (br->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS) {
+    return true;
+  }
   return false;
 }
 
@@ -1719,9 +1722,15 @@ static float *sculpt_face_sets_automasking_init(Sculpt *sd, Object *ob, float *a
 
 #define EDGE_DISTANCE_INF -1
 
-static float *sculpt_boundary_edges_automasking_init(Object *ob,
-                                                     int propagation_steps,
-                                                     float *automask_factor)
+typedef enum eBoundaryAutomaskMode {
+  AUTOMASK_INIT_BOUNDARY_EDGES = 1,
+  AUTOMASK_INIT_BOUNDARY_FACE_SETS = 2,
+} eBoundaryAutomaskMode;
+
+static float *sculpt_boundary_automasking_init(Object *ob,
+                                               eBoundaryAutomaskMode mode,
+                                               int propagation_steps,
+                                               float *automask_factor)
 {
   SculptSession *ss = ob->sculpt;
 
@@ -1735,8 +1744,17 @@ static float *sculpt_boundary_edges_automasking_init(Object *ob,
 
   for (int i = 0; i < totvert; i++) {
     edge_distance[i] = EDGE_DISTANCE_INF;
-    if (!sculpt_vertex_is_boundary(ss, i)) {
-      edge_distance[i] = 0;
+    switch (mode) {
+      case AUTOMASK_INIT_BOUNDARY_EDGES:
+        if (!sculpt_vertex_is_boundary(ss, i)) {
+          edge_distance[i] = 0;
+        }
+        break;
+      case AUTOMASK_INIT_BOUNDARY_FACE_SETS:
+        if (!SCULPT_vertex_has_unique_face_set(ss, i)) {
+          edge_distance[i] = 0;
+        }
+        break;
     }
   }
 
@@ -1794,8 +1812,17 @@ static void sculpt_automasking_init(Sculpt *sd, Object *ob)
 
   if (brush->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
     SCULPT_vertex_random_access_init(ss);
-    sculpt_boundary_edges_automasking_init(
-        ob, brush->automasking_boundary_edges_propagation_steps, ss->cache->automask);
+    sculpt_boundary_automasking_init(ob,
+                                     AUTOMASK_INIT_BOUNDARY_EDGES,
+                                     brush->automasking_boundary_edges_propagation_steps,
+                                     ss->cache->automask);
+  }
+  if (brush->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS) {
+    SCULPT_vertex_random_access_init(ss);
+    sculpt_boundary_automasking_init(ob,
+                                     AUTOMASK_INIT_BOUNDARY_FACE_SETS,
+                                     brush->automasking_boundary_edges_propagation_steps,
+                                     ss->cache->automask);
   }
 }
 
@@ -9871,7 +9898,8 @@ static int sculpt_mesh_filter_invoke(bContext *C, wmOperator *op, const wmEvent
     for (int i = 0; i < totvert; i++) {
       ss->filter_cache->automask[i] = 1.0f;
     }
-    sculpt_boundary_edges_automasking_init(ob, 1, ss->filter_cache->automask);
+    sculpt_boundary_automasking_init(
+        ob, AUTOMASK_INIT_BOUNDARY_EDGES, 1, ss->filter_cache->automask);
   }
 
   WM_event_add_modal_handler(C, op);

source/blender/makesdna/DNA_brush_types.h

@@ -332,6 +332,7 @@ typedef enum eAutomasking_flag {
   BRUSH_AUTOMASKING_TOPOLOGY = (1 << 0),
   BRUSH_AUTOMASKING_FACE_SETS = (1 << 1),
   BRUSH_AUTOMASKING_BOUNDARY_EDGES = (1 << 2),
+  BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS = (1 << 3),
 } eAutomasking_flag;
 
 typedef struct Brush {

source/blender/makesrna/intern/rna_brush.c

@@ -2392,6 +2392,14 @@ static void rna_def_brush(BlenderRNA *brna)
       prop, "Mesh Boundary Auto-masking", "Do not affect non manifold boundary edges");
   RNA_def_property_update(prop, 0, "rna_Brush_update");
 
+  prop = RNA_def_property(srna, "use_automasking_boundary_face_sets", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(
+      prop, NULL, "automasking_flags", BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS);
+  RNA_def_property_ui_text(prop,
+                           "Face Sets Boundary Automasking",
+                           "Do not affect vertices that belong to a Face Set boundary");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
   prop = RNA_def_property(srna, "use_scene_spacing", PROP_ENUM, PROP_NONE);
   RNA_def_property_enum_bitflag_sdna(prop, NULL, "flag");
   RNA_def_property_enum_items(prop, brush_spacing_unit_items);