Fix Mask Slice deleting the Face Sets

This was a TODO in the code. Previously the Face Set datalayer was
deleted and recreated with a constant ID of 0. Now the datalayer is
preserved and set to the SculptSession after slicing the mask and a new
ID is calculated for the new faces that the slicing operation produced,
so they can be easily isolated for further tweaking.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D8583

source/blender/editors/include/ED_sculpt.h

@@ -53,6 +53,10 @@ void ED_sculpt_undosys_type(struct UndoType *ut);
 void ED_sculpt_undo_geometry_begin(struct Object *ob, const char *name);
 void ED_sculpt_undo_geometry_end(struct Object *ob);
 
+/* Face sets. */
+int ED_sculpt_face_sets_find_next_available_id(struct Mesh *mesh);
+void ED_sculpt_face_sets_initialize_none_to_id(struct Mesh *mesh, const int new_id);
+
 /* Undo for changes happening on a base mesh for multires sculpting.
  * if there is no multires sculpt active regular undo is used. */
 void ED_sculpt_undo_push_multires_mesh_begin(struct bContext *C, const char *str);

source/blender/editors/mesh/editmesh_mask_extract.c

@@ -354,10 +354,6 @@ static int paint_mask_slice_exec(bContext *C, wmOperator *op)
 
   if (ob->mode == OB_MODE_SCULPT) {
     ED_sculpt_undo_geometry_begin(ob, "mask slice");
-    /* TODO: The ideal functionality would be to preserve the current face sets and add a new one
-     * for the new triangles, but this data-layer needs to be rebuild in order to make sculpt mode
-     * not crash when modifying the geometry. */
-    CustomData_free_layers(&mesh->pdata, CD_SCULPT_FACE_SETS, mesh->totpoly);
   }
 
   BMesh *bm;
@@ -429,14 +425,14 @@ static int paint_mask_slice_exec(bContext *C, wmOperator *op)
   BKE_mesh_calc_normals(ob->data);
 
   if (ob->mode == OB_MODE_SCULPT) {
-    ED_sculpt_undo_geometry_end(ob);
     SculptSession *ss = ob->sculpt;
-    /* Rebuild a new valid Face Set layer for the object. */
-    ss->face_sets = CustomData_add_layer(
-        &mesh->pdata, CD_SCULPT_FACE_SETS, CD_CALLOC, NULL, mesh->totpoly);
-    for (int i = 0; i < mesh->totpoly; i++) {
-      ss->face_sets[i] = 1;
+    ss->face_sets = CustomData_get_layer(&((Mesh *)ob->data)->pdata, CD_SCULPT_FACE_SETS);
+    if (ss->face_sets) {
+      /* Assign a new Face Set ID to the new faces created by the slice operation. */
+      const int next_face_set_id = ED_sculpt_face_sets_find_next_available_id(ob->data);
+      ED_sculpt_face_sets_initialize_none_to_id(ob->data, next_face_set_id);
     }
+    ED_sculpt_undo_geometry_end(ob);
   }
 
   BKE_mesh_batch_cache_dirty_tag(ob->data, BKE_MESH_BATCH_DIRTY_ALL);

source/blender/editors/sculpt_paint/sculpt_face_set.c

@@ -71,6 +71,37 @@
 #include <math.h>
 #include <stdlib.h>
 
+/* Utils. */
+int ED_sculpt_face_sets_find_next_available_id(struct Mesh *mesh)
+{
+  int *face_sets = CustomData_get_layer(&mesh->pdata, CD_SCULPT_FACE_SETS);
+  if (!face_sets) {
+    return SCULPT_FACE_SET_NONE;
+  }
+
+  int next_face_set_id = 0;
+  for (int i = 0; i < mesh->totpoly; i++) {
+    next_face_set_id = max_ii(next_face_set_id, abs(face_sets[i]));
+  }
+  next_face_set_id++;
+
+  return next_face_set_id;
+}
+
+void ED_sculpt_face_sets_initialize_none_to_id(struct Mesh *mesh, const int new_id)
+{
+  int *face_sets = CustomData_get_layer(&mesh->pdata, CD_SCULPT_FACE_SETS);
+  if (!face_sets) {
+    return;
+  }
+
+  for (int i = 0; i < mesh->totpoly; i++) {
+    if (face_sets[i] == SCULPT_FACE_SET_NONE) {
+      face_sets[i] = new_id;
+    }
+  }
+}
+
 /* Draw Face Sets Brush. */
 
 static void do_draw_face_sets_brush_task_cb_ex(void *__restrict userdata,