Sculpt: face set extrude now kinda works

Face set extrude now mostly works, thought it's
still buggy and unstable in DynTopo mode.

Interfacing PBVH_BMESH with the BMOp API
turned out to have a few nasty gotchas; the last commit
fixed a lot of things but some rethinking of the basic
design still needs to happen.

source/blender/blenkernel/BKE_paint.h

@@ -617,7 +617,7 @@ typedef struct SculptSession {
 
   /* Mesh Face Sets */
   /* Total number of polys of the base mesh. */
-  int totfaces;
+  int totedges, totloops, totfaces;
   /* Face sets store its visibility in the sign of the integer, using the absolute value as the
    * Face Set ID. Positive IDs are visible, negative IDs are hidden.
    * The 0 ID is not used by the tools or the visibility system, it is just used when creating new

source/blender/blenkernel/intern/dyntopo.c

@@ -1257,7 +1257,6 @@ void BKE_pbvh_bmesh_remove_face(PBVH *pbvh, BMFace *f, bool log_face)
   pbvh_bmesh_face_remove(pbvh, f, log_face, true, true);
 }
 
-
 void BKE_pbvh_bmesh_remove_edge(PBVH *pbvh, BMEdge *e, bool log_vert)
 {
   if (log_vert) {
@@ -1293,7 +1292,7 @@ void BKE_pbvh_bmesh_add_face(PBVH *pbvh, struct BMFace *f, bool log_face, bool f
     ni = BM_ELEM_CD_GET_INT(l->radial_next->f, pbvh->cd_face_node_offset);
 
     if (ni >= 0 && (!(pbvh->nodes[ni].flag & PBVH_Leaf) || ni >= pbvh->totnode)) {
-      printf("EEK! ni: %d totnode: %d\n", ni, pbvh->totnode);
+      printf("%s: error: ni: %d totnode: %d\n", __func__, ni, pbvh->totnode);
       l = l->next;
       continue;
     }
@@ -2277,7 +2276,7 @@ static bool check_face_is_tri(PBVH *pbvh, BMFace *f)
     }
 
     if (f == f2) {
-      printf("eek!\n");
+      printf("%s: error\n", __func__);
       continue;
     }
 
@@ -4048,7 +4047,7 @@ static void pbvh_bmesh_collapse_edge(PBVH *pbvh,
   mv3->flag |= mupdateflag;
 
   if (BM_ELEM_CD_GET_INT(v_conn, pbvh->cd_vert_node_offset) == DYNTOPO_NODE_NONE) {
-    printf("eek!\n");
+    printf("%s: error\n", __func__);
   }
 
   for (int i = 0; i < 3; i++) {
@@ -6092,7 +6091,7 @@ static void pbvh_split_edges(EdgeQueueContext *eq_ctx,
     }
     else {
       BM_ELEM_CD_SET_INT(newv, pbvh->cd_vert_node_offset, DYNTOPO_NODE_NONE);
-      printf("eek!");
+      printf("%s: error!\n", __func__);
     }
   }
 
@@ -6239,7 +6238,7 @@ static void pbvh_split_edges(EdgeQueueContext *eq_ctx,
         BMFace *f = newfaces[j];
 
         if (f->len != 3) {
-          printf("eek! f->len was not 3! len: %d\n", f->len);
+          printf("%s: f->len was not 3! len: %d\n", __func__, f->len);
         }
       }
 

source/blender/blenkernel/intern/paint.c

@@ -1719,6 +1719,8 @@ static void sculpt_update_object(Depsgraph *depsgraph,
     ss->totvert = me_eval->totvert;
     ss->totpoly = me_eval->totpoly;
     ss->totfaces = me->totpoly;
+    ss->totloops = me->totloop;
+    ss->totedges = me->totedge;
 
     /* These are assigned to the base mesh in Multires. This is needed because Face Sets operators
      * and tools use the Face Sets data from the base mesh when Multires is active. */
@@ -1741,6 +1743,9 @@ static void sculpt_update_object(Depsgraph *depsgraph,
     ss->vmask = CustomData_get_layer(&me->vdata, CD_PAINT_MASK);
     ss->vcol = CustomData_get_layer(&me->vdata, CD_PROP_COLOR);
 
+    ss->totloops = me->totloop;
+    ss->totedges = me->totedge;
+
     ss->vdata = &me->vdata;
     ss->edata = &me->edata;
     ss->ldata = &me->ldata;

source/blender/bmesh/intern/bmesh_mesh.h

@@ -160,3 +160,5 @@ void BM_mesh_vert_coords_apply_with_mat4(BMesh *bm,
   ((bm->idmap.flag & BM_NO_REUSE_IDS) ? \
        BLI_ghash_lookup(bm->idmap.ghash, POINTER_FROM_UINT(id)) : \
        bm->idmap.map[id])
+
+bool BM_elem_is_free(BMElem *elem, int htype);

source/blender/bmesh/intern/bmesh_operators.c

@@ -1273,7 +1273,7 @@ ATTR_NO_OPT static void bmo_flag_layer_do(BMesh *bm,
     int cd_tflags = CustomData_get_offset(cdatas[i], CD_TOOLFLAGS);
 
     if (cd_tflags == -1) {
-      printf("eek!\n");
+      printf("%s: missing toolflags cd layer!\n", __func__);
     }
 
     callback(bm, cd_tflags, iters[i], types[i], tots[i], new_totflags, pools[i]);

source/blender/bmesh/intern/bmesh_structure.c

@@ -23,6 +23,7 @@
  * Low level routines for manipulating the BM structure.
  */
 
+#include "BLI_asan.h"
 #include "BLI_utildefines.h"
 
 #include "bmesh.h"
@@ -642,3 +643,26 @@ bool bmesh_loop_validate(BMFace *f)
 
   return true;
 }
+
+static int sizes[] = {-1,
+                      (int)sizeof(BMVert),
+                      (int)sizeof(BMEdge),
+                      0,
+                      (int)sizeof(BMLoop),
+                      -1,
+                      -1,
+                      -1,
+                      (int)sizeof(BMFace)};
+
+bool BM_elem_is_free(BMElem *elem, int htype)
+{
+  BLI_asan_unpoison(elem, sizes[htype]);
+
+  bool ret = elem->head.htype != htype;
+
+  if (ret) {
+    BLI_asan_poison(elem, sizes[htype]);
+  }
+
+  return ret;
+}

source/blender/editors/sculpt_paint/sculpt.c

@@ -160,7 +160,7 @@ float SCULPT_get_float_intern(const SculptSession *ss,
     return BKE_brush_channelset_get_float(sd->channels, idname, mapdata);
   }
   else {
-    // eek!
+    // should not happen!
     return 0.0f;
   }
 }
@@ -185,7 +185,7 @@ int SCULPT_get_int_intern(const SculptSession *ss,
     return BKE_brush_channelset_get_int(sd->channels, idname, mapdata);
   }
   else {
-    // eek!
+    // should not happen!
     return 0;
   }
 }
@@ -209,7 +209,7 @@ int SCULPT_get_vector_intern(
     return BKE_brush_channelset_get_vector(sd->channels, idname, out, mapdata);
   }
   else {
-    // eek!
+    // should not happen!
     return 0;
   }
 }
@@ -233,6 +233,28 @@ void SCULPT_vertex_random_access_ensure(SculptSession *ss)
   }
 }
 
+void SCULPT_face_normal_get(SculptSession *ss, SculptFaceRef face, float no[3])
+{
+  switch (BKE_pbvh_type(ss->pbvh)) {
+    case PBVH_BMESH: {
+      BMFace *f = (BMFace *)face.i;
+
+      copy_v3_v3(no, f->no);
+      break;
+    }
+
+    case PBVH_FACES:
+    case PBVH_GRIDS: {
+      MPoly *mp = ss->mpoly + face.i;
+      BKE_mesh_calc_poly_normal(mp, ss->mloop + mp->loopstart, ss->mvert, no);
+      break;
+    }
+    default:  // failed
+      zero_v3(no);
+      break;
+  }
+}
+
 /* Sculpt PBVH abstraction API
  *
  * This is read-only, for writing use PBVH vertex iterators. There vd.index matches
@@ -11728,7 +11750,7 @@ void sculpt_stroke_update_step(bContext *C, struct PaintStroke *stroke, PointerR
   }
 
   if (!brush->channels) {
-    // eek!
+    // should not happen!
     printf("had to create brush->channels for brush '%s'!", brush->id.name + 2);
 
     brush->channels = BKE_brush_channelset_create();

source/blender/editors/sculpt_paint/sculpt_intern.h

@@ -319,6 +319,23 @@ void SCULPT_visibility_sync_all_vertex_to_face_sets(struct SculptSession *ss);
 
 /* Face Sets API */
 
+typedef struct SculptFaceSetIsland {
+  SculptFaceRef *faces;
+  int totface;
+} SculptFaceSetIsland;
+
+typedef struct SculptFaceSetIslands {
+  SculptFaceSetIsland *islands;
+  int totisland;
+} SculptFaceSetIslands;
+
+SculptFaceSetIslands *SCULPT_face_set_islands_get(SculptSession *ss, int fset);
+void SCULPT_face_set_islands_free(SculptSession *ss, SculptFaceSetIslands *islands);
+SculptFaceSetIsland *SCULPT_face_set_island_get(SculptSession *ss, SculptFaceRef face, int fset);
+void SCULPT_face_set_island_free(SculptFaceSetIsland *island);
+
+void SCULPT_face_normal_get(SculptSession *ss, SculptFaceRef face, float no[3]);
+
 int SCULPT_active_face_set_get(SculptSession *ss);
 int SCULPT_vertex_face_set_get(SculptSession *ss, SculptVertRef vertex);
 void SCULPT_vertex_face_set_set(SculptSession *ss, SculptVertRef vertex, int face_set);
@@ -1348,7 +1365,6 @@ typedef struct StrokeCache {
   struct BrushCommandList *commandlist;
   bool use_plane_trim;
 } StrokeCache;
-
 /* Sculpt Filters */
 typedef enum SculptFilterOrientation {
   SCULPT_FILTER_ORIENTATION_LOCAL = 0,