BMesh: remove redundant mesh-backups from EDBM_op_* API

Using BMesh operators through the edit-mesh API created a full copy
of the mesh so it was possible to restore the mesh in case
one of the operators raised an error.

Remove support for automatic backup/restore from the EDBM_op_* API's
as it adds significant overhead and was rarely used.

Operators that need this can use the BMBackup API to backup & restore
the mesh in case of failure.

Add warning levels to BMO_error_raise so operators can report problems
without it being interpreted as a request to cancel the operation.

For high-poly meshes creating and freeing a full copy is an expensive
operation, removing this gives a speedup of ~1.77x for most operators
except for "connect_verts" / "connect_vert_pair"
which still uses this functionality.

source/blender/blenkernel/BKE_editmesh.h

@@ -53,10 +53,6 @@ struct Scene;
 typedef struct BMEditMesh {
   struct BMesh *bm;
 
-  /* This is for undoing failed operations. */
-  struct BMEditMesh *emcopy;
-  int emcopyusers;
-
   /* we store tessellations as triplets of three loops,
    * which each define a triangle. */
   struct BMLoop *(*looptris)[3];

source/blender/bmesh/intern/bmesh_error.h

@@ -24,17 +24,46 @@
 
 /*----------- bmop error system ----------*/
 
+/**
+ * \note More can be added as needed.
+ */
+typedef enum eBMOpErrorLevel {
+  /**
+   * Use when the operation could not succeed,
+   * typically from input that isn't sufficient for completing the operation.
+   */
+  BMO_ERROR_CANCEL = 0,
+  /**
+   * Use this when one or more operations could not succeed,
+   * when the resulting mesh can be used (since some operations succeeded or no change was made).
+   * This is used by default.
+   */
+  BMO_ERROR_WARN = 1,
+  /**
+   * The mesh resulting from this operation should not be used (where possible).
+   * It should not be left in a corrupt state either.
+   *
+   * See #BMBackup type & function calls.
+   */
+  BMO_ERROR_FATAL = 2,
+} eBMOpErrorLevel;
+
 /* Pushes an error onto the bmesh error stack.
  * if msg is null, then the default message for the `errcode` is used. */
-void BMO_error_raise(BMesh *bm, BMOperator *owner, const char *msg) ATTR_NONNULL(1, 2, 3);
+void BMO_error_raise(BMesh *bm, BMOperator *owner, eBMOpErrorLevel level, const char *msg)
+    ATTR_NONNULL(1, 2, 4);
 
 /* Gets the topmost error from the stack.
  * returns error code or 0 if no error. */
-bool BMO_error_get(BMesh *bm, const char **r_msg, BMOperator **r_op);
-bool BMO_error_occurred(BMesh *bm);
+bool BMO_error_get(BMesh *bm, const char **r_msg, BMOperator **r_op, eBMOpErrorLevel *r_level);
+bool BMO_error_get_at_level(BMesh *bm,
+                            eBMOpErrorLevel level,
+                            const char **r_msg,
+                            BMOperator **r_op);
+bool BMO_error_occurred_at_level(BMesh *bm, eBMOpErrorLevel level);
 
 /* Same as #BMO_error_get, only pops the error off the stack as well. */
-bool BMO_error_pop(BMesh *bm, const char **r_msg, BMOperator **r_op);
+bool BMO_error_pop(BMesh *bm, const char **r_msg, BMOperator **r_op, eBMOpErrorLevel *r_level);
 void BMO_error_clear(BMesh *bm);
 
 /* This is meant for handling errors, like self-intersection test failures.

source/blender/bmesh/intern/bmesh_operators.c

@@ -1553,32 +1553,39 @@ typedef struct BMOpError {
   struct BMOpError *next, *prev;
   BMOperator *op;
   const char *msg;
+  eBMOpErrorLevel level;
 } BMOpError;
 
 void BMO_error_clear(BMesh *bm)
 {
-  while (BMO_error_pop(bm, NULL, NULL)) {
+  while (BMO_error_pop(bm, NULL, NULL, NULL)) {
     /* pass */
   }
 }
 
-void BMO_error_raise(BMesh *bm, BMOperator *owner, const char *msg)
+void BMO_error_raise(BMesh *bm, BMOperator *owner, eBMOpErrorLevel level, const char *msg)
 {
   BMOpError *err = MEM_callocN(sizeof(BMOpError), "bmop_error");
 
   err->msg = msg;
   err->op = owner;
+  err->level = level;
 
   BLI_addhead(&bm->errorstack, err);
 }
 
-bool BMO_error_occurred(BMesh *bm)
+bool BMO_error_occurred_at_level(BMesh *bm, eBMOpErrorLevel level)
 {
-  return (BLI_listbase_is_empty(&bm->errorstack) == false);
+  for (const BMOpError *err = bm->errorstack.first; err; err = err->next) {
+    if (err->level == level) {
+      return true;
+    }
+  }
+  return false;
 }
 
 /* returns error code or 0 if no error */
-bool BMO_error_get(BMesh *bm, const char **r_msg, BMOperator **r_op)
+bool BMO_error_get(BMesh *bm, const char **r_msg, BMOperator **r_op, eBMOpErrorLevel *r_level)
 {
   BMOpError *err = bm->errorstack.first;
   if (err == NULL) {
@@ -1591,13 +1598,36 @@ bool BMO_error_get(BMesh *bm, const char **r_msg, BMOperator **r_op)
   if (r_op) {
     *r_op = err->op;
   }
+  if (r_level) {
+    *r_level = err->level;
+  }
 
   return true;
 }
 
-bool BMO_error_pop(BMesh *bm, const char **msg, BMOperator **op)
+bool BMO_error_get_at_level(BMesh *bm,
+                            eBMOpErrorLevel level,
+                            const char **r_msg,
+                            BMOperator **r_op)
 {
-  bool result = BMO_error_get(bm, msg, op);
+  for (BMOpError *err = bm->errorstack.first; err; err = err->next) {
+    if (err->level >= level) {
+      if (r_msg) {
+        *r_msg = err->msg;
+      }
+      if (r_op) {
+        *r_op = err->op;
+      }
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool BMO_error_pop(BMesh *bm, const char **r_msg, BMOperator **r_op, eBMOpErrorLevel *r_level)
+{
+  bool result = BMO_error_get(bm, r_msg, r_op, r_level);
 
   if (result) {
     BMOpError *err = bm->errorstack.first;

source/blender/bmesh/operators/bmo_bisect_plane.c

@@ -50,7 +50,7 @@ void bmo_bisect_plane_exec(BMesh *bm, BMOperator *op)
   BMO_slot_vec_get(op->slots_in, "plane_no", plane_no);
 
   if (is_zero_v3(plane_no)) {
-    BMO_error_raise(bm, op, "Zero normal given");
+    BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Zero normal given");
     return;
   }
 

source/blender/bmesh/operators/bmo_bridge.c

@@ -576,12 +576,12 @@ void bmo_bridge_loops_exec(BMesh *bm, BMOperator *op)
   BM_mesh_edgeloops_calc_center(bm, &eloops);
 
   if (count < 2) {
-    BMO_error_raise(bm, op, "Select at least two edge loops");
+    BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Select at least two edge loops");
     goto cleanup;
   }
 
   if (use_pairs && (count % 2)) {
-    BMO_error_raise(bm, op, "Select an even number of loops to bridge pairs");
+    BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Select an even number of loops to bridge pairs");
     goto cleanup;
   }
 
@@ -595,7 +595,7 @@ void bmo_bridge_loops_exec(BMesh *bm, BMOperator *op)
       }
     }
     if (!match) {
-      BMO_error_raise(bm, op, "Selected loops must have equal edge counts");
+      BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Selected loops must have equal edge counts");
       goto cleanup;
     }
   }

source/blender/bmesh/operators/bmo_connect.c

@@ -211,7 +211,7 @@ void bmo_connect_verts_exec(BMesh *bm, BMOperator *op)
   /* connect faces */
   while ((f = BLI_LINKSTACK_POP(faces))) {
     if (bm_face_connect_verts(bm, f, check_degenerate) == -1) {
-      BMO_error_raise(bm, op, "Could not connect vertices");
+      BMO_error_raise(bm, op, BMO_ERROR_FATAL, "Could not connect vertices");
     }
   }
 

source/blender/bmesh/operators/bmo_dissolve.c

@@ -257,7 +257,7 @@ void bmo_dissolve_faces_exec(BMesh *bm, BMOperator *op)
     }
   }
 
-  BLI_assert(!BMO_error_occurred(bm));
+  BLI_assert(!BMO_error_occurred_at_level(bm, BMO_ERROR_FATAL));
 
   BMO_slot_buffer_from_enabled_flag(bm, op, op->slots_out, "region.out", BM_FACE, FACE_NEW);
 

source/blender/bmesh/operators/bmo_fill_grid.c

@@ -619,6 +619,7 @@ void bmo_grid_fill_exec(BMesh *bm, BMOperator *op)
      * extract two 'rail' loops from a single edge loop, see T72075. */
     BMO_error_raise(bm,
                     op,
+                    BMO_ERROR_CANCEL,
                     "Select two edge loops "
                     "or a single closed edge loop from which two edge loops can be calculated");
     goto cleanup;
@@ -633,7 +634,7 @@ void bmo_grid_fill_exec(BMesh *bm, BMOperator *op)
   v_b_last = ((LinkData *)BM_edgeloop_verts_get(estore_b)->last)->data;
 
   if (BM_edgeloop_is_closed(estore_a) || BM_edgeloop_is_closed(estore_b)) {
-    BMO_error_raise(bm, op, "Closed loops unsupported");
+    BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Closed loops unsupported");
     goto cleanup;
   }
 
@@ -671,7 +672,7 @@ void bmo_grid_fill_exec(BMesh *bm, BMOperator *op)
   bm_edgeloop_flag_set(estore_b, BM_ELEM_HIDDEN, false);
 
   if (BLI_listbase_is_empty(&eloops_rail)) {
-    BMO_error_raise(bm, op, "Loops are not connected by wire/boundary edges");
+    BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Loops are not connected by wire/boundary edges");
     goto cleanup;
   }
 
@@ -679,7 +680,7 @@ void bmo_grid_fill_exec(BMesh *bm, BMOperator *op)
   BLI_assert(v_a_last != v_b_last);
 
   if (BM_edgeloop_overlap_check(estore_rail_a, estore_rail_b)) {
-    BMO_error_raise(bm, op, "Connecting edge loops overlap");
+    BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Connecting edge loops overlap");
     goto cleanup;
   }
 

source/blender/bmesh/operators/bmo_hull.c

@@ -554,7 +554,7 @@ void bmo_convex_hull_exec(BMesh *bm, BMOperator *op)
 
   /* Verify that at least three verts in the input */
   if (!hull_num_input_verts_is_ok(op)) {
-    BMO_error_raise(bm, op, "Requires at least three vertices");
+    BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Requires at least three vertices");
     return;
   }
 

source/blender/bmesh/operators/bmo_subdivide_edgering.c

@@ -1143,7 +1143,7 @@ void bmo_subdivide_edgering_exec(BMesh *bm, BMOperator *op)
   count = BM_mesh_edgeloops_find(bm, &eloops_rim, bm_edge_rim_test_cb, (void *)bm);
 
   if (count < 2) {
-    BMO_error_raise(bm, op, "No edge rings found");
+    BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "No edge rings found");
     goto cleanup;
   }
   else if (count == 2) {
@@ -1167,7 +1167,7 @@ void bmo_subdivide_edgering_exec(BMesh *bm, BMOperator *op)
       changed = true;
     }
     else {
-      BMO_error_raise(bm, op, "Edge-ring pair isn't connected");
+      BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Edge-ring pair isn't connected");
       goto cleanup;
     }
   }
@@ -1179,7 +1179,7 @@ void bmo_subdivide_edgering_exec(BMesh *bm, BMOperator *op)
     LoopPairStore **lpair_arr;
 
     if (eloop_pairs_gs == NULL) {
-      BMO_error_raise(bm, op, "Edge-rings are not connected");
+      BMO_error_raise(bm, op, BMO_ERROR_CANCEL, "Edge-rings are not connected");
       goto cleanup;
     }
 

source/blender/editors/mesh/editmesh_tools.c

@@ -1264,9 +1264,12 @@ static bool edbm_connect_vert_pair(BMEditMesh *em, struct Mesh *me, wmOperator *
     }
   }
   if (checks_succeded) {
+    BMBackup em_backup = EDBM_redo_state_store(em);
+
     BM_custom_loop_normals_to_vector_layer(bm);
 
     BMO_op_exec(bm, &bmop);
+    const bool failure = BMO_error_occurred_at_level(bm, BMO_ERROR_FATAL);
     len = BMO_slot_get(bmop.slots_out, "edges.out")->len;
 
     if (len && is_pair) {
@@ -1275,9 +1278,15 @@ static bool edbm_connect_vert_pair(BMEditMesh *em, struct Mesh *me, wmOperator *
           em->bm, bmop.slots_out, "edges.out", BM_EDGE, BM_ELEM_SELECT, true);
     }
 
-    if (!EDBM_op_finish(em, &bmop, op, true)) {
+    bool em_backup_free = true;
+    if (!EDBM_op_finish(em, &bmop, op, false)) {
       len = 0;
     }
+    else if (failure) {
+      len = 0;
+      EDBM_redo_state_free(&em_backup, em, true);
+      em_backup_free = false;
+    }
     else {
       /* so newly created edges get the selection state from the vertex */
       EDBM_selectmode_flush(em);
@@ -1291,6 +1300,10 @@ static bool edbm_connect_vert_pair(BMEditMesh *em, struct Mesh *me, wmOperator *
                       .is_destructive = true,
                   });
     }
+
+    if (em_backup_free) {
+      EDBM_redo_state_free(&em_backup, NULL, false);
+    }
   }
   MEM_freeN(verts);
 
@@ -7304,7 +7317,7 @@ static int edbm_bridge_edge_loops_for_single_editmesh(wmOperator *op,
 
   BMO_op_exec(em->bm, &bmop);
 
-  if (!BMO_error_occurred(em->bm)) {
+  if (!BMO_error_occurred_at_level(em->bm, BMO_ERROR_CANCEL)) {
     /* when merge is used the edges are joined and remain selected */
     if (use_merge == false) {
       EDBM_flag_disable_all(em, BM_ELEM_SELECT);
@@ -7670,7 +7683,7 @@ static int edbm_convex_hull_exec(bContext *C, wmOperator *op)
     BMO_op_exec(em->bm, &bmop);
 
     /* Hull fails if input is coplanar */
-    if (BMO_error_occurred(em->bm)) {
+    if (BMO_error_occurred_at_level(em->bm, BMO_ERROR_CANCEL)) {
       EDBM_op_finish(em, &bmop, op, true);
       continue;
     }

source/blender/editors/mesh/editmesh_utils.c

@@ -115,45 +115,6 @@ void EDBM_redo_state_free(BMBackup *backup, BMEditMesh *em, int recalctess)
 
 /** \} */
 
-/* -------------------------------------------------------------------- */
-/** \name Edit-Mesh Copy API (Internal)
- * \{ */
-
-static void edbm_op_emcopy_incref_and_ensure(BMEditMesh *em, const BMOperator *UNUSED(bmop))
-{
-  if (em->emcopy == NULL) {
-    em->emcopy = BKE_editmesh_copy(em);
-  }
-  em->emcopyusers++;
-}
-
-static void edbm_op_emcopy_decref(BMEditMesh *em, const BMOperator *UNUSED(bmop))
-{
-  em->emcopyusers--;
-  if (em->emcopyusers < 0) {
-    printf("warning: em->emcopyusers was less than zero.\n");
-  }
-  if (em->emcopyusers <= 0) {
-    BKE_editmesh_free(em->emcopy);
-    MEM_freeN(em->emcopy);
-    em->emcopy = NULL;
-  }
-}
-
-static void edbm_op_emcopy_restore_and_clear(BMEditMesh *em, const BMOperator *UNUSED(bmop))
-{
-  BLI_assert(em->emcopy != NULL);
-  BMEditMesh *emcopy = em->emcopy;
-  EDBM_mesh_free(em);
-  *em = *emcopy;
-
-  MEM_freeN(emcopy);
-  em->emcopyusers = 0;
-  em->emcopy = NULL;
-}
-
-/** \} */
-
 /* -------------------------------------------------------------------- */
 /** \name BMesh Operator (BMO) API Wrapper
  * \{ */
@@ -171,8 +132,6 @@ bool EDBM_op_init(BMEditMesh *em, BMOperator *bmop, wmOperator *op, const char *
     return false;
   }
 
-  edbm_op_emcopy_incref_and_ensure(em, bmop);
-
   va_end(list);
 
   return true;
@@ -187,29 +146,62 @@ bool EDBM_op_finish(BMEditMesh *em, BMOperator *bmop, wmOperator *op, const bool
 {
   const char *errmsg;
 
+#ifndef NDEBUG
+  struct {
+    int verts_len, edges_len, loops_len, faces_len;
+  } em_state_prev = {
+      .verts_len = em->bm->totvert,
+      .edges_len = em->bm->totedge,
+      .loops_len = em->bm->totloop,
+      .faces_len = em->bm->totface,
+  };
+#endif
+
   BMO_op_finish(em->bm, bmop);
 
-  if (BMO_error_get(em->bm, &errmsg, NULL)) {
-    BLI_assert(em->emcopy != NULL);
+  bool changed = false;
+  bool changed_was_set = false;
+
+  eBMOpErrorLevel level;
+  while (BMO_error_pop(em->bm, &errmsg, NULL, &level)) {
+    ReportType type = RPT_INFO;
+    switch (level) {
+      case BMO_ERROR_CANCEL: {
+        changed_was_set = true;
+        break;
+      }
+      case BMO_ERROR_WARN: {
+        type = RPT_WARNING;
+        changed_was_set = true;
+        changed = true;
+        break;
+      }
+      case BMO_ERROR_FATAL: {
+        type = RPT_ERROR;
+        changed_was_set = true;
+        changed = true;
+        break;
+      }
+    }
 
     if (do_report) {
-      BKE_report(op->reports, RPT_ERROR, errmsg);
+      BKE_report(op->reports, type, errmsg);
     }
-
-    edbm_op_emcopy_restore_and_clear(em, bmop);
-
-    /* when copying, tessellation isn't to for faster copying,
-     * but means we need to re-tessellate here */
-    if (em->looptris == NULL) {
-      BKE_editmesh_looptri_calc(em);
-    }
-
-    return false;
+  }
+  if (changed_was_set == false) {
+    changed = true;
   }
 
-  edbm_op_emcopy_decref(em, bmop);
+#ifndef NDEBUG
+  if (changed == false) {
+    BLI_assert((em_state_prev.verts_len == em->bm->totvert) &&
+               (em_state_prev.edges_len == em->bm->totedge) &&
+               (em_state_prev.loops_len == em->bm->totloop) &&
+               (em_state_prev.faces_len == em->bm->totface));
+  }
+#endif
 
-  return true;
+  return changed;
 }
 
 bool EDBM_op_callf(BMEditMesh *em, wmOperator *op, const char *fmt, ...)
@@ -226,8 +218,6 @@ bool EDBM_op_callf(BMEditMesh *em, wmOperator *op, const char *fmt, ...)
     return false;
   }
 
-  edbm_op_emcopy_incref_and_ensure(em, &bmop);
-
   BMO_op_exec(bm, &bmop);
 
   va_end(list);
@@ -255,8 +245,6 @@ bool EDBM_op_call_and_selectf(BMEditMesh *em,
     return false;
   }
 
-  edbm_op_emcopy_incref_and_ensure(em, &bmop);
-
   BMO_op_exec(bm, &bmop);
 
   slot_select_out = BMO_slot_get(bmop.slots_out, select_slot_out);
@@ -287,8 +275,6 @@ bool EDBM_op_call_silentf(BMEditMesh *em, const char *fmt, ...)
     return false;
   }
 
-  edbm_op_emcopy_incref_and_ensure(em, &bmop);
-
   BMO_op_exec(bm, &bmop);
 
   va_end(list);

source/blender/modifiers/intern/MOD_skin.c

@@ -296,7 +296,7 @@ static bool build_hull(SkinOutput *so, Frame **frames, int totframe)
       bm, &op, (BMO_FLAG_DEFAULTS & ~BMO_FLAG_RESPECT_HIDE), "convex_hull input=%hv", BM_ELEM_TAG);
   BMO_op_exec(bm, &op);
 
-  if (BMO_error_occurred(bm)) {
+  if (BMO_error_occurred_at_level(bm, BMO_ERROR_CANCEL)) {
     BMO_op_finish(bm, &op);
     return false;
   }

source/blender/python/bmesh/bmesh_py_ops_call.c

@@ -44,10 +44,10 @@ BLI_STATIC_ASSERT(sizeof(PyC_FlagSet) == sizeof(BMO_FlagSet), "size mismatch");
 
 static int bpy_bm_op_as_py_error(BMesh *bm)
 {
-  if (BMO_error_occurred(bm)) {
+  if (BMO_error_occurred_at_level(bm, BMO_ERROR_FATAL)) {
     /* NOTE: we could have multiple errors. */
     const char *errmsg;
-    if (BMO_error_get(bm, &errmsg, NULL)) {
+    if (BMO_error_get(bm, &errmsg, NULL, NULL)) {
       PyErr_Format(PyExc_RuntimeError, "bmesh operator: %.200s", errmsg);
       BMO_error_clear(bm);
       return -1;