Object: refactor mode switching

Functionality here has become confusing over time,
this removes duplicate, similar functions, preferring to set the mode
instead of toggle, enter, exit.

Mode switching utility function behaved differently regarding undo,
'ED_object_mode_toggle' for example didn't skip the undo push
where 'ED_object_mode_set' did.
Some callers chose these functions based on the intended undo behavior,
even when toggling didn't make sense.

There was also ED_object_mode_generic_enter which was similar to
ED_object_mode_set, instead of the reverse of ED_object_mode_generic_exit.

Simplify object mode switching internals:

- Replace ED_object_mode_generic_enter with ED_object_mode_set.
- Remove ED_object_mode_toggle as nearly all callers needed to check
  the current mode so toggling would set the mode argument correctly.
- Use ED_object_mode_set for the object mode switching operator
  to simplify logic.
- Add ED_object_mode_set_ex which has an argument to disable undo,
  needed when loading undo data needs to set the mode.
- Remove unused ED_object_mode_exit.

source/blender/editors/animation/keyframing.c

@@ -1827,7 +1827,7 @@ static int insert_key_exec(bContext *C, wmOperator *op)
    * updated since the last switching to the edit mode will be keyframed correctly
    */
   if (obedit && ANIM_keyingset_find_id(ks, (ID *)obedit->data)) {
-    ED_object_mode_toggle(C, OB_MODE_EDIT);
+    ED_object_mode_set(C, OB_MODE_OBJECT);
     ob_edit_mode = true;
   }
 
@@ -1843,7 +1843,7 @@ static int insert_key_exec(bContext *C, wmOperator *op)
 
   /* restore the edit mode if necessary */
   if (ob_edit_mode) {
-    ED_object_mode_toggle(C, OB_MODE_EDIT);
+    ED_object_mode_set(C, OB_MODE_EDIT);
   }
 
   /* report failure or do updates? */

source/blender/editors/curve/editfont_undo.c

@@ -359,7 +359,7 @@ static void font_undosys_step_decode(
     struct bContext *C, struct Main *bmain, UndoStep *us_p, int UNUSED(dir), bool UNUSED(is_final))
 {
   /* TODO(campbell): undo_system: use low-level API to set mode. */
-  ED_object_mode_set(C, OB_MODE_EDIT);
+  ED_object_mode_set_ex(C, OB_MODE_EDIT, false, NULL);
   BLI_assert(font_undosys_poll(C));
 
   FontUndoStep *us = (FontUndoStep *)us_p;

source/blender/editors/include/ED_object.h

@@ -329,11 +329,12 @@ bool ED_object_mode_compat_set(struct bContext *C,
                                struct Object *ob,
                                eObjectMode mode,
                                struct ReportList *reports);
-void ED_object_mode_toggle(struct bContext *C, eObjectMode mode);
-void ED_object_mode_set(struct bContext *C, eObjectMode mode);
-void ED_object_mode_exit(struct bContext *C, struct Depsgraph *depsgraph);
+bool ED_object_mode_set_ex(struct bContext *C,
+                           eObjectMode mode,
+                           bool use_undo,
+                           struct ReportList *reports);
+bool ED_object_mode_set(struct bContext *C, eObjectMode mode);
 
-bool ED_object_mode_generic_enter(struct bContext *C, eObjectMode object_mode);
 void ED_object_mode_generic_exit(struct Main *bmain,
                                  struct Depsgraph *depsgraph,
                                  struct Scene *scene,

source/blender/editors/io/io_alembic.c

@@ -663,7 +663,7 @@ static int wm_alembic_import_exec(bContext *C, wmOperator *op)
   /* Switch out of edit mode to avoid being stuck in it (T54326). */
   Object *obedit = CTX_data_edit_object(C);
   if (obedit) {
-    ED_object_mode_toggle(C, OB_MODE_EDIT);
+    ED_object_mode_set(C, OB_MODE_OBJECT);
   }
 
   bool ok = ABC_import(C,

source/blender/editors/object/object_edit.c

@@ -1495,10 +1495,6 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
    * Notes:
    * - Code below avoids calling mode switching functions more than once,
    *   as this causes unnecessary calculations and undo steps to be added.
-   * - Even though toggle is called (#ED_object_mode_toggle),
-   *   this is just used to enable/disable the mode based on checking the current mode.
-   * - Code would read better if we used #ED_object_mode_set,
-   *   this needs some refactoring as it handles undo differently (TODO).
    * - The previous mode (#Object.restore_mode) is object mode by default.
    *
    * Supported Cases:
@@ -1513,14 +1509,7 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
    */
   if (toggle == false) {
     if (ob->mode != mode) {
-      if (mode == OB_MODE_OBJECT) {
-        /* Exit the current mode into object mode. */
-        ED_object_mode_compat_set(C, ob, OB_MODE_OBJECT, op->reports);
-      }
-      else {
-        /* Enter 'mode'. */
-        ED_object_mode_toggle(C, mode);
-      }
+      ED_object_mode_set_ex(C, mode, true, op->reports);
     }
   }
   else {
@@ -1529,16 +1518,14 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
      * otherwise there is nothing to do. */
     if (mode == OB_MODE_OBJECT) {
       if (ob->mode != OB_MODE_OBJECT) {
-        /* Set object mode if the object is not already in object mode. */
-        if (ED_object_mode_compat_set(C, ob, OB_MODE_OBJECT, op->reports)) {
+        if (ED_object_mode_set_ex(C, OB_MODE_OBJECT, true, op->reports)) {
           /* Store old mode so we know what to go back to. */
           ob->restore_mode = mode_prev;
         }
       }
       else {
         if (ob->restore_mode != OB_MODE_OBJECT) {
-          /* Enter 'restore_mode'. */
-          ED_object_mode_toggle(C, ob->restore_mode);
+          ED_object_mode_set_ex(C, ob->restore_mode, true, op->reports);
         }
       }
     }
@@ -1546,20 +1533,17 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
       /* Non-object modes, enter the 'mode' unless it's already set,
        * in that case use restore mode. */
       if (ob->mode != mode) {
-        ED_object_mode_toggle(C, mode);
-        if (ob->mode == mode) {
+        if (ED_object_mode_set_ex(C, mode, true, op->reports)) {
           /* Store old mode so we know what to go back to. */
           ob->restore_mode = mode_prev;
         }
       }
       else {
         if (ob->restore_mode != OB_MODE_OBJECT) {
-          /* Enter 'restore_mode'. */
-          ED_object_mode_toggle(C, ob->restore_mode);
+          ED_object_mode_set_ex(C, ob->restore_mode, true, op->reports);
         }
         else {
-          /* Enter 'mode'. */
-          ED_object_mode_toggle(C, mode);
+          ED_object_mode_set_ex(C, OB_MODE_OBJECT, true, op->reports);
         }
       }
     }

source/blender/editors/object/object_modes.c

@@ -167,41 +167,6 @@ bool ED_object_mode_compat_set(bContext *C, Object *ob, eObjectMode mode, Report
   return ok;
 }
 
-void ED_object_mode_toggle(bContext *C, eObjectMode mode)
-{
-  if (mode != OB_MODE_OBJECT) {
-    const char *opstring = object_mode_op_string(mode);
-
-    if (opstring) {
-      wmOperatorType *ot = WM_operatortype_find(opstring, false);
-      WM_operator_name_call_ptr(C, ot, WM_OP_EXEC_REGION_WIN, NULL);
-    }
-  }
-}
-
-/* Wrapper for operator  */
-void ED_object_mode_set(bContext *C, eObjectMode mode)
-{
-  wmWindowManager *wm = CTX_wm_manager(C);
-  wm->op_undo_depth++;
-  /* needed so we don't do undo pushes. */
-  ED_object_mode_generic_enter(C, mode);
-  wm->op_undo_depth--;
-}
-
-void ED_object_mode_exit(bContext *C, Depsgraph *depsgraph)
-{
-  struct Main *bmain = CTX_data_main(C);
-  Scene *scene = CTX_data_scene(C);
-  ViewLayer *view_layer = CTX_data_view_layer(C);
-  FOREACH_OBJECT_BEGIN (view_layer, ob) {
-    if (ob->mode & OB_MODE_ALL_MODE_DATA) {
-      ED_object_mode_generic_exit(bmain, depsgraph, scene, ob);
-    }
-  }
-  FOREACH_OBJECT_END;
-}
-
 /** \} */
 
 /* -------------------------------------------------------------------- */
@@ -212,23 +177,50 @@ void ED_object_mode_exit(bContext *C, Depsgraph *depsgraph)
  *
  * \{ */
 
-bool ED_object_mode_generic_enter(struct bContext *C, eObjectMode object_mode)
+bool ED_object_mode_set_ex(bContext *C, eObjectMode mode, bool use_undo, ReportList *reports)
 {
+  wmWindowManager *wm = CTX_wm_manager(C);
   ViewLayer *view_layer = CTX_data_view_layer(C);
   Object *ob = OBACT(view_layer);
   if (ob == NULL) {
-    return (object_mode == OB_MODE_OBJECT);
+    return (mode == OB_MODE_OBJECT);
   }
-  if (ob->mode == object_mode) {
+
+  if ((ob->type == OB_GPENCIL) && (mode == OB_MODE_EDIT)) {
+    mode = OB_MODE_EDIT_GPENCIL;
+  }
+
+  if (ob->mode == mode) {
     return true;
   }
-  wmOperatorType *ot = WM_operatortype_find("OBJECT_OT_mode_set", false);
-  PointerRNA ptr;
-  WM_operator_properties_create_ptr(&ptr, ot);
-  RNA_enum_set(&ptr, "mode", object_mode);
-  WM_operator_name_call_ptr(C, ot, WM_OP_INVOKE_DEFAULT, &ptr);
-  WM_operator_properties_free(&ptr);
-  return (ob->mode == object_mode);
+
+  if (!ED_object_mode_compat_test(ob, mode)) {
+    return false;
+  }
+
+  const char *opstring = object_mode_op_string((mode == OB_MODE_OBJECT) ? ob->mode : mode);
+  wmOperatorType *ot = WM_operatortype_find(opstring, false);
+
+  if (!use_undo) {
+    wm->op_undo_depth++;
+  }
+  WM_operator_name_call_ptr(C, ot, WM_OP_EXEC_REGION_WIN, NULL);
+  if (!use_undo) {
+    wm->op_undo_depth--;
+  }
+
+  if (ob->mode != mode) {
+    BKE_reportf(reports, RPT_ERROR, "Unable to execute '%s', error changing modes", ot->name);
+    return false;
+  }
+
+  return true;
+}
+
+bool ED_object_mode_set(bContext *C, eObjectMode mode)
+{
+  /* Don't do undo push by default, since this may be called by lower level code. */
+  return ED_object_mode_set_ex(C, mode, true, NULL);
 }
 
 /**

source/blender/editors/physics/particle_edit_undo.c

@@ -253,7 +253,7 @@ static void particle_undosys_step_decode(struct bContext *C,
 {
   Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C);
   /* TODO(campbell): undo_system: use low-level API to set mode. */
-  ED_object_mode_set(C, OB_MODE_PARTICLE_EDIT);
+  ED_object_mode_set_ex(C, OB_MODE_PARTICLE_EDIT, false, NULL);
   BLI_assert(particle_undosys_poll(C));
 
   ParticleUndoStep *us = (ParticleUndoStep *)us_p;

source/blender/editors/screen/workspace_edit.c

@@ -83,8 +83,7 @@ static void workspace_change_update(WorkSpace *workspace_new,
   eObjectMode mode_new = workspace_new->object_mode;
 
   if (mode_old != mode_new) {
-    ED_object_mode_compat_set(C, ob_act, mode_new, &wm->reports);
-    ED_object_mode_toggle(C, mode_new);
+    ED_object_mode_set(C, mode_new);
   }
 #endif
 }
@@ -175,7 +174,7 @@ bool ED_workspace_change(WorkSpace *workspace_new, bContext *C, wmWindowManager
 
   /* Automatic mode switching. */
   if (workspace_new->object_mode != workspace_old->object_mode) {
-    ED_object_mode_generic_enter(C, workspace_new->object_mode);
+    ED_object_mode_set(C, workspace_new->object_mode);
   }
 
   return true;

source/blender/editors/space_image/image_undo.c

@@ -958,7 +958,7 @@ static void image_undosys_step_decode(
   }
 
   if (us->paint_mode == PAINT_MODE_TEXTURE_3D) {
-    ED_object_mode_set(C, OB_MODE_TEXTURE_PAINT);
+    ED_object_mode_set_ex(C, OB_MODE_TEXTURE_PAINT, false, NULL);
   }
 
   /* Refresh texture slots. */

source/blender/editors/space_view3d/view3d_buttons.c

@@ -1642,7 +1642,7 @@ static int view3d_object_mode_menu(bContext *C, wmOperator *op)
     return OPERATOR_CANCELLED;
   }
   else if (((ob->mode & OB_MODE_EDIT) == 0) && (ELEM(ob->type, OB_ARMATURE))) {
-    ED_object_mode_toggle(C, OB_MODE_POSE);
+    ED_object_mode_set(C, (ob->mode == OB_MODE_OBJECT) ? OB_MODE_POSE : OB_MODE_OBJECT);
     return OPERATOR_CANCELLED;
   }
   else {

source/blender/editors/util/ed_util.c

@@ -174,7 +174,7 @@ void ED_editors_init(bContext *C)
       else {
         /* TODO(campbell): avoid operator calls. */
         if (obact == ob) {
-          ED_object_mode_toggle(C, mode);
+          ED_object_mode_set(C, mode);
         }
       }
     }