Outliner: Avoid rebuilding tree on selection/active changes

We can avoid the rather expensive outliner tree rebuilds and only redraw
if nothing but the selection or active item changes. This should give a
bit of speedup for heavy scenes.

For this to work I had to correct a few notifiers, some were only
sending selection/active change notifiers that actually did things like
adding objects. I also added a more precise notifier type for when the
active collection changes. At the notifier subtype/action level we're
not even close to running out of bits, so this should be fine.
Also had to correct a wrong notifier check (was using `&` rather than
`==`).

source/blender/editors/object/object_add.c

@@ -1450,6 +1450,7 @@ static int collection_instance_add_exec(bContext *C, wmOperator *op)
     DEG_relations_tag_update(bmain);
     DEG_id_tag_update(&scene->id, ID_RECALC_SELECT);
     WM_event_add_notifier(C, NC_SCENE | ND_OB_ACTIVE, scene);
+    WM_event_add_notifier(C, NC_SCENE | ND_LAYER_CONTENT, scene);
 
     return OPERATOR_FINISHED;
   }
@@ -2783,6 +2784,7 @@ static int object_convert_exec(bContext *C, wmOperator *op)
   DEG_id_tag_update(&scene->id, ID_RECALC_SELECT);
   WM_event_add_notifier(C, NC_OBJECT | ND_DRAW, scene);
   WM_event_add_notifier(C, NC_SCENE | ND_OB_SELECT, scene);
+  WM_event_add_notifier(C, NC_SCENE | ND_LAYER_CONTENT, scene);
 
   return OPERATOR_FINISHED;
 }
@@ -3003,6 +3005,7 @@ static int duplicate_exec(bContext *C, wmOperator *op)
   DEG_id_tag_update(&scene->id, ID_RECALC_COPY_ON_WRITE | ID_RECALC_SELECT);
 
   WM_event_add_notifier(C, NC_SCENE | ND_OB_SELECT, scene);
+  WM_event_add_notifier(C, NC_SCENE | ND_LAYER_CONTENT, scene);
 
   return OPERATOR_FINISHED;
 }
@@ -3094,6 +3097,7 @@ static int object_add_named_exec(bContext *C, wmOperator *op)
   DEG_id_tag_update(&scene->id, ID_RECALC_SELECT);
   WM_event_add_notifier(C, NC_SCENE | ND_OB_SELECT, scene);
   WM_event_add_notifier(C, NC_SCENE | ND_OB_ACTIVE, scene);
+  WM_event_add_notifier(C, NC_SCENE | ND_LAYER_CONTENT, scene);
   ED_outliner_select_sync_from_object_tag(C);
 
   return OPERATOR_FINISHED;

source/blender/editors/space_outliner/outliner_edit.c

@@ -193,13 +193,7 @@ static int outliner_item_openclose_modal(bContext *C, wmOperator *op, const wmEv
       if (te->xs == data->x_location) {
         outliner_item_openclose(te, data->open, false);
 
-        /* Avoid rebuild if possible. */
-        if (outliner_element_needs_rebuild_on_open_change(TREESTORE(te))) {
-          ED_region_tag_redraw(region);
-        }
-        else {
-          ED_region_tag_redraw_no_rebuild(region);
-        }
+        outliner_tag_redraw_avoid_rebuild_on_open_change(space_outliner, region);
       }
     }
 
@@ -239,13 +233,7 @@ static int outliner_item_openclose_invoke(bContext *C, wmOperator *op, const wmE
                       (toggle_all && (outliner_flag_is_any_test(&te->subtree, TSE_CLOSED, 1)));
 
     outliner_item_openclose(te, open, toggle_all);
-    /* Avoid rebuild if possible. */
-    if (outliner_element_needs_rebuild_on_open_change(TREESTORE(te))) {
-      ED_region_tag_redraw(region);
-    }
-    else {
-      ED_region_tag_redraw_no_rebuild(region);
-    }
+    outliner_tag_redraw_avoid_rebuild_on_open_change(space_outliner, region);
 
     /* Only toggle once for single click toggling */
     if (event->type == LEFTMOUSE) {

source/blender/editors/space_outliner/outliner_intern.h

@@ -237,7 +237,7 @@ void outliner_build_tree(struct Main *mainvar,
                          struct SpaceOutliner *space_outliner,
                          struct ARegion *region);
 
-bool outliner_element_needs_rebuild_on_open_change(const TreeStoreElem *tselem);
+bool outliner_mode_requires_always_rebuild(const struct SpaceOutliner *space_outliner);
 
 typedef struct IDsSelectedData {
   struct ListBase selected_array;
@@ -515,6 +515,8 @@ float outliner_restrict_columns_width(const struct SpaceOutliner *space_outliner
 TreeElement *outliner_find_element_with_flag(const ListBase *lb, short flag);
 bool outliner_is_element_visible(const TreeElement *te);
 void outliner_scroll_view(struct ARegion *region, int delta_y);
+void outliner_tag_redraw_avoid_rebuild_on_open_change(const struct SpaceOutliner *space_outliner,
+                                                      struct ARegion *region);
 
 /* outliner_sync.c ---------------------------------------------- */
 

source/blender/editors/space_outliner/outliner_select.c

@@ -1000,7 +1000,9 @@ static eOLDrawState tree_element_active_master_collection(bContext *C,
     ViewLayer *view_layer = CTX_data_view_layer(C);
     LayerCollection *layer_collection = view_layer->layer_collections.first;
     BKE_layer_collection_activate(view_layer, layer_collection);
-    WM_main_add_notifier(NC_SCENE | ND_LAYER, NULL);
+    /* A very precise notifier - ND_LAYER alone is quite vague, we want to avoid unnecessary work
+     * when only the active collection changes. */
+    WM_main_add_notifier(NC_SCENE | ND_LAYER | NS_LAYER_COLLECTION | NA_ACTIVATED, NULL);
   }
 
   return OL_DRAWSEL_NONE;
@@ -1022,7 +1024,9 @@ static eOLDrawState tree_element_active_layer_collection(bContext *C,
     LayerCollection *layer_collection = te->directdata;
     ViewLayer *view_layer = BKE_view_layer_find_from_collection(scene, layer_collection);
     BKE_layer_collection_activate(view_layer, layer_collection);
-    WM_main_add_notifier(NC_SCENE | ND_LAYER, NULL);
+    /* A very precise notifier - ND_LAYER alone is quite vague, we want to avoid unnecessary work
+     * when only the active collection changes. */
+    WM_main_add_notifier(NC_SCENE | ND_LAYER | NS_LAYER_COLLECTION | NA_ACTIVATED, NULL);
   }
 
   return OL_DRAWSEL_NONE;
@@ -1507,7 +1511,7 @@ static int outliner_box_select_exec(bContext *C, wmOperator *op)
 
   DEG_id_tag_update(&scene->id, ID_RECALC_SELECT);
   WM_event_add_notifier(C, NC_SCENE | ND_OB_SELECT, scene);
-  ED_region_tag_redraw(region);
+  ED_region_tag_redraw_no_rebuild(region);
 
   ED_outliner_select_sync_from_outliner(C, space_outliner);
 
@@ -1729,7 +1733,7 @@ static int outliner_walk_select_invoke(bContext *C, wmOperator *op, const wmEven
   outliner_walk_scroll(region, active_te);
 
   ED_outliner_select_sync_from_outliner(C, space_outliner);
-  ED_region_tag_redraw(region);
+  outliner_tag_redraw_avoid_rebuild_on_open_change(space_outliner, region);
 
   return OPERATOR_FINISHED;
 }

source/blender/editors/space_outliner/outliner_tools.c

@@ -1630,6 +1630,7 @@ static int outliner_delete_exec(bContext *C, wmOperator *op)
 
   DEG_id_tag_update(&scene->id, ID_RECALC_SELECT);
   WM_event_add_notifier(C, NC_SCENE | ND_OB_SELECT, scene);
+  WM_event_add_notifier(C, NC_SCENE | ND_LAYER_CONTENT, scene);
   ED_outliner_select_sync_from_object_tag(C);
 
   return OPERATOR_FINISHED;

source/blender/editors/space_outliner/outliner_tree.c

@@ -244,14 +244,12 @@ static TreeElement *outliner_add_element(SpaceOutliner *space_outliner,
 /* -------------------------------------------------------- */
 
 /**
- * Check if an element type needs a full rebuild if the open/collapsed state changes.
- * These element types don't add children if collapsed.
- *
- * This current check isn't great really. A per element-type flag would be preferable.
+ * Check if a display mode needs a full rebuild if the open/collapsed state changes.
+ * Element types in these modes don't actually add children if collapsed, so the rebuild is needed.
  */
-bool outliner_element_needs_rebuild_on_open_change(const TreeStoreElem *tselem)
+bool outliner_mode_requires_always_rebuild(const SpaceOutliner *space_outliner)
 {
-  return ELEM(tselem->type, TSE_RNA_STRUCT, TSE_RNA_PROPERTY, TSE_KEYMAP);
+  return ELEM(space_outliner->outlinevis, SO_DATA_API);
 }
 
 /* special handling of hierarchical non-lib data */

source/blender/editors/space_outliner/outliner_utils.c

@@ -37,6 +37,7 @@
 
 #include "ED_armature.h"
 #include "ED_outliner.h"
+#include "ED_screen.h"
 
 #include "UI_interface.h"
 #include "UI_view2d.h"
@@ -455,6 +456,23 @@ void outliner_scroll_view(ARegion *region, int delta_y)
   }
 }
 
+/**
+ * The outliner should generally use #ED_region_tag_redraw_no_rebuild() to avoid unnecessary tree
+ * rebuilds. If elements are open or closed, we may still have to rebuild.
+ * Upon changing the open/closed state, call this to avoid rebuilds if possible.
+ */
+void outliner_tag_redraw_avoid_rebuild_on_open_change(const SpaceOutliner *space_outliner,
+                                                      ARegion *region)
+{
+  /* Avoid rebuild if possible. */
+  if (outliner_mode_requires_always_rebuild(space_outliner)) {
+    ED_region_tag_redraw(region);
+  }
+  else {
+    ED_region_tag_redraw_no_rebuild(region);
+  }
+}
+
 /* Get base of object under cursor. Used for eyedropper tool */
 Base *ED_outliner_give_base_under_cursor(bContext *C, const int mval[2])
 {

source/blender/editors/space_outliner/space_outliner.c

@@ -114,6 +114,8 @@ static void outliner_main_region_listener(wmWindow *UNUSED(win),
       switch (wmn->data) {
         case ND_OB_ACTIVE:
         case ND_OB_SELECT:
+          ED_region_tag_redraw_no_rebuild(region);
+          break;
         case ND_OB_VISIBLE:
         case ND_OB_RENDER:
         case ND_MODE:
@@ -121,15 +123,23 @@ static void outliner_main_region_listener(wmWindow *UNUSED(win),
         case ND_FRAME:
         case ND_RENDER_OPTIONS:
         case ND_SEQUENCER:
-        case ND_LAYER:
         case ND_LAYER_CONTENT:
         case ND_WORLD:
         case ND_SCENEBROWSE:
+          ED_region_tag_redraw(region);
+          break;
+        case ND_LAYER:
+          /* Avoid rebuild if only the active collection changes */
+          if ((wmn->subtype == NS_LAYER_COLLECTION) && (wmn->action == NA_ACTIVATED)) {
+            ED_region_tag_redraw_no_rebuild(region);
+            break;
+          }
+
           ED_region_tag_redraw(region);
           break;
       }
-      if (wmn->action & NA_EDITED) {
-        ED_region_tag_redraw(region);
+      if (wmn->action == NA_EDITED) {
+        ED_region_tag_redraw_no_rebuild(region);
       }
       break;
     case NC_OBJECT:
@@ -181,7 +191,7 @@ static void outliner_main_region_listener(wmWindow *UNUSED(win),
     case NC_MATERIAL:
       switch (wmn->data) {
         case ND_SHADING_LINKS:
-          ED_region_tag_redraw(region);
+          ED_region_tag_redraw_no_rebuild(region);
           break;
       }
       break;

source/blender/windowmanager/WM_types.h

@@ -440,6 +440,9 @@ typedef struct wmNotifier {
 #define NS_VIEW3D_GPU (16 << 8)
 #define NS_VIEW3D_SHADING (16 << 9)
 
+/* subtype layer editing */
+#define NS_LAYER_COLLECTION (24 << 8)
+
 /* action classification */
 #define NOTE_ACTION (0x000000FF)
 #define NA_EDITED 1
@@ -448,7 +451,8 @@ typedef struct wmNotifier {
 #define NA_REMOVED 4
 #define NA_RENAME 5
 #define NA_SELECTED 6
-#define NA_PAINTING 7
+#define NA_ACTIVATED 7
+#define NA_PAINTING 8
 
 /* ************** Gesture Manager data ************** */