Improve handling of (in)direclty linked status for linked IDs.

This commit essentially ensures before writing .blend file that only
actualy locally used linked IDs are tagged as `LIB_TAG_EXTERN` (and
therefore get a reference stored in the .blend file).

Previously, a linked ID tagged as directly linked would never get back
to the indirectly linked status. Consequence was a lot of 'needless'
references to linked data in .blend files.

This commit also introduces a new flag for lib_query ID usage types,
`IDWALK_CB_DIRECT_WEAK_LINK`, used currently for base's Object
pointer, and for LayerCollection's Collection pointer.

NOTE: A side-effect of this patch is that even IDs explicitely linked by
the user won't be kept anymore when writing files, i.e. they will not be
there anymore after a file reload, if they are not actually used.

Overhead of new process is below 0.1% in whole file saving process in
a Heist production file e.g.

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D16605

source/blender/blenkernel/BKE_lib_query.h

@@ -38,44 +38,53 @@ enum {
    * Indicates whether this is direct (i.e. by local data) or indirect (i.e. by linked data) usage.
    */
   IDWALK_CB_INDIRECT_USAGE = (1 << 2),
+  /**
+   * Indicates that this is a direct weak link usage, i.e. if the user is a local ID, and is using
+   * (pointing to) a linked ID, that usage does not make the linked ID directly linked.
+   *
+   * E.g. usages of linked collections or objects by ViewLayerCollections or Bases in scenes.
+   *
+   * See also #LIB_INDIRECT_WEAK_LINK in DNA_ID.h
+   */
+  IDWALK_CB_DIRECT_WEAK_LINK = (1 << 3),
 
   /**
    * That ID is used as mere sub-data by its owner (only case currently: those root nodetrees in
    * materials etc., and the Scene's master collections).
    * This means callback shall not *do* anything, only use this as informative data if it needs it.
    */
-  IDWALK_CB_EMBEDDED = (1 << 3),
+  IDWALK_CB_EMBEDDED = (1 << 4),
 
   /**
    * That ID is not really used by its owner, it's just an internal hint/helper.
    * This marks the 'from' pointers issue, like Key->from.
    * How to handle that kind of cases totally depends on what caller code is doing... */
-  IDWALK_CB_LOOPBACK = (1 << 4),
+  IDWALK_CB_LOOPBACK = (1 << 5),
 
   /** That ID is used as library override's reference by its owner. */
-  IDWALK_CB_OVERRIDE_LIBRARY_REFERENCE = (1 << 5),
+  IDWALK_CB_OVERRIDE_LIBRARY_REFERENCE = (1 << 6),
 
   /** That ID pointer is not overridable. */
-  IDWALK_CB_OVERRIDE_LIBRARY_NOT_OVERRIDABLE = (1 << 6),
+  IDWALK_CB_OVERRIDE_LIBRARY_NOT_OVERRIDABLE = (1 << 7),
 
   /**
    * Indicates that this is an internal runtime ID pointer, like e.g. `ID.newid` or `ID.original`.
    * \note Those should be ignored in most cases, and won't be processed/generated anyway unless
    * `IDWALK_DO_INTERNAL_RUNTIME_POINTERS` option is enabled.
    */
-  IDWALK_CB_INTERNAL = (1 << 7),
+  IDWALK_CB_INTERNAL = (1 << 8),
 
   /**
    * This ID usage is fully refcounted.
    * Callback is responsible to deal accordingly with #ID.us if needed.
    */
-  IDWALK_CB_USER = (1 << 8),
+  IDWALK_CB_USER = (1 << 9),
   /**
    * This ID usage is not refcounted, but at least one user should be generated by it (to avoid
    * e.g. losing the used ID on save/reload).
    * Callback is responsible to deal accordingly with #ID.us if needed.
    */
-  IDWALK_CB_USER_ONE = (1 << 9),
+  IDWALK_CB_USER_ONE = (1 << 10),
 };
 
 enum {

source/blender/blenkernel/intern/scene.cc

@@ -761,7 +761,8 @@ static void scene_foreach_layer_collection(LibraryForeachIDData *data, ListBase
                          (lc->collection->id.flag & LIB_EMBEDDED_DATA) != 0) ?
                             IDWALK_CB_EMBEDDED :
                             IDWALK_CB_NOP;
-    BKE_LIB_FOREACHID_PROCESS_IDSUPER(data, lc->collection, cb_flag);
+    BKE_LIB_FOREACHID_PROCESS_IDSUPER(
+        data, lc->collection, cb_flag | IDWALK_CB_DIRECT_WEAK_LINK);
     scene_foreach_layer_collection(data, &lc->layer_collections);
   }
 }
@@ -834,8 +835,11 @@ static void scene_foreach_id(ID *id, LibraryForeachIDData *data)
     BKE_LIB_FOREACHID_PROCESS_IDSUPER(data, view_layer->mat_override, IDWALK_CB_USER);
     BKE_view_layer_synced_ensure(scene, view_layer);
     LISTBASE_FOREACH (Base *, base, BKE_view_layer_object_bases_get(view_layer)) {
-      BKE_LIB_FOREACHID_PROCESS_IDSUPER(
-          data, base->object, IDWALK_CB_NOP | IDWALK_CB_OVERRIDE_LIBRARY_NOT_OVERRIDABLE);
+      BKE_LIB_FOREACHID_PROCESS_IDSUPER(data,
+                                        base->object,
+                                        IDWALK_CB_NOP |
+                                            IDWALK_CB_OVERRIDE_LIBRARY_NOT_OVERRIDABLE |
+                                            IDWALK_CB_DIRECT_WEAK_LINK);
     }
 
     BKE_LIB_FOREACHID_PROCESS_FUNCTION_CALL(

source/blender/blenloader/intern/writefile.cc

@@ -102,6 +102,7 @@
 #include "BKE_layer.h"
 #include "BKE_lib_id.h"
 #include "BKE_lib_override.h"
+#include "BKE_lib_query.h"
 #include "BKE_main.h"
 #include "BKE_node.h"
 #include "BKE_packedFile.h"
@@ -1083,6 +1084,30 @@ static void write_thumb(WriteData *wd, const BlendThumbnail *thumb)
 /** \name File Writing (Private)
  * \{ */
 
+/* Helper callback for checking linked IDs used by given ID (assumed local), to ensure directly
+ * linked data is tagged accordingly. */
+static int write_id_direct_linked_data_process_cb(LibraryIDLinkCallbackData *cb_data)
+{
+  ID *id_self = cb_data->id_self;
+  ID *id = *cb_data->id_pointer;
+  const int cb_flag = cb_data->cb_flag;
+
+  if (id == nullptr || !ID_IS_LINKED(id)) {
+    return IDWALK_RET_NOP;
+  }
+  BLI_assert(!ID_IS_LINKED(id_self));
+  BLI_assert((cb_flag & IDWALK_CB_INDIRECT_USAGE) == 0);
+
+  if (cb_flag & IDWALK_CB_DIRECT_WEAK_LINK) {
+    id_lib_indirect_weak_link(id);
+  }
+  else {
+    id_lib_extern(id);
+  }
+
+  return IDWALK_RET_NOP;
+}
+
 /* if MemFile * there's filesave to memory */
 static bool write_file_handle(Main *mainvar,
                               WriteWrap *ww,
@@ -1097,11 +1122,24 @@ static bool write_file_handle(Main *mainvar,
   char buf[16];
   WriteData *wd;
 
-  blo_split_main(&mainlist, mainvar);
-
   wd = mywrite_begin(ww, compare, current);
   BlendWriter writer = {wd};
 
+  /* Clear 'directly linked' flag for all linked data, these are not necessarily valid/up-to-date
+   * info, they will be re-generated while write code is processing local IDs below. */
+  if (!wd->use_memfile) {
+    ID *id_iter;
+    FOREACH_MAIN_ID_BEGIN (mainvar, id_iter) {
+      if (ID_IS_LINKED(id_iter)) {
+        id_iter->tag |= LIB_TAG_INDIRECT;
+        id_iter->tag &= ~LIB_TAG_EXTERN;
+      }
+    }
+    FOREACH_MAIN_ID_END;
+  }
+
+  blo_split_main(&mainlist, mainvar);
+
   BLI_snprintf(buf,
                sizeof(buf),
                "BLENDER%c%c%.3d",
@@ -1169,6 +1207,12 @@ static bool write_file_handle(Main *mainvar,
         const bool do_override = !ELEM(override_storage, nullptr, bmain) &&
                                  ID_IS_OVERRIDE_LIBRARY_REAL(id);
 
+        /* If not writing undo data, properly set directly linked IDs as `LIB_TAG_EXTERN`. */
+        if (!wd->use_memfile) {
+          BKE_library_foreach_ID_link(
+              bmain, id, write_id_direct_linked_data_process_cb, nullptr, IDWALK_READONLY);
+        }
+
         if (do_override) {
           BKE_lib_override_library_operations_store_start(bmain, override_storage, id);
         }

tests/python/bl_blendfile_liblink.py

@@ -247,7 +247,7 @@ class TestBlendLibLinkIndirect(TestBlendLibLinkHelper):
 
         assert material.users == 2
         # Currently linked data which has no more local user never gets reset to indirectly linked status.
-        # ~ assert material.is_library_indirect == True
+        assert material.is_library_indirect == True
 
         bpy.ops.wm.open_mainfile(filepath=output_work_path, load_ui=False)
 
@@ -264,7 +264,7 @@ class TestBlendLibLinkIndirect(TestBlendLibLinkHelper):
         assert material.users == 2  # Fake user is not cleared when linking.
         # Currently even re-reading the .blend file will not properly reset tag for indirectly linked data,
         # if their reference was written in the .blend file.
-        # ~ assert material.is_library_indirect == True
+        assert material.is_library_indirect == True
 
         assert mesh.library is not None
         assert mesh.use_fake_user is False