Load catalogs for "All" asset library

Merges the catalog definitions from all asset libraries in to the
storage of the "All" one, builds the catalog tree and refreshes data as
needed. This doesn't allow writing changes back to the catalog
definition files, so the UI probably shouldn't allow edits.

source/blender/asset_system/AS_asset_catalog.hh

@@ -67,6 +67,12 @@ class AssetCatalogService {
   /** Load asset catalog definitions from the given file or directory. */
   void load_from_disk(const CatalogFilePath &file_or_directory_path);
 
+  /**
+   * Duplicate the catalogs from \a other_service into this one. Does not rebuild the tree, this
+   * needs to be done by the caller (call #rebuild_tree()!).
+   */
+  void add_from_existing(const AssetCatalogService &other_service);
+
   /**
    * Write the catalog definitions to disk.
    *
@@ -105,6 +111,15 @@ class AssetCatalogService {
    */
   void reload_catalogs();
 
+  /**
+   * Make sure the tree is updated to the latest collection of catalogs stored in this service.
+   * Does not depend on a CDF file being available so this can be called on a service that stores
+   * catalogs that are not stored in a CDF.
+   * Most API functions that modify catalog data will trigger this, unless otherwise specified (for
+   * batch operations).
+   */
+  void rebuild_tree();
+
   /** Return catalog with the given ID. Return nullptr if not found. */
   AssetCatalog *find_catalog(CatalogID catalog_id) const;
 
@@ -222,7 +237,6 @@ class AssetCatalogService {
       const CatalogFilePath &blend_file_path);
 
   std::unique_ptr<AssetCatalogTree> read_into_tree();
-  void rebuild_tree();
 
   /**
    * For every catalog, ensure that its parent path also has a known catalog.
@@ -270,6 +284,11 @@ class AssetCatalogCollection {
   AssetCatalogCollection(AssetCatalogCollection &&other) noexcept = default;
 
   std::unique_ptr<AssetCatalogCollection> deep_copy() const;
+  /**
+   * Copy the catalogs from \a other and append them to this collection. Copies no other data
+   * otherwise (but marks as having unsaved changes).
+   */
+  void add_catalogs_from_existing(const AssetCatalogCollection &other);
 
  protected:
   static OwningAssetCatalogMap copy_catalog_map(const OwningAssetCatalogMap &orig);

source/blender/asset_system/AS_asset_library.hh

@@ -56,6 +56,8 @@ class AssetLibrary {
    */
   std::unique_ptr<AssetStorage> asset_storage_;
 
+  std::function<void()> on_refresh_;
+
   bCallbackFuncStore on_save_callback_store_{};
 
  public:

source/blender/asset_system/intern/asset_catalog.cc

@@ -323,6 +323,11 @@ void AssetCatalogService::load_from_disk(const CatalogFilePath &file_or_director
   rebuild_tree();
 }
 
+void AssetCatalogService::add_from_existing(const AssetCatalogService &other_service)
+{
+  catalog_collection_->add_catalogs_from_existing(*other_service.catalog_collection_);
+}
+
 void AssetCatalogService::load_directory_recursive(const CatalogFilePath &directory_path)
 {
   /* TODO(@sybren): implement proper multi-file support. For now, just load
@@ -658,15 +663,25 @@ std::unique_ptr<AssetCatalogCollection> AssetCatalogCollection::deep_copy() cons
   return copy;
 }
 
+static void copy_catalog_map_into_existing(const OwningAssetCatalogMap &source,
+                                           OwningAssetCatalogMap &dest)
+{
+  for (const auto &orig_catalog_uptr : source.values()) {
+    auto copy_catalog_uptr = std::make_unique<AssetCatalog>(*orig_catalog_uptr);
+    dest.add_new(copy_catalog_uptr->catalog_id, std::move(copy_catalog_uptr));
+  }
+}
+
+void AssetCatalogCollection::add_catalogs_from_existing(const AssetCatalogCollection &other)
+{
+  has_unsaved_changes_ = true;
+  copy_catalog_map_into_existing(other.catalogs_, catalogs_);
+}
+
 OwningAssetCatalogMap AssetCatalogCollection::copy_catalog_map(const OwningAssetCatalogMap &orig)
 {
   OwningAssetCatalogMap copy;
-
-  for (const auto &orig_catalog_uptr : orig.values()) {
-    auto copy_catalog_uptr = std::make_unique<AssetCatalog>(*orig_catalog_uptr);
-    copy.add_new(copy_catalog_uptr->catalog_id, std::move(copy_catalog_uptr));
-  }
-
+  copy_catalog_map_into_existing(orig, copy);
   return copy;
 }
 

source/blender/asset_system/intern/asset_library.cc

@@ -160,7 +160,9 @@ void AssetLibrary::load_catalogs()
 
 void AssetLibrary::refresh()
 {
-  this->catalog_service->reload_catalogs();
+  if (on_refresh_) {
+    on_refresh_();
+  }
 }
 
 AssetRepresentation &AssetLibrary::add_external_asset(StringRef relative_asset_path,

source/blender/asset_system/intern/asset_library_service.cc

@@ -16,6 +16,7 @@
 
 #include "CLG_log.h"
 
+#include "AS_asset_catalog_tree.hh"
 #include "AS_asset_library.hh"
 #include "asset_library_service.hh"
 #include "utils.hh"
@@ -107,6 +108,8 @@ AssetLibrary *AssetLibraryService::get_asset_library_on_disk(StringRefNull root_
 
   lib->on_blend_save_handler_register();
   lib->load_catalogs();
+  /* Reload catalogs on refresh. */
+  lib->on_refresh_ = [lib]() { lib->catalog_service->reload_catalogs(); };
 
   on_disk_libraries_.add_new(normalized_root_path, std::move(lib_uptr));
   CLOG_INFO(&LOG, 2, "get \"%s\" (loaded)", normalized_root_path.c_str());
@@ -117,6 +120,7 @@ AssetLibrary *AssetLibraryService::get_asset_library_current_file()
 {
   if (current_file_library_) {
     CLOG_INFO(&LOG, 2, "get current file lib (cached)");
+    current_file_library_->refresh();
   }
   else {
     CLOG_INFO(&LOG, 2, "get current file lib (loaded)");
@@ -130,14 +134,7 @@ AssetLibrary *AssetLibraryService::get_asset_library_current_file()
 
 AssetLibrary *AssetLibraryService::get_asset_library_all(const Main *bmain)
 {
-  if (all_library_) {
-    CLOG_INFO(&LOG, 2, "get all lib (cached)");
-  }
-  else {
-    CLOG_INFO(&LOG, 2, "get all lib (loaded)");
-    all_library_ = std::make_unique<AssetLibrary>();
-  }
-
+  /* (Re-)load all other asset libraries. */
   for (AssetLibraryReference &library_ref : all_valid_asset_library_refs()) {
     /* Skip self :) */
     if (library_ref.type == ASSET_LIBRARY_ALL) {
@@ -148,7 +145,32 @@ AssetLibrary *AssetLibraryService::get_asset_library_all(const Main *bmain)
     get_asset_library(bmain, library_ref);
   }
 
-  return all_library_.get();
+  if (all_library_) {
+    CLOG_INFO(&LOG, 2, "get all lib (cached)");
+    all_library_->refresh();
+    return all_library_.get();
+  }
+
+  CLOG_INFO(&LOG, 2, "get all lib (loaded)");
+  all_library_ = std::make_unique<AssetLibrary>();
+
+  AssetLibrary &all_library = *all_library_;
+  auto build_catalogs_fn = [&all_library]() {
+    /* Start with empty catalog storage. */
+    all_library.catalog_service = std::make_unique<AssetCatalogService>();
+
+    /* (Re-)load catalogs on refresh. */
+    AssetLibrary::foreach_loaded([&all_library](AssetLibrary &nested) {
+      nested.catalog_service->reload_catalogs();
+      all_library.catalog_service->add_from_existing(*nested.catalog_service);
+    });
+    all_library.catalog_service->rebuild_tree();
+  };
+
+  build_catalogs_fn();
+  all_library.on_refresh_ = build_catalogs_fn;
+
+  return &all_library;
 }
 
 std::string AssetLibraryService::root_path_from_library_ref(