Integrate "All" library better with the asset system

Now it actually loads data from all asset libraries when this is
selected. The asset representations still need to be loaded by the file
browser backend, this won't change for now.

This adds the concept of nested asset libraries, which I'd prefer to
keep as implementation detail and not expose in the API. But for now
it's needed (for the asset representation loading by the file browser
backend).

source/blender/asset_system/AS_asset_library.hh

@@ -34,7 +34,9 @@ class AssetStorage;
  * to also include asset indexes and more.
  */
 class AssetLibrary {
-  bCallbackFuncStore on_save_callback_store_{};
+  /** If this is an asset library on disk, the top-level directory path. Normalized using
+   * #normalize_directory_path().*/
+  std::string root_path_;
 
   /** Storage for assets (better said their representations) that are considered to be part of this
    * library. Assets are not automatically loaded into this when loading an asset library. Assets
@@ -51,6 +53,12 @@ class AssetLibrary {
    */
   std::unique_ptr<AssetStorage> asset_storage_;
 
+  /** In some cases an asset library is a combination of multiple other ones, these are then
+   * referenced here. "All" asset library only currently. */
+  Vector<AssetLibrary *> nested_libs_; /* Non-owning pointers. */
+
+  bCallbackFuncStore on_save_callback_store_{};
+
  public:
   /* Controlled by #ED_asset_catalogs_set_save_catalogs_when_file_is_saved,
    * for managing the "Save Catalog Changes" in the quit-confirmation dialog box. */
@@ -58,11 +66,16 @@ class AssetLibrary {
 
   std::unique_ptr<AssetCatalogService> catalog_service;
 
+  friend class AssetLibraryService;
+
  public:
-  AssetLibrary();
+  /**
+   * \param root_path: If this is an asset library on disk, the top-level directory path.
+   */
+  AssetLibrary(StringRef root_path = "");
   ~AssetLibrary();
 
-  void load_catalogs(StringRefNull library_root_directory);
+  void load_catalogs();
 
   /** Load catalogs that have changed on disk. */
   void refresh();
@@ -85,6 +98,11 @@ class AssetLibrary {
    *         case when the reference is dangling). */
   bool remove_asset(AssetRepresentation &asset);
 
+  /** In some cases an asset library is a combination of multiple other ones ("All" asset library
+   * only currently). Iterate over the contained asset libraries, executing \a fn for each of them.
+   */
+  void foreach_nested(FunctionRef<void(AssetLibrary &nested_library)> fn);
+
   /**
    * Remap ID pointers for local ID assets, see #BKE_lib_remap.h. When an ID pointer would be
    * mapped to null (typically when an ID gets removed), the asset is removed, because we don't
@@ -105,6 +123,8 @@ class AssetLibrary {
 
   void on_blend_save_post(Main *bmain, PointerRNA **pointers, int num_pointers);
 
+  StringRefNull root_path() const;
+
  private:
   std::optional<int> find_asset_index(const AssetRepresentation &asset);
 };

source/blender/asset_system/CMakeLists.txt

@@ -21,6 +21,7 @@ set(SRC
   intern/asset_library_service.cc
   intern/asset_representation.cc
   intern/asset_storage.cc
+  intern/utils.cc
 
   AS_asset_catalog.hh
   AS_asset_catalog_path.hh
@@ -29,6 +30,7 @@ set(SRC
   AS_asset_representation.hh
   intern/asset_library_service.hh
   intern/asset_storage.hh
+  intern/utils.hh
 
   AS_asset_library.h
   AS_asset_representation.h

source/blender/asset_system/intern/asset_library.cc

@@ -22,6 +22,7 @@
 
 #include "asset_library_service.hh"
 #include "asset_storage.hh"
+#include "utils.hh"
 
 using namespace blender;
 using namespace blender::asset_system;
@@ -126,8 +127,9 @@ void AS_asset_library_remap_ids(const IDRemapper *mappings)
 
 namespace blender::asset_system {
 
-AssetLibrary::AssetLibrary()
-    : asset_storage_(std::make_unique<AssetStorage>()),
+AssetLibrary::AssetLibrary(StringRef root_path)
+    : root_path_(utils::normalize_directory_path(root_path)),
+      asset_storage_(std::make_unique<AssetStorage>()),
       catalog_service(std::make_unique<AssetCatalogService>())
 {
 }
@@ -139,9 +141,9 @@ AssetLibrary::~AssetLibrary()
   }
 }
 
-void AssetLibrary::load_catalogs(StringRefNull library_root_directory)
+void AssetLibrary::load_catalogs()
 {
-  auto catalog_service = std::make_unique<AssetCatalogService>(library_root_directory);
+  auto catalog_service = std::make_unique<AssetCatalogService>(root_path_);
   catalog_service->load_from_disk();
   this->catalog_service = std::move(catalog_service);
 }
@@ -167,6 +169,18 @@ bool AssetLibrary::remove_asset(AssetRepresentation &asset)
   return asset_storage_->remove_asset(asset);
 }
 
+void AssetLibrary::foreach_nested(FunctionRef<void(AssetLibrary &)> fn)
+{
+  for (AssetLibrary *library : nested_libs_) {
+    if (!library) {
+      BLI_assert_unreachable();
+      continue;
+    }
+
+    fn(*library);
+  }
+}
+
 void AssetLibrary::remap_ids_and_remove_invalid(const IDRemapper &mappings)
 {
   asset_storage_->remap_ids_and_remove_invalid(mappings);
@@ -215,6 +229,11 @@ void AssetLibrary::on_blend_save_post(struct Main *main,
   }
 }
 
+StringRefNull AssetLibrary::root_path() const
+{
+  return root_path_;
+}
+
 void AssetLibrary::refresh_catalog_simplename(struct AssetMetaData *asset_data)
 {
   if (BLI_uuid_is_nil(asset_data->catalog_id)) {

source/blender/asset_system/intern/asset_library_service.cc

@@ -4,9 +4,6 @@
  * \ingroup asset_system
  */
 
-#include "asset_library_service.hh"
-#include "AS_asset_library.hh"
-
 #include "BKE_blender.h"
 #include "BKE_preferences.h"
 
@@ -19,6 +16,10 @@
 
 #include "CLG_log.h"
 
+#include "AS_asset_library.hh"
+#include "asset_library_service.hh"
+#include "utils.hh"
+
 /* When enabled, use a pre file load handler (#BKE_CB_EVT_LOAD_PRE) callback to destroy the asset
  * library service. Without this an explicit call from the file loading code is needed to do this,
  * which is not as nice.
@@ -57,69 +58,58 @@ void AssetLibraryService::destroy()
 AssetLibrary *AssetLibraryService::get_asset_library(
     const Main *bmain, const AssetLibraryReference &library_reference)
 {
-  if (library_reference.type == ASSET_LIBRARY_LOCAL) {
-    /* For the "Current File" library  we get the asset library root path based on main. */
-    std::string root_path = bmain ? AS_asset_library_find_suitable_root_path_from_main(bmain) : "";
+  const eAssetLibraryType type = eAssetLibraryType(library_reference.type);
 
-    if (root_path.empty()) {
-      /* File wasn't saved yet. */
-      return get_asset_library_current_file();
-    }
+  switch (type) {
+    case ASSET_LIBRARY_LOCAL: {
+      /* For the "Current File" library  we get the asset library root path based on main. */
+      std::string root_path = bmain ? AS_asset_library_find_suitable_root_path_from_main(bmain) :
+                                      "";
 
-    return get_asset_library_on_disk(root_path);
-  }
-
-  /* TODO */
-  if (library_reference.type == ASSET_LIBRARY_ALL) {
-    return get_asset_library_current_file();
-  }
-
-  if (library_reference.type == ASSET_LIBRARY_CUSTOM) {
-    std::string root_path = root_path_from_library_ref(library_reference);
-
-    if (!root_path.empty()) {
+      if (root_path.empty()) {
+        /* File wasn't saved yet. */
+        return get_asset_library_current_file();
+      }
       return get_asset_library_on_disk(root_path);
     }
+    case ASSET_LIBRARY_ALL:
+      return get_asset_library_all(bmain);
+    case ASSET_LIBRARY_CUSTOM: {
+      std::string root_path = root_path_from_library_ref(library_reference);
+
+      if (!root_path.empty()) {
+        return get_asset_library_on_disk(root_path);
+      }
+    } break;
   }
 
   return nullptr;
 }
 
-namespace {
-std::string normalize_directory_path(StringRefNull directory)
+AssetLibrary *AssetLibraryService::get_asset_library_on_disk(StringRefNull root_path)
 {
-
-  char dir_normalized[PATH_MAX];
-  STRNCPY(dir_normalized, directory.c_str());
-  BLI_path_normalize_dir(nullptr, dir_normalized, sizeof(dir_normalized));
-  return std::string(dir_normalized);
-}
-}  // namespace
-
-AssetLibrary *AssetLibraryService::get_asset_library_on_disk(StringRefNull top_level_directory)
-{
-  BLI_assert_msg(!top_level_directory.is_empty(),
+  BLI_assert_msg(!root_path.is_empty(),
                  "top level directory must be given for on-disk asset library");
 
-  std::string top_dir_trailing_slash = normalize_directory_path(top_level_directory);
+  std::string normalized_root_path = utils::normalize_directory_path(root_path);
 
   std::unique_ptr<AssetLibrary> *lib_uptr_ptr = on_disk_libraries_.lookup_ptr(
-      top_dir_trailing_slash);
+      normalized_root_path);
   if (lib_uptr_ptr != nullptr) {
-    CLOG_INFO(&LOG, 2, "get \"%s\" (cached)", top_dir_trailing_slash.c_str());
+    CLOG_INFO(&LOG, 2, "get \"%s\" (cached)", normalized_root_path.c_str());
     AssetLibrary *lib = lib_uptr_ptr->get();
     lib->refresh();
     return lib;
   }
 
-  std::unique_ptr lib_uptr = std::make_unique<AssetLibrary>();
+  std::unique_ptr lib_uptr = std::make_unique<AssetLibrary>(normalized_root_path);
   AssetLibrary *lib = lib_uptr.get();
 
   lib->on_blend_save_handler_register();
-  lib->load_catalogs(top_dir_trailing_slash);
+  lib->load_catalogs();
 
-  on_disk_libraries_.add_new(top_dir_trailing_slash, std::move(lib_uptr));
-  CLOG_INFO(&LOG, 2, "get \"%s\" (loaded)", top_dir_trailing_slash.c_str());
+  on_disk_libraries_.add_new(normalized_root_path, std::move(lib_uptr));
+  CLOG_INFO(&LOG, 2, "get \"%s\" (loaded)", normalized_root_path.c_str());
   return lib;
 }
 
@@ -138,6 +128,30 @@ AssetLibrary *AssetLibraryService::get_asset_library_current_file()
   return lib;
 }
 
+AssetLibrary *AssetLibraryService::get_asset_library_all(const Main *bmain)
+{
+  if (all_library_) {
+    CLOG_INFO(&LOG, 2, "get all lib (cached)");
+    all_library_->nested_libs_.clear();
+  }
+  else {
+    CLOG_INFO(&LOG, 2, "get all lib (loaded)");
+    all_library_ = std::make_unique<AssetLibrary>();
+  }
+
+  for (AssetLibraryReference &library_ref : all_valid_asset_library_refs()) {
+    /* Skip self :) */
+    if (library_ref.type == ASSET_LIBRARY_ALL) {
+      continue;
+    }
+
+    AssetLibrary *nested_lib = get_asset_library(bmain, library_ref);
+    all_library_->nested_libs_.append(nested_lib);
+  }
+
+  return all_library_.get();
+}
+
 std::string AssetLibraryService::root_path_from_library_ref(
     const AssetLibraryReference &library_reference)
 {
@@ -145,22 +159,16 @@ std::string AssetLibraryService::root_path_from_library_ref(
     return "";
   }
 
-  const char *top_level_directory = nullptr;
-
   BLI_assert(library_reference.type == ASSET_LIBRARY_CUSTOM);
   BLI_assert(library_reference.custom_library_index >= 0);
 
   bUserAssetLibrary *user_library = BKE_preferences_asset_library_find_from_index(
       &U, library_reference.custom_library_index);
-  if (user_library) {
-    top_level_directory = user_library->path;
-  }
-
-  if (!top_level_directory) {
+  if (!user_library || !user_library->path[0]) {
     return "";
   }
 
-  return normalize_directory_path(top_level_directory);
+  return user_library->path;
 }
 
 void AssetLibraryService::allocate_service_instance()

source/blender/asset_system/intern/asset_library_service.hh

@@ -33,12 +33,15 @@ namespace blender::asset_system {
 class AssetLibraryService {
   static std::unique_ptr<AssetLibraryService> instance_;
 
-  /* Mapping absolute path of the library's top-level directory to the AssetLibrary instance. */
+  /* Mapping absolute path of the library's root path (normalize with #normalize_directory_path()!)
+   * the AssetLibrary instance. */
   Map<std::string, std::unique_ptr<AssetLibrary>> on_disk_libraries_;
   /** Library without a known path, i.e. the "Current File" library if the file isn't saved yet. If
    * the file was saved, a valid path for the library can be determined and #on_disk_libraries_
    * above should be used. */
   std::unique_ptr<AssetLibrary> current_file_library_;
+  /** The "all" asset library, merging all other libraries into one. */
+  std::unique_ptr<AssetLibrary> all_library_;
 
   /* Handlers for managing the life cycle of the AssetLibraryService instance. */
   bCallbackFuncStore on_load_callback_store_;
@@ -67,6 +70,9 @@ class AssetLibraryService {
   /** Get the "Current File" asset library. */
   AssetLibrary *get_asset_library_current_file();
 
+  /** Get the "All" asset library, merging all others into one. */
+  AssetLibrary *get_asset_library_all(const Main *bmain);
+
   /** Returns whether there are any known asset libraries with unsaved catalog edits. */
   bool has_any_unsaved_catalogs() const;
 

source/blender/asset_system/intern/utils.cc

@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+/** \file
+ * \ingroup asset_system
+ */
+
+#include "BLI_fileops.h"
+#include "BLI_path_util.h"
+#include "BLI_string.h"
+
+#include "utils.hh"
+
+namespace blender::asset_system::utils {
+
+std::string normalize_directory_path(StringRef directory)
+{
+  if (directory.is_empty()) {
+    return "";
+  }
+
+  char dir_normalized[PATH_MAX];
+  BLI_strncpy(dir_normalized,
+              directory.data(),
+              /* + 1 for null terminator. */
+              std::min(directory.size() + 1, int64_t(sizeof(dir_normalized))));
+  BLI_path_normalize_dir(nullptr, dir_normalized, sizeof(dir_normalized));
+  return std::string(dir_normalized);
+}
+
+}  // namespace blender::asset_system::utils

source/blender/asset_system/intern/utils.hh

@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+/** \file
+ * \ingroup asset_system
+ */
+
+#pragma once
+
+#include "BLI_string_ref.hh"
+
+namespace blender::asset_system::utils {
+
+/**
+ * Returns a normalized directory path with a trailing slash, and a maximum length of #PATH_MAX.
+ * Slashes are not converted to native format (they probably should be though?).
+ */
+std::string normalize_directory_path(StringRef directory);
+
+}  // namespace blender::asset_system::utils

source/blender/editors/space_file/filelist.cc

@@ -320,10 +320,10 @@ static void filelist_readjob_main_assets(FileListReadJob *job_params,
                                          bool *stop,
                                          bool *do_update,
                                          float *progress);
-static void filelist_readjob_all_asset_libraries(FileListReadJob *job_params,
-                                                 bool *stop,
-                                                 bool *do_update,
-                                                 float *progress);
+static void filelist_readjob_all_asset_library(FileListReadJob *job_params,
+                                               bool *stop,
+                                               bool *do_update,
+                                               float *progress);
 
 /* helper, could probably go in BKE actually? */
 static int groupname_to_code(const char *group);
@@ -1781,7 +1781,7 @@ void filelist_settype(FileList *filelist, short type)
       break;
     case FILE_ASSET_LIBRARY_ALL:
       filelist->check_dir_fn = filelist_checkdir_return_always_valid;
-      filelist->read_job_fn = filelist_readjob_all_asset_libraries;
+      filelist->read_job_fn = filelist_readjob_all_asset_library;
       filelist->prepare_filter_fn = prepare_filter_asset_library;
       filelist->filter_fn = is_filtered_asset_library;
       filelist->tags |= FILELIST_TAGS_USES_MAIN_DATA;
@@ -3784,6 +3784,8 @@ static void filelist_readjob_asset_library(FileListReadJob *job_params,
   /* A valid, but empty file-list from now. */
   filelist->filelist.entries_num = 0;
 
+  BLI_assert(job_params->filelist->asset_library_ref != nullptr);
+
   /* NOP if already read. */
   filelist_readjob_load_asset_library_data(job_params, do_update);
 
@@ -3821,10 +3823,10 @@ static void filelist_readjob_main_assets(FileListReadJob *job_params,
   filelist_readjob_main_assets_add_items(job_params, stop, do_update, progress);
 }
 
-static void filelist_readjob_all_asset_libraries(FileListReadJob *job_params,
-                                                 bool *stop,
-                                                 bool *do_update,
-                                                 float *progress)
+static void filelist_readjob_all_asset_library(FileListReadJob *job_params,
+                                               bool *stop,
+                                               bool *do_update,
+                                               float *progress)
 {
   FileList *filelist = job_params->tmp_filelist; /* Use the thread-safe filelist queue. */
   BLI_assert(BLI_listbase_is_empty(&filelist->filelist.entries) &&
@@ -3843,16 +3845,19 @@ static void filelist_readjob_all_asset_libraries(FileListReadJob *job_params,
   }
   /* TODO propertly update progress? */
 
-  for (const AssetLibraryReference &library_ref : asset_system::all_valid_asset_library_refs()) {
-    if (library_ref.type == ASSET_LIBRARY_LOCAL) {
-      /* Already added main assets above. */
-      continue;
+  BLI_assert(filelist->asset_library != nullptr);
+
+  /* Add assets from asset libraries on disk. */
+  filelist->asset_library->foreach_nested([&](asset_system::AssetLibrary &nested_library) {
+    StringRefNull root_path = nested_library.root_path();
+    if (root_path.is_empty()) {
+      return;
     }
-    std::string library_path = AS_asset_library_root_path_from_library_ref(library_ref);
-    BLI_strncpy(filelist->filelist.root, library_path.c_str(), sizeof(filelist->filelist.root));
+
+    BLI_strncpy(filelist->filelist.root, root_path.c_str(), sizeof(filelist->filelist.root));
 
     filelist_readjob_recursive_dir_add_items(true, job_params, stop, do_update, progress);
-  }
+  });
 }
 
 /**