T91406: Asset Library Indexing
Asset library indexing would store indexes of asset files to speed up
asset library browsing.
* Indexes are read when they are up to date
** Index should exist
** Index last modify data should be later than the file it indexes
** Index version should match
* The index of a file containing no assets can be load without opening
the index file. The size of the file should be below a 32 bytes.
* Indexes are stored on a persistent cache folder.
* Unused index files are automatically removed.
The structure of the index files contains all data needed for browsing assets:
```
{
"version": <file version number>,
"entries": [{
"name": "<asset name>",
"catalog_id": "<catalog_id>",
"catalog_name": "<catalog_name>",
"description": "<description>",
"author": "<author>",
"tags": ["<tag>"]
}]
}
```
Reviewed By: sybren, Severin
Maniphest Tasks: T91406
Differential Revision: https://developer.blender.org/D12693
This commit is contained in:
Jeroen Bakker
parent
16fc0da0e7
commit
dbeab82a89
Notes:
blender-bot
2023-02-13 23:16:02 +01:00
Referenced by issue #91406, AssetBrowser: Indexing Asset Library
|
|
@ -24,6 +24,7 @@ set(INC
|
|||
../../makesdna
|
||||
../../makesrna
|
||||
../../windowmanager
|
||||
../../../../intern/clog
|
||||
../../../../intern/guardedalloc
|
||||
)
|
||||
|
||||
|
|
@ -34,6 +35,7 @@ set(SRC
|
|||
intern/asset_catalog.cc
|
||||
intern/asset_filter.cc
|
||||
intern/asset_handle.cc
|
||||
intern/asset_indexer.cc
|
||||
intern/asset_library_reference.cc
|
||||
intern/asset_library_reference_enum.cc
|
||||
intern/asset_list.cc
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup edasset
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "ED_file_indexer.h"
|
||||
|
||||
/**
|
||||
* File Indexer Service for indexing asset files.
|
||||
*
|
||||
* Opening and parsing a large collection of asset files inside a library can take a lot of time.
|
||||
* To reduce the time it takes the files are indexed.
|
||||
*
|
||||
* - Index files are created for each blend file in the asset library, even when the blend file
|
||||
* doesn't contain any assets.
|
||||
* - Indexes are stored in an persistent cache folder (`BKE_appdir_folder_caches` +
|
||||
* `asset_library_indexes/{asset_library_dir}/{asset_index_file.json}`).
|
||||
* - The content of the indexes are used when:
|
||||
* - Index exists and can be opened
|
||||
* - Last modification date is earlier than the file it represents.
|
||||
* - The index file version is the latest.
|
||||
* - Blend files without any assets can be determined by the size of the index file for some
|
||||
* additional performance.
|
||||
*/
|
||||
extern const FileIndexerType file_indexer_asset;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
@ -0,0 +1,781 @@
|
|||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup edasset
|
||||
*/
|
||||
|
||||
#include <fstream>
|
||||
#include <iomanip>
|
||||
#include <optional>
|
||||
|
||||
#include "ED_asset_indexer.h"
|
||||
|
||||
#include "DNA_asset_types.h"
|
||||
#include "DNA_userdef_types.h"
|
||||
|
||||
#include "BLI_fileops.h"
|
||||
#include "BLI_hash.hh"
|
||||
#include "BLI_linklist.h"
|
||||
#include "BLI_path_util.h"
|
||||
#include "BLI_serialize.hh"
|
||||
#include "BLI_set.hh"
|
||||
#include "BLI_string_ref.hh"
|
||||
#include "BLI_uuid.h"
|
||||
|
||||
#include "BKE_appdir.h"
|
||||
#include "BKE_asset.h"
|
||||
#include "BKE_asset_catalog.hh"
|
||||
#include "BKE_preferences.h"
|
||||
|
||||
#include "CLG_log.h"
|
||||
|
||||
static CLG_LogRef LOG = {"ed.asset"};
|
||||
|
||||
namespace blender::ed::asset::index {
|
||||
|
||||
using namespace blender::io::serialize;
|
||||
using namespace blender::bke;
|
||||
|
||||
/**
|
||||
* \file asset_indexer.cc
|
||||
* \brief Indexer for asset libraries.
|
||||
*
|
||||
* Indexes are stored per input file. Each index can contain zero to multiple asset entries.
|
||||
* The indexes are grouped together per asset library. They are stored in
|
||||
* #BKE_appdir_folder_caches +
|
||||
* /asset-library-indices/<asset-library-hash>/<asset-index-hash>_<asset_file>.index.json.
|
||||
*
|
||||
* The structure of an index file is
|
||||
* \code
|
||||
* {
|
||||
* "version": <file version number>,
|
||||
* "entries": [{
|
||||
* "name": "<asset name>",
|
||||
* "catalog_id": "<catalog_id>",
|
||||
* "catalog_name": "<catalog_name>",
|
||||
* "description": "<description>",
|
||||
* "author": "<author>",
|
||||
* "tags": ["<tag>"]
|
||||
* }]
|
||||
* }
|
||||
* \endcode
|
||||
*
|
||||
* NOTE: entries, author, description and tags are optional attributes.
|
||||
*
|
||||
* NOTE: File browser uses name and idcode separate. Inside the index they are joined together like
|
||||
* #ID.name.
|
||||
* NOTE: File browser group name isn't stored in the index as it is a translatable name.
|
||||
*/
|
||||
constexpr StringRef ATTRIBUTE_VERSION("version");
|
||||
constexpr StringRef ATTRIBUTE_ENTRIES("entries");
|
||||
constexpr StringRef ATTRIBUTE_ENTRIES_NAME("name");
|
||||
constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_ID("catalog_id");
|
||||
constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_NAME("catalog_name");
|
||||
constexpr StringRef ATTRIBUTE_ENTRIES_DESCRIPTION("description");
|
||||
constexpr StringRef ATTRIBUTE_ENTRIES_AUTHOR("author");
|
||||
constexpr StringRef ATTRIBUTE_ENTRIES_TAGS("tags");
|
||||
|
||||
/** Abstract class for #BlendFile and #AssetIndexFile. */
|
||||
class AbstractFile {
|
||||
public:
|
||||
virtual ~AbstractFile() = default;
|
||||
|
||||
virtual const char *get_file_path() const = 0;
|
||||
|
||||
bool exists() const
|
||||
{
|
||||
return BLI_exists(get_file_path());
|
||||
}
|
||||
|
||||
size_t get_file_size() const
|
||||
{
|
||||
return BLI_file_size(get_file_path());
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* \brief Reference to a blend file that can be indexed.
|
||||
*/
|
||||
class BlendFile : public AbstractFile {
|
||||
StringRefNull file_path_;
|
||||
|
||||
public:
|
||||
BlendFile(StringRefNull file_path) : file_path_(file_path)
|
||||
{
|
||||
}
|
||||
|
||||
uint64_t hash() const
|
||||
{
|
||||
DefaultHash<StringRefNull> hasher;
|
||||
return hasher(file_path_);
|
||||
}
|
||||
|
||||
std::string get_filename() const
|
||||
{
|
||||
char filename[FILE_MAX];
|
||||
BLI_split_file_part(get_file_path(), filename, sizeof(filename));
|
||||
return std::string(filename);
|
||||
}
|
||||
|
||||
const char *get_file_path() const override
|
||||
{
|
||||
return file_path_.c_str();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* \brief Single entry inside a #AssetIndexFile for reading.
|
||||
*/
|
||||
struct AssetEntryReader {
|
||||
private:
|
||||
/**
|
||||
* \brief Lookup table containing the elements of the entry.
|
||||
*/
|
||||
ObjectValue::Lookup lookup;
|
||||
|
||||
StringRefNull get_name_with_idcode() const
|
||||
{
|
||||
return lookup.lookup(ATTRIBUTE_ENTRIES_NAME)->as_string_value()->value();
|
||||
}
|
||||
|
||||
public:
|
||||
AssetEntryReader(const ObjectValue &entry) : lookup(entry.create_lookup())
|
||||
{
|
||||
}
|
||||
|
||||
ID_Type get_idcode() const
|
||||
{
|
||||
const StringRefNull name_with_idcode = get_name_with_idcode();
|
||||
return GS(name_with_idcode.c_str());
|
||||
}
|
||||
|
||||
StringRef get_name() const
|
||||
{
|
||||
const StringRefNull name_with_idcode = get_name_with_idcode();
|
||||
return name_with_idcode.substr(2);
|
||||
}
|
||||
|
||||
bool has_description() const
|
||||
{
|
||||
return lookup.contains(ATTRIBUTE_ENTRIES_DESCRIPTION);
|
||||
}
|
||||
|
||||
StringRefNull get_description() const
|
||||
{
|
||||
return lookup.lookup(ATTRIBUTE_ENTRIES_DESCRIPTION)->as_string_value()->value();
|
||||
}
|
||||
|
||||
bool has_author() const
|
||||
{
|
||||
return lookup.contains(ATTRIBUTE_ENTRIES_AUTHOR);
|
||||
}
|
||||
|
||||
StringRefNull get_author() const
|
||||
{
|
||||
return lookup.lookup(ATTRIBUTE_ENTRIES_AUTHOR)->as_string_value()->value();
|
||||
}
|
||||
|
||||
StringRefNull get_catalog_name() const
|
||||
{
|
||||
return lookup.lookup(ATTRIBUTE_ENTRIES_CATALOG_NAME)->as_string_value()->value();
|
||||
}
|
||||
|
||||
CatalogID get_catalog_id() const
|
||||
{
|
||||
const std::string &catalog_id =
|
||||
lookup.lookup(ATTRIBUTE_ENTRIES_CATALOG_ID)->as_string_value()->value();
|
||||
CatalogID catalog_uuid(catalog_id);
|
||||
return catalog_uuid;
|
||||
}
|
||||
|
||||
void add_tags_to_meta_data(AssetMetaData *asset_data) const
|
||||
{
|
||||
const ObjectValue::LookupValue *value_ptr = lookup.lookup_ptr(ATTRIBUTE_ENTRIES_TAGS);
|
||||
if (value_ptr == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ArrayValue *array_value = (*value_ptr)->as_array_value();
|
||||
const ArrayValue::Items &elements = array_value->elements();
|
||||
for (const ArrayValue::Item &item : elements) {
|
||||
const StringRefNull tag_name = item->as_string_value()->value();
|
||||
BKE_asset_metadata_tag_add(asset_data, tag_name.c_str());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
struct AssetEntryWriter {
|
||||
private:
|
||||
ObjectValue::Items &attributes;
|
||||
|
||||
public:
|
||||
AssetEntryWriter(ObjectValue &entry) : attributes(entry.elements())
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief add id + name to the attributes.
|
||||
*
|
||||
* NOTE: id and name are encoded like #ID.name
|
||||
*/
|
||||
void add_id_name(const short idcode, const StringRefNull name)
|
||||
{
|
||||
char idcode_prefix[2];
|
||||
/* Similar to `BKE_libblock_alloc`. */
|
||||
*((short *)idcode_prefix) = idcode;
|
||||
std::string name_with_idcode = std::string(idcode_prefix) + name;
|
||||
|
||||
attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_NAME, new StringValue(name_with_idcode)));
|
||||
}
|
||||
|
||||
void add_catalog_id(const CatalogID &catalog_id)
|
||||
{
|
||||
char catalog_id_str[UUID_STRING_LEN];
|
||||
BLI_uuid_format(catalog_id_str, catalog_id);
|
||||
attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_CATALOG_ID, new StringValue(catalog_id_str)));
|
||||
}
|
||||
|
||||
void add_catalog_name(const StringRefNull catalog_name)
|
||||
{
|
||||
attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_CATALOG_NAME, new StringValue(catalog_name)));
|
||||
}
|
||||
|
||||
void add_description(const StringRefNull description)
|
||||
{
|
||||
attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_DESCRIPTION, new StringValue(description)));
|
||||
}
|
||||
|
||||
void add_author(const StringRefNull author)
|
||||
{
|
||||
attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_AUTHOR, new StringValue(author)));
|
||||
}
|
||||
|
||||
void add_tags(const ListBase /* AssetTag */ *asset_tags)
|
||||
{
|
||||
ArrayValue *tags = new ArrayValue();
|
||||
attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_TAGS, tags));
|
||||
ArrayValue::Items &tag_items = tags->elements();
|
||||
|
||||
LISTBASE_FOREACH (AssetTag *, tag, asset_tags) {
|
||||
tag_items.append_as(new StringValue(tag->name));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static void init_value_from_file_indexer_entry(AssetEntryWriter &result,
|
||||
const FileIndexerEntry *indexer_entry)
|
||||
{
|
||||
const BLODataBlockInfo &datablock_info = indexer_entry->datablock_info;
|
||||
|
||||
result.add_id_name(indexer_entry->idcode, datablock_info.name);
|
||||
|
||||
const AssetMetaData &asset_data = *datablock_info.asset_data;
|
||||
result.add_catalog_id(asset_data.catalog_id);
|
||||
result.add_catalog_name(asset_data.catalog_simple_name);
|
||||
|
||||
if (asset_data.description != nullptr) {
|
||||
result.add_description(asset_data.description);
|
||||
}
|
||||
if (asset_data.author != nullptr) {
|
||||
result.add_author(asset_data.author);
|
||||
}
|
||||
|
||||
if (!BLI_listbase_is_empty(&asset_data.tags)) {
|
||||
result.add_tags(&asset_data.tags);
|
||||
}
|
||||
|
||||
/* TODO: asset_data.IDProperties */
|
||||
}
|
||||
|
||||
static void init_value_from_file_indexer_entries(ObjectValue &result,
|
||||
const FileIndexerEntries &indexer_entries)
|
||||
{
|
||||
ArrayValue *entries = new ArrayValue();
|
||||
ArrayValue::Items &items = entries->elements();
|
||||
|
||||
for (LinkNode *ln = indexer_entries.entries; ln; ln = ln->next) {
|
||||
const FileIndexerEntry *indexer_entry = static_cast<const FileIndexerEntry *>(ln->link);
|
||||
/* We also get non asset types (brushes, workspaces), when browsing using the asset browser. */
|
||||
if (indexer_entry->datablock_info.asset_data == nullptr) {
|
||||
continue;
|
||||
}
|
||||
ObjectValue *entry_value = new ObjectValue();
|
||||
AssetEntryWriter entry(*entry_value);
|
||||
init_value_from_file_indexer_entry(entry, indexer_entry);
|
||||
items.append_as(entry_value);
|
||||
}
|
||||
|
||||
/* When no entries to index, we should not store the entries attribute as this would make the
|
||||
* size bigger than the #MIN_FILE_SIZE_WITH_ENTRIES. */
|
||||
if (items.is_empty()) {
|
||||
delete entries;
|
||||
return;
|
||||
}
|
||||
|
||||
ObjectValue::Items &attributes = result.elements();
|
||||
attributes.append_as(std::pair(ATTRIBUTE_ENTRIES, entries));
|
||||
}
|
||||
|
||||
static void init_indexer_entry_from_value(FileIndexerEntry &indexer_entry,
|
||||
const AssetEntryReader &entry)
|
||||
{
|
||||
indexer_entry.idcode = entry.get_idcode();
|
||||
|
||||
const std::string &name = entry.get_name();
|
||||
BLI_strncpy(
|
||||
indexer_entry.datablock_info.name, name.c_str(), sizeof(indexer_entry.datablock_info.name));
|
||||
|
||||
AssetMetaData *asset_data = BKE_asset_metadata_create();
|
||||
indexer_entry.datablock_info.asset_data = asset_data;
|
||||
|
||||
if (entry.has_description()) {
|
||||
const std::string &description = entry.get_description();
|
||||
char *description_c_str = static_cast<char *>(MEM_mallocN(description.length() + 1, __func__));
|
||||
BLI_strncpy(description_c_str, description.c_str(), description.length() + 1);
|
||||
asset_data->description = description_c_str;
|
||||
}
|
||||
if (entry.has_author()) {
|
||||
const std::string &author = entry.get_author();
|
||||
char *author_c_str = static_cast<char *>(MEM_mallocN(author.length() + 1, __func__));
|
||||
BLI_strncpy(author_c_str, author.c_str(), author.length() + 1);
|
||||
asset_data->author = author_c_str;
|
||||
}
|
||||
|
||||
const std::string &catalog_name = entry.get_catalog_name();
|
||||
BLI_strncpy(asset_data->catalog_simple_name,
|
||||
catalog_name.c_str(),
|
||||
sizeof(asset_data->catalog_simple_name));
|
||||
|
||||
asset_data->catalog_id = entry.get_catalog_id();
|
||||
|
||||
entry.add_tags_to_meta_data(asset_data);
|
||||
}
|
||||
|
||||
static int init_indexer_entries_from_value(FileIndexerEntries &indexer_entries,
|
||||
const ObjectValue &value)
|
||||
{
|
||||
const ObjectValue::Lookup attributes = value.create_lookup();
|
||||
const ObjectValue::LookupValue *entries_value = attributes.lookup_ptr(ATTRIBUTE_ENTRIES);
|
||||
BLI_assert(entries_value != nullptr);
|
||||
|
||||
if (entries_value == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int num_entries_read = 0;
|
||||
const ArrayValue::Items elements = (*entries_value)->as_array_value()->elements();
|
||||
for (ArrayValue::Item element : elements) {
|
||||
const AssetEntryReader asset_entry(*element->as_object_value());
|
||||
|
||||
FileIndexerEntry *entry = static_cast<FileIndexerEntry *>(
|
||||
MEM_callocN(sizeof(FileIndexerEntry), __func__));
|
||||
init_indexer_entry_from_value(*entry, asset_entry);
|
||||
|
||||
BLI_linklist_prepend(&indexer_entries.entries, entry);
|
||||
num_entries_read += 1;
|
||||
}
|
||||
|
||||
return num_entries_read;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief References the asset library directory.
|
||||
*
|
||||
* The #AssetLibraryIndex instance is used to keep track of unused file indices. When reading any
|
||||
* used indices are removed from the list and when reading is finished the unused
|
||||
* indices are removed.
|
||||
*/
|
||||
struct AssetLibraryIndex {
|
||||
/**
|
||||
* Tracks indices that haven't been used yet.
|
||||
*
|
||||
* Contains absolute paths to the indices.
|
||||
*/
|
||||
Set<std::string> unused_file_indices;
|
||||
|
||||
/**
|
||||
* \brief Absolute path where the indices of `library` are stored.
|
||||
*
|
||||
* \NOTE: includes trailing directory separator.
|
||||
*/
|
||||
std::string indices_base_path;
|
||||
|
||||
std::string library_path;
|
||||
|
||||
public:
|
||||
AssetLibraryIndex(const StringRef library_path) : library_path(library_path)
|
||||
{
|
||||
init_indices_base_path();
|
||||
}
|
||||
|
||||
uint64_t hash() const
|
||||
{
|
||||
DefaultHash<StringRefNull> hasher;
|
||||
return hasher(get_library_file_path());
|
||||
}
|
||||
|
||||
StringRefNull get_library_file_path() const
|
||||
{
|
||||
return library_path;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Initializes #AssetLibraryIndex.indices_base_path.
|
||||
*
|
||||
* `BKE_appdir_folder_caches/asset-library-indices/<asset-library-name-hash>/`
|
||||
*/
|
||||
void init_indices_base_path()
|
||||
{
|
||||
char index_path[FILE_MAX];
|
||||
BKE_appdir_folder_caches(index_path, sizeof(index_path));
|
||||
|
||||
BLI_path_append(index_path, sizeof(index_path), "asset-library-indices");
|
||||
|
||||
std::stringstream ss;
|
||||
ss << std::setfill('0') << std::setw(16) << std::hex << hash() << "/";
|
||||
BLI_path_append(index_path, sizeof(index_path), ss.str().c_str());
|
||||
|
||||
indices_base_path = std::string(index_path);
|
||||
}
|
||||
|
||||
/**
|
||||
* \return absolute path to the index file of the given `asset_file`.
|
||||
*
|
||||
* `{indices_base_path}/{asset-file_hash}_{asset-file-filename}.index.json`.
|
||||
*/
|
||||
std::string index_file_path(const BlendFile &asset_file) const
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << indices_base_path;
|
||||
ss << std::setfill('0') << std::setw(16) << std::hex << asset_file.hash() << "_"
|
||||
<< asset_file.get_filename() << ".index.json";
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize to keep track of unused file indices.
|
||||
*/
|
||||
void init_unused_index_files()
|
||||
{
|
||||
const char *index_path = indices_base_path.c_str();
|
||||
if (!BLI_is_dir(index_path)) {
|
||||
return;
|
||||
}
|
||||
struct direntry *dir_entries = nullptr;
|
||||
int num_entries = BLI_filelist_dir_contents(index_path, &dir_entries);
|
||||
for (int i = 0; i < num_entries; i++) {
|
||||
struct direntry *entry = &dir_entries[i];
|
||||
if (BLI_str_endswith(entry->relname, ".index.json")) {
|
||||
unused_file_indices.add_as(std::string(entry->path));
|
||||
}
|
||||
}
|
||||
|
||||
BLI_filelist_free(dir_entries, num_entries);
|
||||
}
|
||||
|
||||
void mark_as_used(const std::string &filename)
|
||||
{
|
||||
unused_file_indices.remove(filename);
|
||||
}
|
||||
|
||||
int remove_unused_index_files() const
|
||||
{
|
||||
int num_files_deleted = 0;
|
||||
for (const std::string &unused_index : unused_file_indices) {
|
||||
const char *file_path = unused_index.c_str();
|
||||
CLOG_INFO(&LOG, 2, "Remove unused index file [%s].", file_path);
|
||||
BLI_delete(file_path, false, false);
|
||||
num_files_deleted++;
|
||||
}
|
||||
|
||||
return num_files_deleted;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Instance of this class represents the contents of an asset index file.
|
||||
*
|
||||
* /code
|
||||
* {
|
||||
* "version": {version},
|
||||
* "entries": ...
|
||||
* }
|
||||
* /endcode
|
||||
*/
|
||||
struct AssetIndex {
|
||||
/**
|
||||
* \brief Version to store in new index files.
|
||||
*
|
||||
* Versions are written to each index file. When reading the version is checked against
|
||||
* `CURRENT_VERSION` to make sure we can use the index. Developer should increase
|
||||
* `CURRENT_VERSION` when changes are made to the structure of the stored index.
|
||||
*/
|
||||
static const int CURRENT_VERSION = 1;
|
||||
|
||||
/**
|
||||
* Version number to use when version couldn't be read from an index file.
|
||||
*/
|
||||
const int UNKNOWN_VERSION = -1;
|
||||
|
||||
/**
|
||||
* `blender::io::serialize::Value` represeting the contents of an index file.
|
||||
*
|
||||
* Value is used over ObjectValue as the contents of the index could be corrupted and doesn't
|
||||
* represent an object. In case corrupted files are detected the `get_version` would return
|
||||
* UNKNOWN_VERSION.`
|
||||
*/
|
||||
std::unique_ptr<Value> contents;
|
||||
|
||||
/**
|
||||
* Constructor for when creating/updating an asset index file.
|
||||
* #AssetIndex.contents are filled from the given \p indexer_entries.
|
||||
*/
|
||||
AssetIndex(const FileIndexerEntries &indexer_entries)
|
||||
{
|
||||
std::unique_ptr<ObjectValue> root = std::make_unique<ObjectValue>();
|
||||
ObjectValue::Items &root_attributes = root->elements();
|
||||
root_attributes.append_as(std::pair(ATTRIBUTE_VERSION, new IntValue(CURRENT_VERSION)));
|
||||
init_value_from_file_indexer_entries(*root, indexer_entries);
|
||||
|
||||
contents = std::move(root);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor when reading an asset index file.
|
||||
* #AssetIndex.contents are read from the given \p value.
|
||||
*/
|
||||
AssetIndex(std::unique_ptr<Value> &value) : contents(std::move(value))
|
||||
{
|
||||
}
|
||||
|
||||
int get_version() const
|
||||
{
|
||||
const ObjectValue *root = contents->as_object_value();
|
||||
if (root == nullptr) {
|
||||
return UNKNOWN_VERSION;
|
||||
}
|
||||
const ObjectValue::Lookup attributes = root->create_lookup();
|
||||
const ObjectValue::LookupValue *version_value = attributes.lookup_ptr(ATTRIBUTE_VERSION);
|
||||
if (version_value == nullptr) {
|
||||
return UNKNOWN_VERSION;
|
||||
}
|
||||
return (*version_value)->as_int_value()->value();
|
||||
}
|
||||
|
||||
bool is_latest_version() const
|
||||
{
|
||||
return get_version() == CURRENT_VERSION;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the contents of this index into the given \p indexer_entries.
|
||||
*
|
||||
* \return The number of entries read from the given entries.
|
||||
*/
|
||||
int extract_into(FileIndexerEntries &indexer_entries) const
|
||||
{
|
||||
const ObjectValue *root = contents->as_object_value();
|
||||
const int num_entries_read = init_indexer_entries_from_value(indexer_entries, *root);
|
||||
return num_entries_read;
|
||||
}
|
||||
};
|
||||
|
||||
class AssetIndexFile : public AbstractFile {
|
||||
public:
|
||||
AssetLibraryIndex &library_index;
|
||||
/**
|
||||
* Asset index files with a size smaller than this attribute would be considered to not contain
|
||||
* any entries.
|
||||
*/
|
||||
const size_t MIN_FILE_SIZE_WITH_ENTRIES = 32;
|
||||
std::string filename;
|
||||
|
||||
AssetIndexFile(AssetLibraryIndex &library_index, BlendFile &asset_filename)
|
||||
: library_index(library_index), filename(library_index.index_file_path(asset_filename))
|
||||
{
|
||||
}
|
||||
|
||||
void mark_as_used()
|
||||
{
|
||||
library_index.mark_as_used(filename);
|
||||
}
|
||||
|
||||
const char *get_file_path() const override
|
||||
{
|
||||
return filename.c_str();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether the index file is older than the given asset file.
|
||||
*/
|
||||
bool is_older_than(BlendFile &asset_file) const
|
||||
{
|
||||
return BLI_file_older(get_file_path(), asset_file.get_file_path());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the index file contains entries without opening the file.
|
||||
*/
|
||||
bool constains_entries() const
|
||||
{
|
||||
const size_t file_size = get_file_size();
|
||||
return file_size >= MIN_FILE_SIZE_WITH_ENTRIES;
|
||||
}
|
||||
|
||||
std::unique_ptr<AssetIndex> read_contents() const
|
||||
{
|
||||
JsonFormatter formatter;
|
||||
std::ifstream is;
|
||||
is.open(filename);
|
||||
std::unique_ptr<Value> read_data = formatter.deserialize(is);
|
||||
is.close();
|
||||
|
||||
return std::make_unique<AssetIndex>(read_data);
|
||||
}
|
||||
|
||||
bool ensure_parent_path_exists() const
|
||||
{
|
||||
/* `BLI_make_existing_file` only ensures parent path, otherwise than expected from the name of
|
||||
* the function. */
|
||||
return BLI_make_existing_file(get_file_path());
|
||||
}
|
||||
|
||||
void write_contents(AssetIndex &content)
|
||||
{
|
||||
JsonFormatter formatter;
|
||||
if (!ensure_parent_path_exists()) {
|
||||
CLOG_ERROR(&LOG, "Index not created: couldn't create folder [%s].", get_file_path());
|
||||
return;
|
||||
}
|
||||
|
||||
std::ofstream os;
|
||||
os.open(filename, std::ios::out | std::ios::trunc);
|
||||
formatter.serialize(os, *content.contents);
|
||||
os.close();
|
||||
}
|
||||
};
|
||||
|
||||
static eFileIndexerResult read_index(const char *filename,
|
||||
FileIndexerEntries *entries,
|
||||
int *r_read_entries_len,
|
||||
void *user_data)
|
||||
{
|
||||
AssetLibraryIndex &library_index = *static_cast<AssetLibraryIndex *>(user_data);
|
||||
BlendFile asset_file(filename);
|
||||
AssetIndexFile asset_index_file(library_index, asset_file);
|
||||
|
||||
if (!asset_index_file.exists()) {
|
||||
return FILE_INDEXER_NEEDS_UPDATE;
|
||||
}
|
||||
|
||||
/* Mark index as used, even when it will be recreated. When not done it would remove the index
|
||||
* when the indexing has finished (see `AssetLibraryIndex.remove_unused_index_files`), thereby
|
||||
* removing the newly created index.
|
||||
*/
|
||||
asset_index_file.mark_as_used();
|
||||
|
||||
if (asset_index_file.is_older_than(asset_file)) {
|
||||
CLOG_INFO(
|
||||
&LOG,
|
||||
3,
|
||||
"Asset index file [%s] needs to be refreshed as it is older than the asset file [%s].",
|
||||
asset_index_file.filename.c_str(),
|
||||
filename);
|
||||
return FILE_INDEXER_NEEDS_UPDATE;
|
||||
}
|
||||
|
||||
if (!asset_index_file.constains_entries()) {
|
||||
CLOG_INFO(&LOG,
|
||||
3,
|
||||
"Asset file index is to small to contain any entries. [%s]",
|
||||
asset_index_file.filename.c_str());
|
||||
*r_read_entries_len = 0;
|
||||
return FILE_INDEXER_ENTRIES_LOADED;
|
||||
}
|
||||
|
||||
std::unique_ptr<AssetIndex> contents = asset_index_file.read_contents();
|
||||
if (!contents->is_latest_version()) {
|
||||
CLOG_INFO(&LOG,
|
||||
3,
|
||||
"Asset file index is ignored; expected version %d but file is version %d [%s].",
|
||||
AssetIndex::CURRENT_VERSION,
|
||||
contents->get_version(),
|
||||
asset_index_file.filename.c_str());
|
||||
return FILE_INDEXER_NEEDS_UPDATE;
|
||||
}
|
||||
|
||||
const int read_entries_len = contents->extract_into(*entries);
|
||||
CLOG_INFO(&LOG, 1, "Read %d entries from asset index for [%s].", read_entries_len, filename);
|
||||
*r_read_entries_len = read_entries_len;
|
||||
|
||||
return FILE_INDEXER_ENTRIES_LOADED;
|
||||
}
|
||||
|
||||
static void update_index(const char *filename, FileIndexerEntries *entries, void *user_data)
|
||||
{
|
||||
AssetLibraryIndex &library_index = *static_cast<AssetLibraryIndex *>(user_data);
|
||||
BlendFile asset_file(filename);
|
||||
AssetIndexFile asset_index_file(library_index, asset_file);
|
||||
CLOG_INFO(&LOG,
|
||||
1,
|
||||
"Update asset index for [%s] store index in [%s].",
|
||||
asset_file.get_file_path(),
|
||||
asset_index_file.get_file_path());
|
||||
|
||||
AssetIndex content(*entries);
|
||||
asset_index_file.write_contents(content);
|
||||
}
|
||||
|
||||
static void *init_user_data(const char *root_directory, size_t root_directory_maxlen)
|
||||
{
|
||||
AssetLibraryIndex *library_index = OBJECT_GUARDED_NEW(
|
||||
AssetLibraryIndex,
|
||||
StringRef(root_directory, BLI_strnlen(root_directory, root_directory_maxlen)));
|
||||
library_index->init_unused_index_files();
|
||||
return library_index;
|
||||
}
|
||||
|
||||
static void free_user_data(void *user_data)
|
||||
{
|
||||
OBJECT_GUARDED_DELETE(user_data, AssetLibraryIndex);
|
||||
}
|
||||
|
||||
static void filelist_finished(void *user_data)
|
||||
{
|
||||
AssetLibraryIndex &library_index = *static_cast<AssetLibraryIndex *>(user_data);
|
||||
const int num_indices_removed = library_index.remove_unused_index_files();
|
||||
if (num_indices_removed > 0) {
|
||||
CLOG_INFO(&LOG, 1, "Removed %d unused indices.", num_indices_removed);
|
||||
}
|
||||
}
|
||||
|
||||
constexpr FileIndexerType asset_indexer()
|
||||
{
|
||||
FileIndexerType indexer = {nullptr};
|
||||
indexer.read_index = read_index;
|
||||
indexer.update_index = update_index;
|
||||
indexer.init_user_data = init_user_data;
|
||||
indexer.free_user_data = free_user_data;
|
||||
indexer.filelist_finished = filelist_finished;
|
||||
return indexer;
|
||||
}
|
||||
|
||||
} // namespace blender::ed::asset::index
|
||||
|
||||
extern "C" {
|
||||
const FileIndexerType file_indexer_asset = blender::ed::asset::index::asset_indexer();
|
||||
}
|
||||
|
|
@ -44,6 +44,7 @@
|
|||
#include "../space_file/filelist.h"
|
||||
|
||||
#include "ED_asset_handle.h"
|
||||
#include "ED_asset_indexer.h"
|
||||
#include "ED_asset_list.h"
|
||||
#include "ED_asset_list.hh"
|
||||
#include "asset_library_reference.hh"
|
||||
|
|
@ -168,6 +169,7 @@ void AssetList::setup()
|
|||
true,
|
||||
"",
|
||||
"");
|
||||
filelist_setindexer(files, &file_indexer_asset);
|
||||
|
||||
char path[FILE_MAXDIR] = "";
|
||||
if (user_library) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,153 @@
|
|||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup edfile
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "BLO_readfile.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* File indexing for the file/asset browser.
|
||||
*
|
||||
* This file contains an API to create indexing functionality when listing blend files in
|
||||
* the file browser.
|
||||
*
|
||||
* To implement a custom indexer a `FileIndexerType` struct should be made and passed to the
|
||||
* `filelist_setindexer` function.
|
||||
*/
|
||||
|
||||
struct AssetLibraryReference;
|
||||
struct LinkNode;
|
||||
|
||||
/**
|
||||
* Result code of the `read_index` callback.
|
||||
*/
|
||||
typedef enum eFileIndexerResult {
|
||||
/**
|
||||
* File listing entries are loaded from the index. Reading entries from the blend file itself
|
||||
* should be skipped.
|
||||
*/
|
||||
FILE_INDEXER_ENTRIES_LOADED,
|
||||
|
||||
/**
|
||||
* Index isn't available or not upto date. Entries should be read from te blend file and
|
||||
* `update_index` must be called to update the index.
|
||||
*/
|
||||
FILE_INDEXER_NEEDS_UPDATE,
|
||||
} eFileIndexerResult;
|
||||
|
||||
/**
|
||||
* FileIndexerEntry contains all data that is required to create a file listing entry.
|
||||
*/
|
||||
typedef struct FileIndexerEntry {
|
||||
struct BLODataBlockInfo datablock_info;
|
||||
short idcode;
|
||||
} FileIndexerEntry;
|
||||
|
||||
/**
|
||||
* Contains all entries of a blend file.
|
||||
*/
|
||||
typedef struct FileIndexerEntries {
|
||||
struct LinkNode /* FileIndexerEntry */ *entries;
|
||||
} FileIndexerEntries;
|
||||
|
||||
typedef void *(*FileIndexerInitUserDataFunc)(const char *root_directory,
|
||||
size_t root_directory_maxlen);
|
||||
typedef void (*FileIndexerFreeUserDataFunc)(void *);
|
||||
typedef void (*FileIndexerFinishedFunc)(void *);
|
||||
typedef eFileIndexerResult (*FileIndexerReadIndexFunc)(const char *file_name,
|
||||
FileIndexerEntries *entries,
|
||||
int *r_read_entries_len,
|
||||
void *user_data);
|
||||
typedef void (*FileIndexerUpdateIndexFunc)(const char *file_name,
|
||||
FileIndexerEntries *entries,
|
||||
void *user_data);
|
||||
|
||||
typedef struct FileIndexerType {
|
||||
/**
|
||||
* Is called at the beginning of the file listing process. An indexer can
|
||||
* setup needed data. The result of this function will be passed around as `user_data` parameter.
|
||||
*
|
||||
* This is an optional callback.
|
||||
*/
|
||||
FileIndexerInitUserDataFunc init_user_data;
|
||||
|
||||
/**
|
||||
* Is called at the end of the file listing process. An indexer can free the data that it created
|
||||
* during the file listing process.
|
||||
*
|
||||
* This is an optional callback */
|
||||
FileIndexerFreeUserDataFunc free_user_data;
|
||||
|
||||
/**
|
||||
* Is called at the end of the file listing process (before the `free_user_data`) where indexes
|
||||
* can perform clean-ups.
|
||||
*
|
||||
* This is an optinal callback. Called when listing files completed.
|
||||
*/
|
||||
FileIndexerFinishedFunc filelist_finished;
|
||||
|
||||
/**
|
||||
* Is called for each blend file being listed to read data from the index.
|
||||
*
|
||||
* Read entries should be added to given `entries` parameter (type: `FileIndexerEntries`).
|
||||
* `*r_read_entries_len` must be set to the number of read entries.
|
||||
* and the function must return `eFileIndexerResult::FILE_INDEXER_ENTRIES_LOADED`.
|
||||
* In this case the blend file will not be opened and the FileIndexerEntry added to `entries`
|
||||
* will be used as the content of the file.
|
||||
*
|
||||
* When the index isn't available or could not be used no entries must be added to the
|
||||
* entries field, `r_read_entries_len` must be set to `0` and the function must return
|
||||
* `eFileIndexerResult::FILE_INDEXER_NEEDS_UPDATE`. In this case the blend file will read from
|
||||
* the blend file and the `update_index` function will be called.
|
||||
*/
|
||||
FileIndexerReadIndexFunc read_index;
|
||||
|
||||
/**
|
||||
* Update an index of a blend file.
|
||||
*
|
||||
* Is called after reading entries from the file when the result of `read_index` was
|
||||
* `eFileIndexerResult::FILE_INDEXER_NEED_UPDATE`. The callback should update the index so the
|
||||
* next time that read_index is called it will read the entries from the index.
|
||||
*/
|
||||
FileIndexerUpdateIndexFunc update_index;
|
||||
} FileIndexerType;
|
||||
|
||||
/* file_indexer.cc */
|
||||
|
||||
/** Removes all entries inside the given `indexer_entries`. */
|
||||
void ED_file_indexer_entries_clear(FileIndexerEntries *indexer_entries);
|
||||
|
||||
/**
|
||||
* Adds all entries from the given `datablock_infos` to the `indexer_entries`.
|
||||
* The datablock_infos must only contain data for a single IDType. The specific IDType must be
|
||||
* passed in the `idcode` parameter.
|
||||
*/
|
||||
void ED_file_indexer_entries_extend_from_datablock_infos(
|
||||
FileIndexerEntries *indexer_entries,
|
||||
const LinkNode * /* BLODataBlockInfo */ datablock_infos,
|
||||
const int idcode);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
set(INC
|
||||
../include
|
||||
../asset
|
||||
../../blenfont
|
||||
../../blenkernel
|
||||
../../blenlib
|
||||
|
|
@ -36,6 +37,7 @@ set(INC
|
|||
set(SRC
|
||||
asset_catalog_tree_view.cc
|
||||
file_draw.c
|
||||
file_indexer.cc
|
||||
file_ops.c
|
||||
file_panels.c
|
||||
file_utils.c
|
||||
|
|
@ -44,6 +46,7 @@ set(SRC
|
|||
fsmenu.c
|
||||
space_file.c
|
||||
|
||||
file_indexer.h
|
||||
file_intern.h
|
||||
filelist.h
|
||||
fsmenu.h
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup edfile
|
||||
*
|
||||
* This file implements the default file browser indexer and has some helper function to work with
|
||||
* `FileIndexerEntries`.
|
||||
*/
|
||||
#include "file_indexer.h"
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
#include "BLI_linklist.h"
|
||||
#include "BLI_listbase.h"
|
||||
#include "BLI_string.h"
|
||||
#include "BLI_utildefines.h"
|
||||
|
||||
namespace blender::ed::file::indexer {
|
||||
|
||||
static eFileIndexerResult read_index(const char *UNUSED(file_name),
|
||||
FileIndexerEntries *UNUSED(entries),
|
||||
int *UNUSED(r_read_entries_len),
|
||||
void *UNUSED(user_data))
|
||||
{
|
||||
return FILE_INDEXER_NEEDS_UPDATE;
|
||||
}
|
||||
|
||||
static void update_index(const char *UNUSED(file_name),
|
||||
FileIndexerEntries *UNUSED(entries),
|
||||
void *UNUSED(user_data))
|
||||
{
|
||||
}
|
||||
|
||||
constexpr FileIndexerType default_indexer()
|
||||
{
|
||||
FileIndexerType indexer = {nullptr};
|
||||
indexer.read_index = read_index;
|
||||
indexer.update_index = update_index;
|
||||
return indexer;
|
||||
}
|
||||
|
||||
static FileIndexerEntry *file_indexer_entry_create_from_datablock_info(
|
||||
const BLODataBlockInfo *datablock_info, const int idcode)
|
||||
{
|
||||
FileIndexerEntry *entry = static_cast<FileIndexerEntry *>(
|
||||
MEM_mallocN(sizeof(FileIndexerEntry), __func__));
|
||||
entry->datablock_info = *datablock_info;
|
||||
entry->idcode = idcode;
|
||||
return entry;
|
||||
}
|
||||
|
||||
} // namespace blender::ed::file::indexer
|
||||
|
||||
extern "C" {
|
||||
|
||||
void ED_file_indexer_entries_extend_from_datablock_infos(
|
||||
FileIndexerEntries *indexer_entries,
|
||||
const LinkNode * /* BLODataBlockInfo */ datablock_infos,
|
||||
const int idcode)
|
||||
{
|
||||
for (const LinkNode *ln = datablock_infos; ln; ln = ln->next) {
|
||||
const BLODataBlockInfo *datablock_info = static_cast<const BLODataBlockInfo *>(ln->link);
|
||||
FileIndexerEntry *file_indexer_entry =
|
||||
blender::ed::file::indexer::file_indexer_entry_create_from_datablock_info(datablock_info,
|
||||
idcode);
|
||||
BLI_linklist_prepend(&indexer_entries->entries, file_indexer_entry);
|
||||
}
|
||||
}
|
||||
|
||||
static void ED_file_indexer_entry_free(void *indexer_entry)
|
||||
{
|
||||
MEM_freeN(indexer_entry);
|
||||
}
|
||||
|
||||
void ED_file_indexer_entries_clear(FileIndexerEntries *indexer_entries)
|
||||
{
|
||||
BLI_linklist_free(indexer_entries->entries, ED_file_indexer_entry_free);
|
||||
indexer_entries->entries = nullptr;
|
||||
}
|
||||
|
||||
const FileIndexerType file_indexer_noop = blender::ed::file::indexer::default_indexer();
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* \ingroup edfile
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "ED_file_indexer.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Default indexer to use when listing files. The implementation is a no-operation indexing. When
|
||||
* set it won't use indexing. It is added to increase the code clarity.
|
||||
*/
|
||||
extern const FileIndexerType file_indexer_noop;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
@ -89,6 +89,7 @@
|
|||
|
||||
#include "atomic_ops.h"
|
||||
|
||||
#include "file_indexer.h"
|
||||
#include "file_intern.h"
|
||||
#include "filelist.h"
|
||||
|
||||
|
|
@ -396,6 +397,11 @@ typedef struct FileList {
|
|||
|
||||
FileListFilter filter_data;
|
||||
|
||||
/**
|
||||
* File indexer to use. Attribute is always set.
|
||||
*/
|
||||
const struct FileIndexerType *indexer;
|
||||
|
||||
struct FileListIntern filelist_intern;
|
||||
|
||||
struct FileListEntryCache filelist_cache;
|
||||
|
|
@ -1125,6 +1131,19 @@ void filelist_setfilter_options(FileList *filelist,
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the indexer to be used by the filelist.
|
||||
*
|
||||
* The given indexer allocation should be handled by the caller or defined statically.
|
||||
*/
|
||||
void filelist_setindexer(FileList *filelist, const FileIndexerType *indexer)
|
||||
{
|
||||
BLI_assert(filelist);
|
||||
BLI_assert(indexer);
|
||||
|
||||
filelist->indexer = indexer;
|
||||
}
|
||||
|
||||
/**
|
||||
* \param catalog_id: The catalog that should be filtered by if \a catalog_visibility is
|
||||
* #FILE_SHOW_ASSETS_FROM_CATALOG. May be NULL otherwise.
|
||||
|
|
@ -1830,6 +1849,8 @@ FileList *filelist_new(short type)
|
|||
p->filelist.nbr_entries = FILEDIR_NBR_ENTRIES_UNSET;
|
||||
filelist_settype(p, type);
|
||||
|
||||
p->indexer = &file_indexer_noop;
|
||||
|
||||
return p;
|
||||
}
|
||||
|
||||
|
|
@ -3120,6 +3141,29 @@ static FileListInternEntry *filelist_readjob_list_lib_group_create(const int idc
|
|||
return entry;
|
||||
}
|
||||
|
||||
static void filelist_readjob_list_lib_add_datablock(ListBase *entries,
|
||||
const BLODataBlockInfo *datablock_info,
|
||||
const bool prefix_relpath_with_group_name,
|
||||
const int idcode,
|
||||
const char *group_name)
|
||||
{
|
||||
FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__);
|
||||
if (prefix_relpath_with_group_name) {
|
||||
entry->relpath = BLI_sprintfN("%s/%s", group_name, datablock_info->name);
|
||||
}
|
||||
else {
|
||||
entry->relpath = BLI_strdup(datablock_info->name);
|
||||
}
|
||||
entry->typeflag |= FILE_TYPE_BLENDERLIB;
|
||||
if (datablock_info && datablock_info->asset_data) {
|
||||
entry->typeflag |= FILE_TYPE_ASSET;
|
||||
/* Moves ownership! */
|
||||
entry->imported_asset_data = datablock_info->asset_data;
|
||||
}
|
||||
entry->blentype = idcode;
|
||||
BLI_addtail(entries, entry);
|
||||
}
|
||||
|
||||
static void filelist_readjob_list_lib_add_datablocks(ListBase *entries,
|
||||
LinkNode *datablock_infos,
|
||||
const bool prefix_relpath_with_group_name,
|
||||
|
|
@ -3127,29 +3171,71 @@ static void filelist_readjob_list_lib_add_datablocks(ListBase *entries,
|
|||
const char *group_name)
|
||||
{
|
||||
for (LinkNode *ln = datablock_infos; ln; ln = ln->next) {
|
||||
struct BLODataBlockInfo *info = ln->link;
|
||||
FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__);
|
||||
if (prefix_relpath_with_group_name) {
|
||||
entry->relpath = BLI_sprintfN("%s/%s", group_name, info->name);
|
||||
}
|
||||
else {
|
||||
entry->relpath = BLI_strdup(info->name);
|
||||
}
|
||||
entry->typeflag |= FILE_TYPE_BLENDERLIB;
|
||||
if (info && info->asset_data) {
|
||||
entry->typeflag |= FILE_TYPE_ASSET;
|
||||
/* Moves ownership! */
|
||||
entry->imported_asset_data = info->asset_data;
|
||||
}
|
||||
entry->blentype = idcode;
|
||||
BLI_addtail(entries, entry);
|
||||
struct BLODataBlockInfo *datablock_info = ln->link;
|
||||
filelist_readjob_list_lib_add_datablock(
|
||||
entries, datablock_info, prefix_relpath_with_group_name, idcode, group_name);
|
||||
}
|
||||
}
|
||||
|
||||
static void filelist_readjob_list_lib_add_from_indexer_entries(
|
||||
ListBase *entries,
|
||||
const FileIndexerEntries *indexer_entries,
|
||||
const bool prefix_relpath_with_group_name)
|
||||
{
|
||||
for (const LinkNode *ln = indexer_entries->entries; ln; ln = ln->next) {
|
||||
const FileIndexerEntry *indexer_entry = (const FileIndexerEntry *)ln->link;
|
||||
const char *group_name = BKE_idtype_idcode_to_name(indexer_entry->idcode);
|
||||
filelist_readjob_list_lib_add_datablock(entries,
|
||||
&indexer_entry->datablock_info,
|
||||
prefix_relpath_with_group_name,
|
||||
indexer_entry->idcode,
|
||||
group_name);
|
||||
}
|
||||
}
|
||||
|
||||
static FileListInternEntry *filelist_readjob_list_lib_navigate_to_parent_entry_create(void)
|
||||
{
|
||||
FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__);
|
||||
entry->relpath = BLI_strdup(FILENAME_PARENT);
|
||||
entry->typeflag |= (FILE_TYPE_BLENDERLIB | FILE_TYPE_DIR);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structure to keep the file indexer and its user data together.
|
||||
*/
|
||||
typedef struct FileIndexer {
|
||||
const FileIndexerType *callbacks;
|
||||
|
||||
/**
|
||||
* User data. Contains the result of `callbacks.init_user_data`.
|
||||
*/
|
||||
void *user_data;
|
||||
} FileIndexer;
|
||||
|
||||
static int filelist_readjob_list_lib_populate_from_index(ListBase *entries,
|
||||
const ListLibOptions options,
|
||||
const int read_from_index,
|
||||
const FileIndexerEntries *indexer_entries)
|
||||
{
|
||||
int navigate_to_parent_len = 0;
|
||||
if (options & LIST_LIB_ADD_PARENT) {
|
||||
FileListInternEntry *entry = filelist_readjob_list_lib_navigate_to_parent_entry_create();
|
||||
BLI_addtail(entries, entry);
|
||||
navigate_to_parent_len = 1;
|
||||
}
|
||||
|
||||
filelist_readjob_list_lib_add_from_indexer_entries(entries, indexer_entries, true);
|
||||
return read_from_index + navigate_to_parent_len;
|
||||
}
|
||||
|
||||
static int filelist_readjob_list_lib(const char *root,
|
||||
ListBase *entries,
|
||||
const ListLibOptions options)
|
||||
const ListLibOptions options,
|
||||
FileIndexer *indexer_runtime)
|
||||
{
|
||||
BLI_assert(indexer_runtime);
|
||||
|
||||
char dir[FILE_MAX_LIBEXTRA], *group;
|
||||
|
||||
struct BlendHandle *libfiledata = NULL;
|
||||
|
|
@ -3157,13 +3243,37 @@ static int filelist_readjob_list_lib(const char *root,
|
|||
/* Check if the given root is actually a library. All folders are passed to
|
||||
* `filelist_readjob_list_lib` and based on the number of found entries `filelist_readjob_do`
|
||||
* will do a dir listing only when this function does not return any entries. */
|
||||
/* TODO: We should consider introducing its own function to detect if it is a lib and
|
||||
/* TODO(jbakker): We should consider introducing its own function to detect if it is a lib and
|
||||
* call it directly from `filelist_readjob_do` to increase readability. */
|
||||
const bool is_lib = BLO_library_path_explode(root, dir, &group, NULL);
|
||||
if (!is_lib) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const bool group_came_from_path = group != NULL;
|
||||
|
||||
/* Try read from indexer_runtime. */
|
||||
/* Indexing returns all entries in a blend file. We should ignore the index when listing a group
|
||||
* inside a blend file, so the `entries` isn't filled with undesired entries.
|
||||
* This happens when linking or appending data-blocks, where you can navigate into a group (fe
|
||||
* Materials/Objects) where you only want to work with partial indexes.
|
||||
*
|
||||
* Adding support for partial reading/updating indexes would increase the complexity.
|
||||
*/
|
||||
const bool use_indexer = !group_came_from_path;
|
||||
FileIndexerEntries indexer_entries = {NULL};
|
||||
if (use_indexer) {
|
||||
int read_from_index = 0;
|
||||
eFileIndexerResult indexer_result = indexer_runtime->callbacks->read_index(
|
||||
dir, &indexer_entries, &read_from_index, indexer_runtime->user_data);
|
||||
if (indexer_result == FILE_INDEXER_ENTRIES_LOADED) {
|
||||
int entries_read = filelist_readjob_list_lib_populate_from_index(
|
||||
entries, options, read_from_index, &indexer_entries);
|
||||
ED_file_indexer_entries_clear(&indexer_entries);
|
||||
return entries_read;
|
||||
}
|
||||
}
|
||||
|
||||
/* Open the library file. */
|
||||
BlendFileReadReport bf_reports = {.reports = NULL};
|
||||
libfiledata = BLO_blendhandle_from_file(dir, &bf_reports);
|
||||
|
|
@ -3172,18 +3282,18 @@ static int filelist_readjob_list_lib(const char *root,
|
|||
}
|
||||
|
||||
/* Add current parent when requested. */
|
||||
int parent_len = 0;
|
||||
/* Is the navigate to previous level added to the list of entries. When added the return value
|
||||
* should be increased to match the actual number of entries added. It is introduced to keep
|
||||
* the code clean and readable and not counting in a single variable. */
|
||||
int navigate_to_parent_len = 0;
|
||||
if (options & LIST_LIB_ADD_PARENT) {
|
||||
FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__);
|
||||
entry->relpath = BLI_strdup(FILENAME_PARENT);
|
||||
entry->typeflag |= (FILE_TYPE_BLENDERLIB | FILE_TYPE_DIR);
|
||||
FileListInternEntry *entry = filelist_readjob_list_lib_navigate_to_parent_entry_create();
|
||||
BLI_addtail(entries, entry);
|
||||
parent_len = 1;
|
||||
navigate_to_parent_len = 1;
|
||||
}
|
||||
|
||||
int group_len = 0;
|
||||
int datablock_len = 0;
|
||||
const bool group_came_from_path = group != NULL;
|
||||
if (group_came_from_path) {
|
||||
const int idcode = groupname_to_code(group);
|
||||
LinkNode *datablock_infos = BLO_blendhandle_get_datablock_info(
|
||||
|
|
@ -3208,6 +3318,10 @@ static int filelist_readjob_list_lib(const char *root,
|
|||
libfiledata, idcode, options & LIST_LIB_ASSETS_ONLY, &group_datablock_len);
|
||||
filelist_readjob_list_lib_add_datablocks(
|
||||
entries, group_datablock_infos, true, idcode, group_name);
|
||||
if (use_indexer) {
|
||||
ED_file_indexer_entries_extend_from_datablock_infos(
|
||||
&indexer_entries, group_datablock_infos, idcode);
|
||||
}
|
||||
BLI_linklist_freeN(group_datablock_infos);
|
||||
datablock_len += group_datablock_len;
|
||||
}
|
||||
|
|
@ -3218,8 +3332,14 @@ static int filelist_readjob_list_lib(const char *root,
|
|||
|
||||
BLO_blendhandle_close(libfiledata);
|
||||
|
||||
/* Update the index. */
|
||||
if (use_indexer) {
|
||||
indexer_runtime->callbacks->update_index(dir, &indexer_entries, indexer_runtime->user_data);
|
||||
ED_file_indexer_entries_clear(&indexer_entries);
|
||||
}
|
||||
|
||||
/* Return the number of items added to entries. */
|
||||
int added_entries_len = group_len + datablock_len + parent_len;
|
||||
int added_entries_len = group_len + datablock_len + navigate_to_parent_len;
|
||||
return added_entries_len;
|
||||
}
|
||||
|
||||
|
|
@ -3411,11 +3531,11 @@ typedef struct FileListReadJob {
|
|||
* The job system calls #filelist_readjob_update which moves any read file from #tmp_filelist
|
||||
* into #filelist in a thread-safe way.
|
||||
*
|
||||
* #tmp_filelist also keeps an `AssetLibrary *` so that it can be loaded in the same thread, and
|
||||
* moved to #filelist once all categories are loaded.
|
||||
* #tmp_filelist also keeps an `AssetLibrary *` so that it can be loaded in the same thread,
|
||||
* and moved to #filelist once all categories are loaded.
|
||||
*
|
||||
* NOTE: #tmp_filelist is freed in #filelist_readjob_free, so any copied pointers need to be set
|
||||
* to NULL to avoid double-freeing them. */
|
||||
* NOTE: #tmp_filelist is freed in #filelist_readjob_free, so any copied pointers need to be
|
||||
* set to NULL to avoid double-freeing them. */
|
||||
struct FileList *tmp_filelist;
|
||||
} FileListReadJob;
|
||||
|
||||
|
|
@ -3452,8 +3572,8 @@ static bool filelist_readjob_should_recurse_into_entry(const int max_recursion,
|
|||
/* No more levels of recursion left. */
|
||||
return false;
|
||||
}
|
||||
/* Show entries when recursion is set to `Blend file` even when `current_recursion_level` exceeds
|
||||
* `max_recursion`. */
|
||||
/* Show entries when recursion is set to `Blend file` even when `current_recursion_level`
|
||||
* exceeds `max_recursion`. */
|
||||
if (!is_lib && (current_recursion_level >= max_recursion) &&
|
||||
((entry->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) == 0)) {
|
||||
return false;
|
||||
|
|
@ -3501,6 +3621,12 @@ static void filelist_readjob_recursive_dir_add_items(const bool do_lib,
|
|||
BLI_path_normalize_dir(job_params->main_name, dir);
|
||||
td_dir->dir = BLI_strdup(dir);
|
||||
|
||||
/* Init the file indexer. */
|
||||
FileIndexer indexer_runtime = {.callbacks = filelist->indexer};
|
||||
if (indexer_runtime.callbacks->init_user_data) {
|
||||
indexer_runtime.user_data = indexer_runtime.callbacks->init_user_data(dir, sizeof(dir));
|
||||
}
|
||||
|
||||
while (!BLI_stack_is_empty(todo_dirs) && !(*stop)) {
|
||||
FileListInternEntry *entry;
|
||||
int nbr_entries = 0;
|
||||
|
|
@ -3543,7 +3669,8 @@ static void filelist_readjob_recursive_dir_add_items(const bool do_lib,
|
|||
if (filelist->asset_library_ref) {
|
||||
list_lib_options |= LIST_LIB_ASSETS_ONLY;
|
||||
}
|
||||
nbr_entries = filelist_readjob_list_lib(subdir, &entries, list_lib_options);
|
||||
nbr_entries = filelist_readjob_list_lib(
|
||||
subdir, &entries, list_lib_options, &indexer_runtime);
|
||||
if (nbr_entries > 0) {
|
||||
is_lib = true;
|
||||
}
|
||||
|
|
@ -3585,6 +3712,15 @@ static void filelist_readjob_recursive_dir_add_items(const bool do_lib,
|
|||
MEM_freeN(subdir);
|
||||
}
|
||||
|
||||
/* Finalize and free indexer. */
|
||||
if (indexer_runtime.callbacks->filelist_finished && BLI_stack_is_empty(todo_dirs)) {
|
||||
indexer_runtime.callbacks->filelist_finished(indexer_runtime.user_data);
|
||||
}
|
||||
if (indexer_runtime.callbacks->free_user_data && indexer_runtime.user_data) {
|
||||
indexer_runtime.callbacks->free_user_data(indexer_runtime.user_data);
|
||||
indexer_runtime.user_data = NULL;
|
||||
}
|
||||
|
||||
/* If we were interrupted by stop, stack may not be empty and we need to free
|
||||
* pending dir paths. */
|
||||
while (!BLI_stack_is_empty(todo_dirs)) {
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ extern "C" {
|
|||
|
||||
struct AssetLibraryReference;
|
||||
struct BlendHandle;
|
||||
struct FileIndexerType;
|
||||
struct FileList;
|
||||
struct FileSelection;
|
||||
struct bUUID;
|
||||
|
|
@ -72,6 +73,7 @@ void filelist_setfilter_options(struct FileList *filelist,
|
|||
const bool filter_assets_only,
|
||||
const char *filter_glob,
|
||||
const char *filter_search);
|
||||
void filelist_setindexer(struct FileList *filelist, const struct FileIndexerType *indexer);
|
||||
void filelist_set_asset_catalog_filter_options(
|
||||
struct FileList *filelist,
|
||||
eFileSel_Params_AssetCatalogVisibility catalog_visibility,
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@
|
|||
#include "MEM_guardedalloc.h"
|
||||
|
||||
#include "BLI_blenlib.h"
|
||||
#include "BLI_linklist.h"
|
||||
#include "BLI_utildefines.h"
|
||||
|
||||
#include "BKE_appdir.h"
|
||||
|
|
@ -43,6 +44,7 @@
|
|||
#include "WM_message.h"
|
||||
#include "WM_types.h"
|
||||
|
||||
#include "ED_asset_indexer.h"
|
||||
#include "ED_asset.h"
|
||||
#include "ED_fileselect.h"
|
||||
#include "ED_screen.h"
|
||||
|
|
@ -353,6 +355,10 @@ static void file_refresh(const bContext *C, ScrArea *area)
|
|||
sfile->files, asset_params->asset_catalog_visibility, &asset_params->catalog_id);
|
||||
}
|
||||
|
||||
if (ED_fileselect_is_asset_browser(sfile)) {
|
||||
filelist_setindexer(sfile->files, &file_indexer_asset);
|
||||
}
|
||||
|
||||
/* Update the active indices of bookmarks & co. */
|
||||
sfile->systemnr = fsmenu_get_active_indices(fsmenu, FS_CATEGORY_SYSTEM, params->dir);
|
||||
sfile->system_bookmarknr = fsmenu_get_active_indices(
|
||||
|
|
|
|||
Loading…
Reference in New Issue