Blenloader: New API that will be used for blenloader decentralization

Design Task: T76372

This part of a larger refactoring towards a more extensible architecture
in Blender: T75724

The API is defined in `BLO_read_write.h`. It adds the small data structures
`BlendWriter`, `BlendDataReader`, `BlendLibReader` and `BlendExpander`.
Those contain context about the current read/write operation. Furthermore,
it adds many functions with the prefixes `BLO_write_*`, `BLO_read_*` and
`BLO_expand_*`.

Lib linking and expanding will probably be handled by the more generic libquery
system eventually. The corresponding parts of the API can be removed then.
This commit is contained in:
Jacques Lucke 2020-06-05 11:44:36 +02:00
parent ce7409fd13
commit 48075b2c05
4 changed files with 482 additions and 0 deletions

View File

@ -0,0 +1,207 @@
/*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
/** \file
* \ingroup blenloader
*
* This file contains an API that allows different parts of Blender to define what data is stored
* in .blend files.
*
* Four callbacks have to be provided to fully implement .blend I/O for a piece of data. One of
* those is related to file writing and three for file reading. Reading requires multiple
* callbacks, due to the way linking between files works.
*
* Brief description of the individual callbacks:
* - Blend Write: Define which structs and memory buffers are saved.
* - Blend Read Data: Loads structs and memory buffers from file and updates pointers them.
* - Blend Read Lib: Updates pointers to ID data blocks.
* - Blend Expand: Defines which other data blocks should be loaded (possibly from other files).
*
* Each of these callbacks uses a different API functions.
*
* Some parts of Blender, e.g. modifiers, don't require you to implement all four callbacks.
* Instead only the first two are necessary. The other two are handled by general ID management. In
* the future, we might want to get rid of those two callbacks entirely, but for now they are
* necessary.
*/
#ifndef __BLO_READ_WRITE_H__
#define __BLO_READ_WRITE_H__
#ifdef __cplusplus
extern "C" {
#endif
typedef struct BlendWriter BlendWriter;
typedef struct BlendDataReader BlendDataReader;
typedef struct BlendLibReader BlendLibReader;
typedef struct BlendExpander BlendExpander;
/* Blend Write API
* ===============
*
* Most functions fall into one of two categories. Either they write a DNA struct or a raw memory
* buffer to the .blend file.
*
* It is safe to pass NULL as data_ptr. In this case nothing will be stored.
*
* DNA Struct Writing
* ------------------
*
* Functions dealing with DNA structs begin with BLO_write_struct_*.
*
* DNA struct types can be identified in different ways:
* - Run-time Name: The name is provided as const char *.
* - Compile-time Name: The name is provided at compile time. This can be more efficient. Note
* that this optimization is not implemented currently.
* - Struct ID: Every DNA struct type has an integer ID that can be queried with
* BLO_get_struct_id_by_name. Providing this ID can be a useful optimization when many structs
* of the same type are stored AND if those structs are not in a continuous array.
*
* Often only a single instance of a struct is written at once. However, sometimes it is necessary
* to write arrays or linked lists. Separate functions for that are provided as well.
*
* There is a special macro for writing id structs: BLO_write_id_struct. Those are handled
* differently from other structs.
*
* Raw Data Writing
* ----------------
*
* At the core there is BLO_write_raw, which can write arbitrary memory buffers to the file. The
* code that reads this data might have to correct its byte-order. For the common cases there are
* convenience functions that write and read arrays of simple types such as int32. Those will
* correct endianness automatically.
*/
/* Mapping between names and ids. */
int BLO_get_struct_id_by_name(BlendWriter *writer, const char *struct_name);
#define BLO_get_struct_id(writer, struct_name) BLO_get_struct_id_by_name(writer, #struct_name)
/* Write single struct. */
void BLO_write_struct_by_name(BlendWriter *writer, const char *struct_name, const void *data_ptr);
void BLO_write_struct_by_id(BlendWriter *writer, int struct_id, const void *data_ptr);
#define BLO_write_struct(writer, struct_name, data_ptr) \
BLO_write_struct_by_id(writer, BLO_get_struct_id(writer, struct_name), data_ptr)
/* Write struct array. */
void BLO_write_struct_array_by_name(BlendWriter *writer,
const char *struct_name,
int array_size,
const void *data_ptr);
void BLO_write_struct_array_by_id(BlendWriter *writer,
int struct_id,
int array_size,
const void *data_ptr);
#define BLO_write_struct_array(writer, struct_name, array_size, data_ptr) \
BLO_write_struct_array_by_id( \
writer, BLO_get_struct_id(writer, struct_name), array_size, data_ptr)
/* Write struct list. */
void BLO_write_struct_list_by_name(BlendWriter *writer,
const char *struct_name,
struct ListBase *list);
void BLO_write_struct_list_by_id(BlendWriter *writer, int struct_id, struct ListBase *list);
#define BLO_write_struct_list(writer, struct_name, list_ptr) \
BLO_write_struct_list_by_id(writer, BLO_get_struct_id(writer, struct_name), list_ptr)
/* Write id struct. */
void blo_write_id_struct(BlendWriter *writer,
int struct_id,
const void *id_address,
const struct ID *id);
#define BLO_write_id_struct(writer, struct_name, id_address, id) \
blo_write_id_struct(writer, BLO_get_struct_id(writer, struct_name), id_address, id)
/* Write raw data. */
void BLO_write_raw(BlendWriter *writer, int size_in_bytes, const void *data_ptr);
void BLO_write_int32_array(BlendWriter *writer, int size, const int32_t *data_ptr);
void BLO_write_uint32_array(BlendWriter *writer, int size, const uint32_t *data_ptr);
void BLO_write_float_array(BlendWriter *writer, int size, const float *data_ptr);
void BLO_write_float3_array(BlendWriter *writer, int size, const float *data_ptr);
void BLO_write_string(BlendWriter *writer, const char *data_ptr);
/* Misc. */
bool BLO_write_is_undo(BlendWriter *writer);
/* Blend Read Data API
* ===================
*
* Generally, for every BLO_write_* call there should be a corresponding BLO_read_* call.
*
* Most BLO_read_* functions get a pointer to a pointer as argument. That allows the function to
* update the pointer to its new value.
*
* When the given pointer points to a memory buffer that was not stored in the file, the pointer is
* updated to be NULL. When it was pointing to NULL before, it will stay that way.
*
* Examples of matching calls:
* BLO_write_struct(writer, ClothSimSettings, clmd->sim_parms);
* BLO_read_data_address(reader, &clmd->sim_parms);
*
* BLO_write_struct_list(writer, TimeMarker, &action->markers);
* BLO_read_list(reader, &action->markers, NULL);
*
* BLO_write_int32_array(writer, hmd->totindex, hmd->indexar);
* BLO_read_int32_array(reader, hmd->totindex, &hmd->indexar);
*/
void *BLO_read_get_new_data_address(BlendDataReader *reader, const void *old_address);
#define BLO_read_data_address(reader, ptr_p) \
*(ptr_p) = BLO_read_get_new_data_address((reader), *(ptr_p))
typedef void (*BlendReadListFn)(BlendDataReader *reader, void *data);
void BLO_read_list(BlendDataReader *reader, struct ListBase *list, BlendReadListFn callback);
/* Update data pointers and correct byte-order if necessary. */
void BLO_read_int32_array(BlendDataReader *reader, int array_size, int32_t **ptr_p);
void BLO_read_uint32_array(BlendDataReader *reader, int array_size, uint32_t **ptr_p);
void BLO_read_float_array(BlendDataReader *reader, int array_size, float **ptr_p);
void BLO_read_float3_array(BlendDataReader *reader, int array_size, float **ptr_p);
void BLO_read_double_array(BlendDataReader *reader, int array_size, double **ptr_p);
void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p);
/* Misc. */
bool BLO_read_requires_endian_switch(BlendDataReader *reader);
/* Blend Read Lib API
* ===================
*
* This API does almost the same as the Blend Read Data API. However, now only pointers to ID data
* blocks are updated.
*/
ID *BLO_read_get_new_id_address(BlendLibReader *reader, struct Library *lib, struct ID *id);
#define BLO_read_id_address(reader, lib, id_ptr_p) \
*(id_ptr_p) = (void *)BLO_read_get_new_id_address((reader), (lib), (ID *)*(id_ptr_p))
/* Blend Expand API
* ===================
*
* BLO_expand has to be called for every data block that should be loaded. If the data block is in
* a separate .blend file, it will be pulled from there.
*/
void BLO_expand_id(BlendExpander *expander, struct ID *id);
#define BLO_expand(expander, id) BLO_expand_id(expander, (struct ID *)id)
#ifdef __cplusplus
}
#endif
#endif /* __BLO_READ_WRITE_H__ */

View File

@ -64,6 +64,7 @@ set(SRC
BLO_blend_defs.h
BLO_blend_validate.h
BLO_readfile.h
BLO_read_write.h
BLO_undofile.h
BLO_writefile.h
intern/readfile.h

View File

@ -160,6 +160,7 @@
#include "BLO_blend_defs.h"
#include "BLO_blend_validate.h"
#include "BLO_read_write.h"
#include "BLO_readfile.h"
#include "BLO_undofile.h"
@ -713,6 +714,20 @@ static Main *blo_find_main(FileData *fd, const char *filepath, const char *relab
/** \name File Parsing
* \{ */
typedef struct BlendDataReader {
FileData *fd;
} BlendDataReader;
typedef struct BlendLibReader {
FileData *fd;
Main *main;
} BlendLibReader;
typedef struct BlendExpander {
FileData *fd;
Main *main;
} BlendExpander;
static void switch_endian_bh4(BHead4 *bhead)
{
/* the ID_.. codes */
@ -12665,4 +12680,164 @@ static void read_libraries(FileData *basefd, ListBase *mainlist)
BKE_main_free(main_newid);
}
void *BLO_read_get_new_data_address(BlendDataReader *reader, const void *old_address)
{
return newdataadr(reader->fd, old_address);
}
ID *BLO_read_get_new_id_address(BlendLibReader *reader, Library *lib, ID *id)
{
return newlibadr(reader->fd, lib, id);
}
bool BLO_read_requires_endian_switch(BlendDataReader *reader)
{
return (reader->fd->flags & FD_FLAGS_SWITCH_ENDIAN) != 0;
}
/**
* Updates all ->prev and ->next pointers of the list elements.
* Updates the list->first and list->last pointers.
* When not NULL, calls the callback on every element.
*/
void BLO_read_list(BlendDataReader *reader, ListBase *list, BlendReadListFn callback)
{
if (BLI_listbase_is_empty(list)) {
return;
}
BLO_read_data_address(reader, &list->first);
if (callback != NULL) {
callback(reader, list->first);
}
Link *ln = list->first;
Link *prev = NULL;
while (ln) {
BLO_read_data_address(reader, &ln->next);
if (ln->next != NULL && callback != NULL) {
callback(reader, ln->next);
}
ln->prev = prev;
prev = ln;
ln = ln->next;
}
list->last = prev;
}
void BLO_read_int32_array(BlendDataReader *reader, int array_size, int32_t **ptr_p)
{
BLO_read_data_address(reader, ptr_p);
if (BLO_read_requires_endian_switch(reader)) {
BLI_endian_switch_int32_array(*ptr_p, array_size);
}
}
void BLO_read_uint32_array(BlendDataReader *reader, int array_size, uint32_t **ptr_p)
{
BLO_read_data_address(reader, ptr_p);
if (BLO_read_requires_endian_switch(reader)) {
BLI_endian_switch_uint32_array(*ptr_p, array_size);
}
}
void BLO_read_float_array(BlendDataReader *reader, int array_size, float **ptr_p)
{
BLO_read_data_address(reader, ptr_p);
if (BLO_read_requires_endian_switch(reader)) {
BLI_endian_switch_float_array(*ptr_p, array_size);
}
}
void BLO_read_float3_array(BlendDataReader *reader, int array_size, float **ptr_p)
{
BLO_read_float_array(reader, array_size * 3, ptr_p);
}
void BLO_read_double_array(BlendDataReader *reader, int array_size, double **ptr_p)
{
BLO_read_data_address(reader, ptr_p);
if (BLO_read_requires_endian_switch(reader)) {
BLI_endian_switch_double_array(*ptr_p, array_size);
}
}
static void convert_pointer_array_64_to_32(BlendDataReader *reader,
uint array_size,
const uint64_t *src,
uint32_t *dst)
{
/* Match pointer conversion rules from bh4_from_bh8 and cast_pointer. */
if (BLO_read_requires_endian_switch(reader)) {
for (int i = 0; i < array_size; i++) {
uint64_t ptr = src[i];
BLI_endian_switch_uint64(&ptr);
dst[i] = (uint32_t)(ptr >> 3);
}
}
else {
for (int i = 0; i < array_size; i++) {
dst[i] = (uint32_t)(src[i] >> 3);
}
}
}
static void convert_pointer_array_32_to_64(BlendDataReader *UNUSED(reader),
uint array_size,
const uint32_t *src,
uint64_t *dst)
{
/* Match pointer conversion rules from bh8_from_bh4 and cast_pointer. */
for (int i = 0; i < array_size; i++) {
dst[i] = src[i];
}
}
void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p)
{
FileData *fd = reader->fd;
void *orig_array = newdataadr(fd, *ptr_p);
if (orig_array == NULL) {
*ptr_p = NULL;
return;
}
int file_pointer_size = fd->filesdna->pointer_size;
int current_pointer_size = fd->memsdna->pointer_size;
/* Overallocation is fine, but might be better to pass the length as parameter. */
int array_size = MEM_allocN_len(orig_array) / file_pointer_size;
void *final_array = NULL;
if (file_pointer_size == current_pointer_size) {
/* No pointer conversion necessary. */
final_array = orig_array;
}
else if (file_pointer_size == 8 && current_pointer_size == 4) {
/* Convert pointers from 64 to 32 bit. */
final_array = MEM_malloc_arrayN(array_size, 4, "new pointer array");
convert_pointer_array_64_to_32(
reader, array_size, (uint64_t *)orig_array, (uint32_t *)final_array);
MEM_freeN(orig_array);
}
else if (file_pointer_size == 4 && current_pointer_size == 8) {
/* Convert pointers from 32 to 64 bit. */
final_array = MEM_malloc_arrayN(array_size, 8, "new pointer array");
convert_pointer_array_32_to_64(
reader, array_size, (uint32_t *)orig_array, (uint64_t *)final_array);
MEM_freeN(orig_array);
}
else {
BLI_assert(false);
}
*ptr_p = final_array;
}
void BLO_expand_id(BlendExpander *expander, ID *id)
{
expand_doit(expander->fd, expander->main, id);
}
/** \} */

View File

@ -177,6 +177,7 @@
#include "BLO_blend_defs.h"
#include "BLO_blend_validate.h"
#include "BLO_read_write.h"
#include "BLO_readfile.h"
#include "BLO_undofile.h"
#include "BLO_writefile.h"
@ -339,6 +340,10 @@ typedef struct {
WriteWrap *ww;
} WriteData;
typedef struct BlendWriter {
WriteData *wd;
} BlendWriter;
static WriteData *writedata_new(WriteWrap *ww)
{
WriteData *wd = MEM_callocN(sizeof(*wd), "writedata");
@ -4536,4 +4541,98 @@ bool BLO_write_file_mem(Main *mainvar, MemFile *compare, MemFile *current, int w
return (err == 0);
}
void BLO_write_raw(BlendWriter *writer, int size_in_bytes, const void *data_ptr)
{
writedata(writer->wd, DATA, size_in_bytes, data_ptr);
}
void BLO_write_struct_by_name(BlendWriter *writer, const char *struct_name, const void *data_ptr)
{
int struct_id = BLO_get_struct_id_by_name(writer, struct_name);
BLO_write_struct_by_id(writer, struct_id, data_ptr);
}
void BLO_write_struct_array_by_name(BlendWriter *writer,
const char *struct_name,
int array_size,
const void *data_ptr)
{
int struct_id = BLO_get_struct_id_by_name(writer, struct_name);
BLO_write_struct_array_by_id(writer, struct_id, array_size, data_ptr);
}
void BLO_write_struct_by_id(BlendWriter *writer, int struct_id, const void *data_ptr)
{
writestruct_nr(writer->wd, DATA, struct_id, 1, data_ptr);
}
void BLO_write_struct_array_by_id(BlendWriter *writer,
int struct_id,
int array_size,
const void *data_ptr)
{
writestruct_nr(writer->wd, DATA, struct_id, array_size, data_ptr);
}
void BLO_write_struct_list_by_id(BlendWriter *writer, int struct_id, ListBase *list)
{
writelist_nr(writer->wd, DATA, struct_id, list);
}
void BLO_write_struct_list_by_name(BlendWriter *writer, const char *struct_name, ListBase *list)
{
BLO_write_struct_list_by_id(writer, BLO_get_struct_id_by_name(writer, struct_name), list);
}
void blo_write_id_struct(BlendWriter *writer, int struct_id, const void *id_address, const ID *id)
{
writestruct_at_address_nr(writer->wd, GS(id->name), struct_id, 1, id_address, id);
}
int BLO_get_struct_id_by_name(BlendWriter *writer, const char *struct_name)
{
int struct_id = DNA_struct_find_nr(writer->wd->sdna, struct_name);
BLI_assert(struct_id >= 0);
return struct_id;
}
void BLO_write_int32_array(BlendWriter *writer, int size, const int32_t *data_ptr)
{
BLO_write_raw(writer, sizeof(int32_t) * size, data_ptr);
}
void BLO_write_uint32_array(BlendWriter *writer, int size, const uint32_t *data_ptr)
{
BLO_write_raw(writer, sizeof(uint32_t) * size, data_ptr);
}
void BLO_write_float_array(BlendWriter *writer, int size, const float *data_ptr)
{
BLO_write_raw(writer, sizeof(float) * size, data_ptr);
}
void BLO_write_float3_array(BlendWriter *writer, int size, const float *data_ptr)
{
BLO_write_raw(writer, sizeof(float) * 3 * size, data_ptr);
}
/**
* Write a null terminated string.
*/
void BLO_write_string(BlendWriter *writer, const char *str)
{
if (str != NULL) {
BLO_write_raw(writer, strlen(str) + 1, str);
}
}
/**
* Sometimes different data is written depending on whether the file is saved to disk or used for
* undo. This function returns true when the current file-writing is done for undo.
*/
bool BLO_write_is_undo(BlendWriter *writer)
{
return writer->wd->use_memfile;
}
/** \} */