blender
/
blender
129
397
Fork 572
Code Issues 6.2k Pull Requests 750 Projects 19 Wiki Activity

Add experimental global undo speedup. 

The feature is hidden behind an experimental option, you'll have to
enable it in the preferences to try it.

This feature is not yet considered fully stable, crashes may happen, as
well as .blend file corruptions (very unlikely, but still possible).

In a nutshell, the ideas behind this code are to:
* Detect unchanged IDs across an undo step.
* Reuse as much as possible existing IDs memory, even when its content
  did change.
* Re-use existing depsgraphs instead of building new ones from scratch.
* Store accumulated recalc flags, to avoid needless re-compute of things
  that did not change, when the ID itself is detected as modified.

See T60695 and D6580 for more technical details.
This commit is contained in:
Bastien Montagne 2020-03-17 12:29:36 +01:00
parent 9ce3890950
commit b852db57ba
Notes: blender-bot 2023-02-14 07:45:38 +01:00 
Referenced by issue #80203, Crash when changing torus properties
Referenced by issue #80203, Crash when changing torus properties
17 changed files with 534 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
release/scripts/startup/bl_ui/space_userpref.py
View File

@ -2133,6 +2133,25 @@ class USERPREF_PT_experimental_virtual_reality(ExperimentalPanel, Panel):
"""
class USERPREF_PT_experimental_system(ExperimentalPanel, Panel):
bl_label = "System"
def draw(self, context):
prefs = context.preferences
experimental = prefs.experimental
layout = self.layout
layout.use_property_split = True
layout.use_property_decorate = False
task = "T60695"
split = layout.split(factor=0.66)
col = split.split()
col.prop(experimental, "use_undo_speedup")
col = split.split()
col.operator("wm.url_open", text=task, icon='URL').url = self.url_prefix + task
# -----------------------------------------------------------------------------
# Class Registration
@ -2222,6 +2241,8 @@ classes = (
# Popovers.
USERPREF_PT_ndof_settings,
USERPREF_PT_experimental_system,
# Add dynamically generated editor theme panels last,
# so they show up last in the theme section.
*ThemeGenericClassGenerator.generate_panel_classes_from_theme_areas(),

5
source/blender/blenkernel/BKE_blender_undo.h
View File

@ -32,7 +32,10 @@ struct bContext;
struct MemFileUndoData *BKE_memfile_undo_encode(struct Main *bmain,
struct MemFileUndoData *mfu_prev);
bool BKE_memfile_undo_decode(struct MemFileUndoData *mfu, struct bContext *C);
bool BKE_memfile_undo_decode(struct MemFileUndoData *mfu,
const int undo_direction,
const bool use_old_bmain_data,
struct bContext *C);
void BKE_memfile_undo_free(struct MemFileUndoData *mfu);
#ifdef __cplusplus

5
source/blender/blenkernel/BKE_main.h
View File

@ -99,6 +99,11 @@ typedef struct Main {
* use "needs_flush_to_id" in edit data to flag data which needs updating.
*/
char is_memfile_undo_flush_needed;
/**
* Indicates that next memfile undo step should not allow to re-use old bmain when re-read, but
* instead do a complete full re-read/update from stored memfile.
*/
char use_memfile_full_barrier;
BlendThumbnail *blen_thumb;

3
source/blender/blenkernel/BKE_undo_system.h
View File

@ -83,6 +83,9 @@ typedef struct UndoStep {
bool skip;
/** Some situations require the global state to be stored, edge cases when exiting modes. */
bool use_memfile_step;
/** When this is true, undo/memfile read code is allowed to re-use old data-blocks for unchanged
* IDs, and existing depsgraphes. This has to be forbidden in some cases (like renamed IDs). */
bool use_old_bmain_data;
/** For use by undo systems that accumulate changes (text editor, painting). */
bool is_applied;
/* Over alloc 'type->struct_size'. */

13
source/blender/blenkernel/intern/blender_undo.c
View File

@ -61,7 +61,10 @@
#define UNDO_DISK 0
bool BKE_memfile_undo_decode(MemFileUndoData *mfu, bContext *C)
bool BKE_memfile_undo_decode(MemFileUndoData *mfu,
const int undo_direction,
const bool use_old_bmain_data,
bContext *C)
{
Main *bmain = CTX_data_main(C);
char mainstr[sizeof(bmain->name)];
@ -76,8 +79,12 @@ bool BKE_memfile_undo_decode(MemFileUndoData *mfu, bContext *C)
success = BKE_blendfile_read(C, mfu->filename, &(const struct BlendFileReadParams){0}, NULL);
}
else {
success = BKE_blendfile_read_from_memfile(
C, &mfu->memfile, &(const struct BlendFileReadParams){0}, NULL);
struct BlendFileReadParams params = {0};
params.undo_direction = undo_direction > 0 ? 1 : -1;
if (!use_old_bmain_data) {
params.skip_flags |= BLO_READ_SKIP_UNDO_OLD_MAIN;
}
success = BKE_blendfile_read_from_memfile(C, &mfu->memfile, &params, NULL);
}
/* Restore, bmain has been re-allocated. */

38
source/blender/blenkernel/intern/blendfile.c
View File

@ -134,26 +134,29 @@ static void setup_app_userdef(BlendFileData *bfd)
static void setup_app_data(bContext *C,
BlendFileData *bfd,
const char *filepath,
const bool is_startup,
const struct BlendFileReadParams *params,
ReportList *reports)
{
Main *bmain = G_MAIN;
Scene *curscene = NULL;
const bool recover = (G.fileflags & G_FILE_RECOVER) != 0;
const bool is_startup = params->is_startup;
enum {
LOAD_UI = 1,
LOAD_UI_OFF,
LOAD_UNDO,
} mode;
/* may happen with library files - UNDO file should never have NULL cursccene... */
if (ELEM(NULL, bfd->curscreen, bfd->curscene)) {
if (params->undo_direction != 0) {
BLI_assert(bfd->curscene != NULL);
mode = LOAD_UNDO;
}
/* may happen with library files - UNDO file should never have NULL curscene (but may have a
* NULL curscreen)... */
else if (ELEM(NULL, bfd->curscreen, bfd->curscene)) {
BKE_report(reports, RPT_WARNING, "Library file, loading empty scene");
mode = LOAD_UI_OFF;
}
else if (BLI_listbase_is_empty(&bfd->main->screens)) {
mode = LOAD_UNDO;
}
else if (G.fileflags & G_FILE_NO_UI) {
mode = LOAD_UI_OFF;
}
@ -371,7 +374,9 @@ static void setup_app_data(bContext *C,
* means that we do not reset their user count, however we do increase that one when doing
* lib_link on local IDs using linked ones.
* There is no real way to predict amount of changes here, so we have to fully redo
* refcounting . */
* refcounting.
* Now that we re-use (and do not liblink in readfile.c) most local datablocks as well, we have
* to recompute refcount for all local IDs too. */
BKE_main_id_refcount_recompute(bmain, false);
}
}
@ -386,7 +391,7 @@ static void setup_app_blend_file_data(bContext *C,
setup_app_userdef(bfd);
}
if ((params->skip_flags & BLO_READ_SKIP_DATA) == 0) {
setup_app_data(C, bfd, filepath, params->is_startup, reports);
setup_app_data(C, bfd, filepath, params, reports);
}
}
@ -473,16 +478,15 @@ bool BKE_blendfile_read_from_memfile(bContext *C,
Main *bmain = CTX_data_main(C);
BlendFileData *bfd;
bfd = BLO_read_from_memfile(
bmain, BKE_main_blendfile_path(bmain), memfile, params->skip_flags, reports);
bfd = BLO_read_from_memfile(bmain, BKE_main_blendfile_path(bmain), memfile, params, reports);
if (bfd) {
/* remove the unused screens and wm */
while (bfd->main->wm.first) {
BKE_id_free(bfd->main, bfd->main->wm.first);
}
while (bfd->main->screens.first) {
BKE_id_free(bfd->main, bfd->main->screens.first);
}
/* Removing the unused workspaces, screens and wm is useless here, setup_app_data will switch
* those lists with the ones from old bmain, which freeing is much more efficient than
* individual calls to `BKE_id_free()`.
* Further more, those are expected to be empty anyway with new memfile reading code. */
BLI_assert(BLI_listbase_is_empty(&bfd->main->wm));
BLI_assert(BLI_listbase_is_empty(&bfd->main->workspaces));
BLI_assert(BLI_listbase_is_empty(&bfd->main->screens));
setup_app_blend_file_data(C, bfd, "<memory1>", params, reports);
BLO_blendfiledata_free(bfd);

9
source/blender/blenloader/BLO_readfile.h
View File

@ -76,8 +76,11 @@ typedef struct WorkspaceConfigFileData {
} WorkspaceConfigFileData;
struct BlendFileReadParams {
uint skip_flags : 2; /* eBLOReadSkip */
uint skip_flags : 3; /* eBLOReadSkip */
uint is_startup : 1;
/** Whether we are reading the memfile for an undo (< 0) or a redo (> 0). */
int undo_direction : 2;
};
/* skip reading some data-block types (may want to skip screen data too). */
@ -85,6 +88,8 @@ typedef enum eBLOReadSkip {
BLO_READ_SKIP_NONE = 0,
BLO_READ_SKIP_USERDEF = (1 << 0),
BLO_READ_SKIP_DATA = (1 << 1),
/** Do not attempt to re-use IDs from old bmain for unchanged ones in case of undo. */
BLO_READ_SKIP_UNDO_OLD_MAIN = (1 << 2),
} eBLOReadSkip;
#define BLO_READ_SKIP_ALL (BLO_READ_SKIP_USERDEF | BLO_READ_SKIP_DATA)
@ -98,7 +103,7 @@ BlendFileData *BLO_read_from_memory(const void *mem,
BlendFileData *BLO_read_from_memfile(struct Main *oldmain,
const char *filename,
struct MemFile *memfile,
eBLOReadSkip skip_flags,
const struct BlendFileReadParams *params,
struct ReportList *reports);
void BLO_blendfiledata_free(BlendFileData *bfd);

4
source/blender/blenloader/BLO_undofile.h
View File

@ -34,6 +34,10 @@ typedef struct {
unsigned int size;
/** When true, this chunk doesn't own the memory, it's shared with a previous #MemFileChunk */
bool is_identical;
/** When true, this chunk is also identical to the one in the next step (used by undo code to
* detect unchanged IDs).
* Defined when writing the next step (i.e. last undo step has those always false). */
bool is_identical_future;
} MemFileChunk;
typedef struct MemFile {

12
source/blender/blenloader/intern/readblenentry.c
View File

@ -363,17 +363,17 @@ BlendFileData *BLO_read_from_memory(const void *mem,
BlendFileData *BLO_read_from_memfile(Main *oldmain,
const char *filename,
MemFile *memfile,
eBLOReadSkip skip_flags,
const struct BlendFileReadParams *params,
ReportList *reports)
{
BlendFileData *bfd = NULL;
FileData *fd;
ListBase old_mainlist;
fd = blo_filedata_from_memfile(memfile, reports);
fd = blo_filedata_from_memfile(memfile, params, reports);
if (fd) {
fd->reports = reports;
fd->skip_flags = skip_flags;
fd->skip_flags = params->skip_flags;
BLI_strncpy(fd->relabase, filename, sizeof(fd->relabase));
/* clear ob->proxy_from pointers in old main */
@ -384,6 +384,12 @@ BlendFileData *BLO_read_from_memfile(Main *oldmain,
/* add the library pointers in oldmap lookup */
blo_add_library_pointer_map(&old_mainlist, fd);
if ((params->skip_flags & BLO_READ_SKIP_UNDO_OLD_MAIN) == 0) {
/* Build idmap of old main (we only care about local data here, so we can do that after
* split_main() call. */
blo_make_old_idmap_from_main(fd, old_mainlist.first);
}
/* makes lookup of existing images in old main */
blo_make_image_pointer_map(fd, oldmain);

320
source/blender/blenloader/intern/readfile.c
View File

@ -97,6 +97,7 @@
#include "BLI_endian_switch.h"
#include "BLI_blenlib.h"
#include "BLI_linklist.h"
#include "BLI_math.h"
#include "BLI_threads.h"
#include "BLI_mempool.h"
@ -261,6 +262,7 @@ typedef struct BHeadN {
/** When set, the remainder of this allocation is the data, otherwise it needs to be read. */
bool has_data;
#endif
bool is_memchunk_identical;
struct BHead bhead;
} BHeadN;
@ -794,7 +796,7 @@ static BHeadN *get_bhead(FileData *fd)
*/
if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) {
bhead4.code = DATA;
readsize = fd->read(fd, &bhead4, sizeof(bhead4));
readsize = fd->read(fd, &bhead4, sizeof(bhead4), NULL);
if (readsize == sizeof(bhead4) || bhead4.code == ENDB) {
if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
@ -817,7 +819,7 @@ static BHeadN *get_bhead(FileData *fd)
}
else {
bhead8.code = DATA;
readsize = fd->read(fd, &bhead8, sizeof(bhead8));
readsize = fd->read(fd, &bhead8, sizeof(bhead8), NULL);
if (readsize == sizeof(bhead8) || bhead8.code == ENDB) {
if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) {
@ -858,6 +860,7 @@ static BHeadN *get_bhead(FileData *fd)
new_bhead->next = new_bhead->prev = NULL;
new_bhead->file_offset = fd->file_offset;
new_bhead->has_data = false;
new_bhead->is_memchunk_identical = false;
new_bhead->bhead = bhead;
off64_t seek_new = fd->seek(fd, bhead.len, SEEK_CUR);
if (seek_new == -1) {
@ -880,9 +883,10 @@ static BHeadN *get_bhead(FileData *fd)
new_bhead->file_offset = 0; /* don't seek. */
new_bhead->has_data = true;
#endif
new_bhead->is_memchunk_identical = false;
new_bhead->bhead = bhead;
readsize = fd->read(fd, new_bhead + 1, bhead.len);
readsize = fd->read(fd, new_bhead + 1, bhead.len, &new_bhead->is_memchunk_identical);
if (readsize != bhead.len) {
fd->is_eof = true;
@ -972,7 +976,8 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf)
success = false;
}
else {
if (fd->read(fd, buf, new_bhead->bhead.len) != new_bhead->bhead.len) {
if (fd->read(fd, buf, new_bhead->bhead.len, &new_bhead->is_memchunk_identical) !=
new_bhead->bhead.len) {
success = false;
}
}
@ -989,6 +994,7 @@ static BHead *blo_bhead_read_full(FileData *fd, BHead *thisblock)
new_bhead_data->bhead = new_bhead->bhead;
new_bhead_data->file_offset = new_bhead->file_offset;
new_bhead_data->has_data = true;
new_bhead_data->is_memchunk_identical = false;
if (!blo_bhead_read_data(fd, thisblock, new_bhead_data + 1)) {
MEM_freeN(new_bhead_data);
return NULL;
@ -1009,7 +1015,7 @@ static void decode_blender_header(FileData *fd)
int readsize;
/* read in the header data */
readsize = fd->read(fd, header, sizeof(header));
readsize = fd->read(fd, header, sizeof(header), NULL);
if (readsize == sizeof(header) && STREQLEN(header, "BLENDER", 7) && ELEM(header[7], '_', '-') &&
ELEM(header[8], 'v', 'V') &&
@ -1141,7 +1147,10 @@ static int *read_file_thumbnail(FileData *fd)
/* Regular file reading. */
static int fd_read_data_from_file(FileData *filedata, void *buffer, uint size)
static int fd_read_data_from_file(FileData *filedata,
void *buffer,
uint size,
bool *UNUSED(r_is_memchunck_identical))
{
int readsize = read(filedata->filedes, buffer, size);
@ -1163,7 +1172,10 @@ static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int wh
/* GZip file reading. */
static int fd_read_gzip_from_file(FileData *filedata, void *buffer, uint size)
static int fd_read_gzip_from_file(FileData *filedata,
void *buffer,
uint size,
bool *UNUSED(r_is_memchunck_identical))
{
int readsize = gzread(filedata->gzfiledes, buffer, size);
@ -1179,7 +1191,10 @@ static int fd_read_gzip_from_file(FileData *filedata, void *buffer, uint size)
/* Memory reading. */
static int fd_read_from_memory(FileData *filedata, void *buffer, uint size)
static int fd_read_from_memory(FileData *filedata,
void *buffer,
uint size,
bool *UNUSED(r_is_memchunck_identical))
{
/* don't read more bytes then there are available in the buffer */
int readsize = (int)MIN2(size, (uint)(filedata->buffersize - filedata->file_offset));
@ -1192,7 +1207,10 @@ static int fd_read_from_memory(FileData *filedata, void *buffer, uint size)
/* MemFile reading. */
static int fd_read_from_memfile(FileData *filedata, void *buffer, uint size)
static int fd_read_from_memfile(FileData *filedata,
void *buffer,
uint size,
bool *r_is_memchunck_identical)
{
static size_t seek = SIZE_MAX; /* the current position */
static size_t offset = 0; /* size of previous chunks */
@ -1248,6 +1266,15 @@ static int fd_read_from_memfile(FileData *filedata, void *buffer, uint size)
totread += readsize;
filedata->file_offset += readsize;
seek += readsize;
if (r_is_memchunck_identical != NULL) {
/* `is_identical` of current chunk represent whether it changed compared to previous undo
* step. this is fine in redo case (filedata->undo_direction > 0), but not in undo case,
* where we need an extra flag defined when saving the next (future) step after the one we
* want to restore, as we are supposed to 'come from' that future undo step, and not the
* one before current one. */
*r_is_memchunck_identical = filedata->undo_direction > 0 ? chunk->is_identical :
chunk->is_identical_future;
}
} while (totread < size);
return totread;
@ -1414,7 +1441,10 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath)
return NULL;
}
static int fd_read_gzip_from_memory(FileData *filedata, void *buffer, uint size)
static int fd_read_gzip_from_memory(FileData *filedata,
void *buffer,
uint size,
bool *UNUSED(r_is_memchunck_identical))
{
int err;
@ -1485,7 +1515,9 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, ReportList *rep
}
}
FileData *blo_filedata_from_memfile(MemFile *memfile, ReportList *reports)
FileData *blo_filedata_from_memfile(MemFile *memfile,
const struct BlendFileReadParams *params,
ReportList *reports)
{
if (!memfile) {
BKE_report(reports, RPT_WARNING, "Unable to open blend <memory>");
@ -1494,6 +1526,7 @@ FileData *blo_filedata_from_memfile(MemFile *memfile, ReportList *reports)
else {
FileData *fd = filedata_new();
fd->memfile = memfile;
fd->undo_direction = params->undo_direction;
fd->read = fd_read_from_memfile;
fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
@ -1568,6 +1601,9 @@ void blo_filedata_free(FileData *fd)
if (fd->libmap && !(fd->flags & FD_FLAGS_NOT_MY_LIBMAP)) {
oldnewmap_free(fd->libmap);
}
if (fd->old_idmap != NULL) {
BKE_main_idmap_destroy(fd->old_idmap);
}
if (fd->bheadmap) {
MEM_freeN(fd->bheadmap);
}
@ -2187,6 +2223,16 @@ void blo_add_library_pointer_map(ListBase *old_mainlist, FileData *fd)
fd->old_mainlist = old_mainlist;
}
/* Build a GSet of old main (we only care about local data here, so we can do that after
* split_main() call. */
void blo_make_old_idmap_from_main(FileData *fd, Main *bmain)
{
if (fd->old_idmap != NULL) {
BKE_main_idmap_destroy(fd->old_idmap);
}
fd->old_idmap = BKE_main_idmap_create(bmain, false, NULL, MAIN_IDMAP_TYPE_UUID);
}
/** \} */
/* -------------------------------------------------------------------- */
@ -2267,6 +2313,10 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname)
#endif
}
}
if (!BHEADN_FROM_BHEAD(bh)->is_memchunk_identical) {
fd->are_memchunks_identical = false;
}
#ifdef USE_BHEAD_READ_ON_DEMAND
if (bh_orig != bh) {
MEM_freeN(BHEADN_FROM_BHEAD(bh));
@ -2656,17 +2706,17 @@ static void direct_link_id_override_property_cb(FileData *fd, void *data)
link_list_ex(fd, &op->operations, direct_link_id_override_property_operation_cb);
}
static void direct_link_id(FileData *fd, ID *id);
static void direct_link_id(FileData *fd, ID *id, ID *id_old);
static void direct_link_nodetree(FileData *fd, bNodeTree *ntree);
static void direct_link_collection(FileData *fd, Collection *collection);
static void direct_link_id_private_id(FileData *fd, ID *id)
static void direct_link_id_private_id(FileData *fd, ID *id, ID *id_old)
{
/* Handle 'private IDs'. */
bNodeTree **nodetree = BKE_ntree_ptr_from_id(id);
if (nodetree != NULL && *nodetree != NULL) {
*nodetree = newdataadr(fd, *nodetree);
direct_link_id(fd, (ID *)*nodetree);
direct_link_id(fd, (ID *)*nodetree, id_old != NULL ? (ID *)ntreeFromID(id_old) : NULL);
direct_link_nodetree(fd, *nodetree);
}
@ -2674,13 +2724,15 @@ static void direct_link_id_private_id(FileData *fd, ID *id)
Scene *scene = (Scene *)id;
if (scene->master_collection != NULL) {
scene->master_collection = newdataadr(fd, scene->master_collection);
direct_link_id(fd, &scene->master_collection->id);
direct_link_id(fd,
&scene->master_collection->id,
id_old != NULL ? &((Scene *)id_old)->master_collection->id : NULL);
direct_link_collection(fd, scene->master_collection);
}
}
}
static void direct_link_id(FileData *fd, ID *id)
static void direct_link_id(FileData *fd, ID *id, ID *id_old)
{
/*link direct data of ID properties*/
if (id->properties) {
@ -2704,8 +2756,34 @@ static void direct_link_id(FileData *fd, ID *id)
*
* But for regular file load we clear the flag, since the flags might have been changed since
* the version the file has been saved with. */
if (!fd->memfile) {
if (fd->memfile == NULL) {
id->recalc = 0;
id->recalc_undo_accumulated = 0;
}
else if ((fd->skip_flags & BLO_READ_SKIP_UNDO_OLD_MAIN) == 0) {
if (fd->undo_direction < 0) {
/* We are coming from the future (i.e. do an actual undo, and not a redo), and we found an
* old (aka existing) ID: we use its 'accumulated recalc flags since last memfile undo step
* saving' as recalc flags of our newly read ID. */
if (id_old != NULL) {
id->recalc = id_old->recalc_undo_accumulated;
}
}
else {
/* We are coming from the past (i.e. do a redo), we use saved 'accumulated
* recalc flags since last memfile undo step saving' as recalc flags of our newly read ID. */
id->recalc = id->recalc_undo_accumulated;
}
/* In any case, we need to flush the depsgraph's CoWs, as even if the ID address itself did not
* change, internal data most likely have. */
id->recalc |= ID_RECALC_COPY_ON_WRITE;
/* We need to 'accumulate' the accumulated recalc flags of all undo steps until we actually
* perform a depsgraph update, otherwise we'd only ever use the flags from one of the steps,
* and never get proper flags matching all others. */
if (id_old != NULL) {
id->recalc_undo_accumulated |= id_old->recalc_undo_accumulated;
}
}
/* Link direct data of overrides. */
@ -2721,7 +2799,7 @@ static void direct_link_id(FileData *fd, ID *id)
}
/* Handle 'private IDs'. */
direct_link_id_private_id(fd, id);
direct_link_id_private_id(fd, id, id_old);
}
/** \} */
@ -9071,15 +9149,155 @@ static BHead *read_libblock(FileData *fd,
}
/* read libblock */
fd->are_memchunks_identical = true;
id = read_struct(fd, bhead, "lib block");
const short idcode = id != NULL ? GS(id->name) : 0;
BHead *id_bhead = bhead;
/* Used when undoing from memfile, we swap changed IDs into their old addresses when found. */
ID *id_old = NULL;
bool do_id_swap = false;
if (id != NULL) {
const bool do_partial_undo = (fd->skip_flags & BLO_READ_SKIP_UNDO_OLD_MAIN) == 0;
if (id_bhead->code != ID_LINK_PLACEHOLDER) {
/* need a name for the mallocN, just for debugging and sane prints on leaks */
allocname = dataname(idcode);
/* read all data into fd->datamap */
/* TODO: instead of building oldnewmap here we could just quickly check the bheads... could
* save some more ticks. Probably not worth it though, bottleneck is full depsgraph rebuild
* and eval, not actual file reading. */
bhead = read_data_into_oldnewmap(fd, id_bhead, allocname);
DEBUG_PRINTF(
"%s: ID %s is unchanged: %d\n", __func__, id->name, fd->are_memchunks_identical);
if (fd->memfile != NULL) {
BLI_assert(fd->old_idmap != NULL || !do_partial_undo);
/* This code should only ever be reached for local data-blocks. */
BLI_assert(main->curlib == NULL);
/* Find the 'current' existing ID we want to reuse instead of the one we would read from
* the undo memfile. */
DEBUG_PRINTF("\t Looking for ID %s with uuid %u instead of newly read one\n",
id->name,
id->session_uuid);
id_old = do_partial_undo ? BKE_main_idmap_lookup_uuid(fd->old_idmap, id->session_uuid) :
NULL;
bool can_finalize_and_return = false;
if (ELEM(idcode, ID_WM, ID_SCR, ID_WS)) {
/* Read WindowManager, Screen and WorkSpace IDs are never actually used during undo (see
* `setup_app_data()` in `blendfile.c`).
* So we can just abort here, just ensuring libmapping is set accordingly. */
can_finalize_and_return = true;
}
else if (id_old != NULL && fd->are_memchunks_identical) {
/* Do not add LIB_TAG_NEW here, this should not be needed/used in undo case anyway (as
* this is only for do_version-like code), but for sake of consistency, and also because
* it will tell us which ID is re-used from old Main, and which one is actually new. */
id_old->tag = tag | LIB_TAG_NEED_LINK | LIB_TAG_UNDO_OLD_ID_REUSED;
id_old->lib = main->curlib;
id_old->us = ID_FAKE_USERS(id_old);
/* Do not reset id->icon_id here, memory allocated for it remains valid. */
/* Needed because .blend may have been saved with crap value here... */
id_old->newid = NULL;
id_old->orig_id = NULL;
/* About recalc: since that ID did not change at all, we know that its recalc fields also
* remained unchanged, so no need to handle neither recalc nor recalc_undo_future here.
*/
Main *old_bmain = fd->old_mainlist->first;
ListBase *old_lb = which_libbase(old_bmain, idcode);
ListBase *new_lb = which_libbase(main, idcode);
BLI_remlink(old_lb, id_old);
BLI_addtail(new_lb, id_old);
can_finalize_and_return = true;
}
if (can_finalize_and_return) {
DEBUG_PRINTF("Re-using existing ID %s instead of newly read one\n", id_old->name);
oldnewmap_insert(fd->libmap, id_bhead->old, id_old, id_bhead->code);
oldnewmap_insert(fd->libmap, id_old, id_old, id_bhead->code);
if (r_id) {
*r_id = id_old;
}
if (do_partial_undo) {
/* Even though we re-use the old ID as-is, it does not mean that we are 100% safe from
* needing some depsgraph updates for it (it could depend on another ID which address
* did
* not change, but which actual content might have been re-read from the memfile). */
if (fd->undo_direction < 0) {
/* We are coming from the future (i.e. do an actual undo, and not a redo), we use our
* old reused ID's 'accumulated recalc flags since last memfile undo step saving' as
* recalc flags. */
id_old->recalc = id_old->recalc_undo_accumulated;
}
else {
/* We are coming from the past (i.e. do a redo), we use the saved 'accumulated recalc
* flags since last memfile undo step saving' from the newly read ID as recalc flags.
*/
id_old->recalc = id->recalc_undo_accumulated;
}
/* There is no need to flush the depsgraph's CoWs here, since that ID's data itself did
* not change. */
/* We need to 'accumulate' the accumulated recalc flags of all undo steps until we
* actually perform a depsgraph update, otherwise we'd only ever use the flags from one
* of the steps, and never get proper flags matching all others. */
id_old->recalc_undo_accumulated |= id->recalc_undo_accumulated;
}
MEM_freeN(id);
oldnewmap_free_unused(fd->datamap);
oldnewmap_clear(fd->datamap);
return bhead;
}
}
}
if (id) {
const short idcode = GS(id->name);
/* do after read_struct, for dna reconstruct */
lb = which_libbase(main, idcode);
if (lb) {
/* Some re-used old IDs might also use newly read ones, so we have to check for old memory
* addresses for those as well. */
if (fd->memfile != NULL && do_partial_undo && id->lib == NULL) {
BLI_assert(fd->old_idmap != NULL);
DEBUG_PRINTF("\t Looking for ID %s with uuid %u instead of newly read one\n",
id->name,
id->session_uuid);
id_old = BKE_main_idmap_lookup_uuid(fd->old_idmap, id->session_uuid);
if (id_old != NULL) {
BLI_assert(MEM_allocN_len(id) == MEM_allocN_len(id_old));
/* UI IDs are always re-used from old bmain at higher-level calling code, so never swap
* those. Besides maybe custom properties, no other ID should have pointers to those
* anyway...
* And linked IDs are handled separately as well. */
do_id_swap = !ELEM(idcode, ID_WM, ID_SCR, ID_WS) &&
!(id_bhead->code == ID_LINK_PLACEHOLDER);
}
}
/* At this point, we know we are going to keep that newly read & allocated ID, so we need to
* reallocate it to ensure we actually get a unique memory address for it. */
if (!do_id_swap) {
DEBUG_PRINTF("using newly-read ID %s to a new mem address\n", id->name);
}
else {
DEBUG_PRINTF("using newly-read ID %s to its old, already existing address\n", id->name);
}
/* for ID_LINK_PLACEHOLDER check */
oldnewmap_insert(fd->libmap, bhead->old, id, bhead->code);
ID *id_target = do_id_swap ? id_old : id;
oldnewmap_insert(fd->libmap, id_bhead->old, id_target, id_bhead->code);
oldnewmap_insert(fd->libmap, id_old, id_target, id_bhead->code);
BLI_addtail(lb, id);
@ -9100,10 +9318,10 @@ static BHead *read_libblock(FileData *fd,
}
if (r_id) {
*r_id = id;
*r_id = do_id_swap ? id_old : id;
}
if (!id) {
return blo_bhead_next(fd, bhead);
return blo_bhead_next(fd, id_bhead);
}
id->lib = main->curlib;
@ -9113,7 +9331,7 @@ static BHead *read_libblock(FileData *fd,
id->orig_id = NULL;
/* this case cannot be direct_linked: it's just the ID part */
if (bhead->code == ID_LINK_PLACEHOLDER) {
if (id_bhead->code == ID_LINK_PLACEHOLDER) {
/* That way, we know which data-lock needs do_versions (required currently for linking). */
id->tag = tag | LIB_TAG_ID_LINK_PLACEHOLDER | LIB_TAG_NEED_LINK | LIB_TAG_NEW;
@ -9126,23 +9344,17 @@ static BHead *read_libblock(FileData *fd,
}
}
return blo_bhead_next(fd, bhead);
return blo_bhead_next(fd, id_bhead);
}
/* need a name for the mallocN, just for debugging and sane prints on leaks */
allocname = dataname(GS(id->name));
/* read all data into fd->datamap */
bhead = read_data_into_oldnewmap(fd, bhead, allocname);
/* init pointers direct data */
direct_link_id(fd, id);
direct_link_id(fd, id, id_old);
/* That way, we know which data-lock needs do_versions (required currently for linking). */
/* Note: doing this after driect_link_id(), which resets that field. */
/* Note: doing this after direct_link_id(), which resets that field. */
id->tag = tag | LIB_TAG_NEED_LINK | LIB_TAG_NEW;
switch (GS(id->name)) {
switch (idcode) {
case ID_WM:
direct_link_windowmanager(fd, (wmWindowManager *)id);
break;
@ -9266,6 +9478,37 @@ static BHead *read_libblock(FileData *fd,
*r_id = NULL;
}
}
else if (do_id_swap) {
/* During memfile undo, if an ID changed and we cannot directly re-use existing one from old
* bmain, we do a full read of the new id from the memfile, and then fully swap its content
* with the old id. This allows us to keep the same pointer even for modified data, which helps
* reducing further detected changes by the depsgraph (since unchanged IDs remain fully
* unchanged, even if they are using/pointing to a changed one). */
BLI_assert((fd->skip_flags & BLO_READ_SKIP_UNDO_OLD_MAIN) == 0);
Main *old_bmain = fd->old_mainlist->first;
BLI_assert(id_old != NULL);
ListBase *old_lb = which_libbase(old_bmain, idcode);
ListBase *new_lb = which_libbase(main, idcode);
BLI_remlink(old_lb, id_old);
BLI_remlink(new_lb, id);
/* We do not need any remapping from this call here, since no ID pointer is valid in the data
* currently (they are all pointing to old addresses, and need to go through `lib_link`
* process). So we can pass NULL for the Main pointer parameter. */
BKE_lib_id_swap_full(NULL, id, id_old);
BLI_addtail(new_lb, id_old);
BLI_addtail(old_lb, id);
}
else if (fd->memfile != NULL) {
DEBUG_PRINTF("We had to fully re-recreate ID %s (old addr: %p, new addr: %p)...\n",
id->name,
id_old,
id);
}
return (bhead);
}
@ -9437,6 +9680,8 @@ static void do_versions_after_linking(Main *main, ReportList *reports)
static void lib_link_all(FileData *fd, Main *bmain)
{
const bool do_partial_undo = (fd->skip_flags & BLO_READ_SKIP_UNDO_OLD_MAIN) == 0;
ID *id;
FOREACH_MAIN_ID_BEGIN (bmain, id) {
if ((id->tag & LIB_TAG_NEED_LINK) == 0) {
@ -9450,6 +9695,13 @@ static void lib_link_all(FileData *fd, Main *bmain)
continue;
}
if (fd->memfile != NULL && do_partial_undo && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) {
/* This ID has been re-used from 'old' bmain. Since it was therfore unchanged accross current
* undo step, and old IDs re-use their old memory address, we do not need to liblink it at
* all. */
continue;
}
lib_link_id(fd, bmain, id);
/* Note: ID types are processed in reverse order as defined by INDEX_ID_XXX enums in DNA_ID.h.

23
source/blender/blenloader/intern/readfile.h
View File

@ -30,6 +30,8 @@
#include "DNA_space_types.h"
#include "DNA_windowmanager_types.h" /* for ReportType */
struct IDNameLib_Map;
struct GSet;
struct Key;
struct MemFile;
struct Object;
@ -38,6 +40,8 @@ struct PartEff;
struct ReportList;
struct View3D;
typedef struct IDNameLib_Map IDNameLib_Map;
enum eFileDataFlag {
FD_FLAGS_SWITCH_ENDIAN = 1 << 0,
FD_FLAGS_FILE_POINTSIZE_IS_4 = 1 << 1,
@ -57,7 +61,10 @@ enum eFileDataFlag {
typedef int64_t off64_t;
#endif
typedef int(FileDataReadFn)(struct FileData *filedata, void *buffer, unsigned int size);
typedef int(FileDataReadFn)(struct FileData *filedata,
void *buffer,
unsigned int size,
bool *r_is_memchunk_identical);
typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence);
typedef struct FileData {
@ -78,6 +85,14 @@ typedef struct FileData {
const char *buffer;
/** Variables needed for reading from memfile (undo). */
struct MemFile *memfile;
/** Whether all data read from memfile so far was identical
* (i.e. shared with some previous undo step).
* Updated by `fd_read_from_memfile()`, user is responsible to reset it to true when needed.
* Used to detect unchanged IDs. */
bool are_memchunks_identical;
/** Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use
* to detect unchanged data from memfile. */
short undo_direction;
/** Variables needed for reading from file. */
gzFile gzfiledes;
@ -120,6 +135,7 @@ typedef struct FileData {
ListBase *mainlist;
/** Used for undo. */
ListBase *old_mainlist;
struct IDNameLib_Map *old_idmap;
struct ReportList *reports;
} FileData;
@ -135,7 +151,9 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath);
FileData *blo_filedata_from_file(const char *filepath, struct ReportList *reports);
FileData *blo_filedata_from_memory(const void *buffer, int buffersize, struct ReportList *reports);
FileData *blo_filedata_from_memfile(struct MemFile *memfile, struct ReportList *reports);
FileData *blo_filedata_from_memfile(struct MemFile *memfile,
const struct BlendFileReadParams *params,
struct ReportList *reports);
void blo_clear_proxy_pointers_from_lib(struct Main *oldmain);
void blo_make_image_pointer_map(FileData *fd, struct Main *oldmain);
@ -149,6 +167,7 @@ void blo_end_sound_pointer_map(FileData *fd, struct Main *oldmain);
void blo_make_packed_pointer_map(FileData *fd, struct Main *oldmain);
void blo_end_packed_pointer_map(FileData *fd, struct Main *oldmain);
void blo_add_library_pointer_map(ListBase *old_mainlist, FileData *fd);
void blo_make_old_idmap_from_main(FileData *fd, struct Main *bmain);
void blo_filedata_free(FileData *fd);

9
source/blender/blenloader/intern/undofile.c
View File

@ -98,6 +98,7 @@ void memfile_chunk_add(MemFile *memfile, const char *buf, uint size, MemFileChun
curchunk->size = size;
curchunk->buf = NULL;
curchunk->is_identical = false;
curchunk->is_identical_future = false;
BLI_addtail(&memfile->chunks, curchunk);
/* we compare compchunk with buf */
@ -107,6 +108,7 @@ void memfile_chunk_add(MemFile *memfile, const char *buf, uint size, MemFileChun
if (memcmp(compchunk->buf, buf, size) == 0) {
curchunk->buf = compchunk->buf;
curchunk->is_identical = true;
compchunk->is_identical_future = true;
}
}
*compchunk_step = compchunk->next;
@ -126,8 +128,11 @@ struct Main *BLO_memfile_main_get(struct MemFile *memfile,
struct Scene **r_scene)
{
struct Main *bmain_undo = NULL;
BlendFileData *bfd = BLO_read_from_memfile(
oldmain, BKE_main_blendfile_path(oldmain), memfile, BLO_READ_SKIP_NONE, NULL);
BlendFileData *bfd = BLO_read_from_memfile(oldmain,
BKE_main_blendfile_path(oldmain),
memfile,
&(const struct BlendFileReadParams){0},
NULL);
if (bfd) {
bmain_undo = bfd->main;

6
source/blender/blenloader/intern/writefile.c
View File

@ -4027,6 +4027,12 @@ static bool write_file_handle(Main *mainvar,
if (do_override) {
BKE_lib_override_library_operations_store_end(override_storage, id);
}
if (wd->use_memfile) {
/* Very important to do it after every ID write now, otherwise we cannot know whether a
* specific ID changed or not. */
mywrite_flush(wd);
}
}
mywrite_flush(wd);

120
source/blender/editors/undo/memfile_undo.c
View File

@ -23,12 +23,20 @@
#include "BLI_utildefines.h"
#include "BLI_sys_types.h"
#include "BLI_ghash.h"
#include "DNA_object_enums.h"
#include "DNA_object_types.h"
#include "BKE_blender_undo.h"
#include "BKE_context.h"
#include "BKE_undo_system.h"
#include "BKE_lib_id.h"
#include "BKE_lib_query.h"
#include "BKE_main.h"
#include "BKE_scene.h"
#include "BKE_undo_system.h"
#include "../depsgraph/DEG_depsgraph.h"
#include "WM_api.h"
#include "WM_types.h"
@ -85,16 +93,102 @@ static bool memfile_undosys_step_encode(struct bContext *UNUSED(C),
us->data = BKE_memfile_undo_encode(bmain, us_prev ? us_prev->data : NULL);
us->step.data_size = us->data->undo_size;
/* Store the fact that we should not re-use old data with that undo step, and reset the Main
* flag. */
us->step.use_old_bmain_data = !bmain->use_memfile_full_barrier;
bmain->use_memfile_full_barrier = false;
return true;
}
static void memfile_undosys_step_decode(
struct bContext *C, struct Main *bmain, UndoStep *us_p, int UNUSED(dir), bool UNUSED(is_final))
static int memfile_undosys_step_id_reused_cb(LibraryIDLinkCallbackData *cb_data)
{
ID *id_self = cb_data->id_self;
ID **id_pointer = cb_data->id_pointer;
BLI_assert((id_self->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0);
Main *bmain = cb_data->user_data;
ID *id = *id_pointer;
if (id != NULL && id->lib == NULL && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) == 0) {
bool do_stop_iter = true;
if (GS(id_self->name) == ID_OB) {
Object *ob_self = (Object *)id_self;
if (ob_self->type == OB_ARMATURE) {
if (ob_self->data == id) {
BLI_assert(GS(id->name) == ID_AR);
if (ob_self->pose != NULL) {
/* We have a changed/re-read armature used by an unchanged armature object: our beloved
* Bone pointers from the object's pose need their usual special treatment. */
ob_self->pose->flag |= POSE_RECALC;
}
}
else {
/* Cannot stop iteration until we checked ob_self->data pointer... */
do_stop_iter = false;
}
}
}
/* In case an old, re-used ID is using a newly read data-block (i.e. one of its ID pointers got
* updated), we have to tell the depsgraph about it. */
DEG_id_tag_update_ex(bmain, id_self, ID_RECALC_COPY_ON_WRITE);
return do_stop_iter ? IDWALK_RET_STOP_ITER : IDWALK_RET_NOP;
}
return IDWALK_RET_NOP;
}
static void memfile_undosys_step_decode(struct bContext *C,
struct Main *bmain,
UndoStep *us_p,
int undo_direction,
bool UNUSED(is_final))
{
BLI_assert(undo_direction != 0);
bool use_old_bmain_data = true;
if (!U.experimental.use_undo_speedup) {
use_old_bmain_data = false;
}
else if (undo_direction > 0) {
/* Redo case.
* The only time we should have to force a complete redo is when current step is tagged as a
* redo barrier.
* If previous step was not a memfile one should not matter here, current data in old bmain
* should still always be valid for unchanged dtat-blocks. */
if (us_p->use_old_bmain_data == false) {
use_old_bmain_data = false;
}
}
else {
/* Undo case.
* Here we do not care whether current step is an undo barrier, since we are comming from 'the
* future' we can still re-use old data. However, if *next* undo step (i.e. the one immédiately
* in the future, the one we are comming from) is a barrier, then we have to force a complete
* undo.
* Note that non-memfile undo steps **should** not be an issue anymore, since we handle
* fine-grained update flags now.
*/
UndoStep *us_next = us_p->next;
if (us_next != NULL) {
if (us_next->use_old_bmain_data == false) {
use_old_bmain_data = false;
}
}
}
/* Extract depsgraphs from current bmain (which may be freed during undo step reading),
* and store them for re-use. */
GHash *depsgraphs = NULL;
if (use_old_bmain_data) {
depsgraphs = BKE_scene_undo_depsgraphs_extract(bmain);
}
ED_editors_exit(bmain, false);
MemFileUndoStep *us = (MemFileUndoStep *)us_p;
BKE_memfile_undo_decode(us->data, C);
BKE_memfile_undo_decode(us->data, undo_direction, use_old_bmain_data, C);
for (UndoStep *us_iter = us_p->next; us_iter; us_iter = us_iter->next) {
if (BKE_UNDOSYS_TYPE_IS_MEMFILE_SKIP(us_iter->type)) {
@ -113,6 +207,24 @@ static void memfile_undosys_step_decode(
bmain = CTX_data_main(C);
ED_editors_init_for_undo(bmain);
if (use_old_bmain_data) {
/* Restore previous depsgraphs into current bmain. */
BKE_scene_undo_depsgraphs_restore(bmain, depsgraphs);
/* We need to inform depsgraph about re-used old IDs that would be using newly read
* data-blocks, at least COW evaluated copies need to be updated... */
ID *id = NULL;
FOREACH_MAIN_ID_BEGIN (bmain, id) {
if (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) {
BKE_library_foreach_ID_link(
bmain, id, memfile_undosys_step_id_reused_cb, bmain, IDWALK_READONLY);
}
}
FOREACH_MAIN_ID_END;
BKE_main_id_tag_all(bmain, LIB_TAG_UNDO_OLD_ID_REUSED, false);
}
WM_event_add_notifier(C, NC_SCENE | ND_LAYER_CONTENT, CTX_data_scene(C));
}

4
source/blender/makesdna/DNA_ID.h
View File

@ -559,6 +559,10 @@ enum {
/* Datablock was not allocated by standard system (BKE_libblock_alloc), do not free its memory
* (usual type-specific freeing is called though). */
LIB_TAG_NOT_ALLOCATED = 1 << 18,
/* RESET_AFTER_USE Used by undo system to tag unchanged IDs re-used from old Main (instead of
* read from memfile). */
LIB_TAG_UNDO_OLD_ID_REUSED = 1 << 19,
};
/* Tag given ID for an update in all the dependency graphs. */

3
source/blender/makesdna/DNA_userdef_types.h
View File

@ -613,7 +613,8 @@ typedef struct UserDef_FileSpaceData {
} UserDef_FileSpaceData;
typedef struct UserDef_Experimental {
char _pad0[8]; /* makesdna does not allow empty structs. */
char use_undo_speedup;
char _pad0[7]; /* makesdna does not allow empty structs. */
} UserDef_Experimental;
#define USER_EXPERIMENTAL_TEST(userdef, member) \

8
source/blender/makesrna/intern/rna_userdef.c
View File

@ -5918,12 +5918,20 @@ static void rna_def_userdef_filepaths(BlenderRNA *brna)
static void rna_def_userdef_experimental(BlenderRNA *brna)
{
StructRNA *srna;
PropertyRNA *prop;
srna = RNA_def_struct(brna, "PreferencesExperimental", NULL);
RNA_def_struct_sdna(srna, "UserDef_Experimental");
RNA_def_struct_nested(brna, srna, "Preferences");
RNA_def_struct_clear_flag(srna, STRUCT_UNDO);
RNA_def_struct_ui_text(srna, "Experimental", "Experimental features");
prop = RNA_def_property(srna, "use_undo_speedup", PROP_BOOLEAN, PROP_NONE);
RNA_def_property_boolean_sdna(prop, NULL, "use_undo_speedup", 1);
RNA_def_property_ui_text(
prop,
"Undo Speedup",
"Use new undo speedup (WARNING: can lead to crashes and serious .blend file corruption)");
}
static void rna_def_userdef_addon_collection(BlenderRNA *brna, PropertyRNA *cprop)