DNA: optimize struct reconstruction by doing some preprocessing

When loading large files that are more than a couple weeks old
(such that DNA has changed in that time), a significant amount of
time is spent in `DNA_struct_reconstruct`. This function takes a struct
in the old layout and creates a struct in the new layout from it.

This was slow because it was computing the diff between the struct
layouts every time a struct is updated. Now the steps for the struct
reconstruction is computed only once per struct. This information is
then used to actually reconstruct all structs that changed.

I measured about 10-20% speedup when loading Spring files.
E.g. `10.6s -> 8.7s` for `06_055_A.anim.blend` in BKE_blendfile_read`.

This percentage varies a lot based on the number of blocks that have
to be reconstructed and how much DNA has changed since they have
been written. In none of my tests was the new code slower than the
old code.

Reviewers: campbellbarton

Differential Revision: https://developer.blender.org/D8959

source/blender/blenloader/intern/readfile.c

@@ -779,7 +779,7 @@ static void bh4_from_bh8(BHead *bhead, BHead8 *bhead8, int do_endian_swap)
      * 0x0000000000000000000012345678 would become 0x12345678000000000000000000000000
      */
     if (do_endian_swap) {
-      BLI_endian_switch_int64(&bhead8->old);
+      BLI_endian_switch_uint64(&bhead8->old);
     }
 
     /* this patch is to avoid a long long being read from not-eight aligned positions
@@ -1114,6 +1114,8 @@ static bool read_file_dna(FileData *fd, const char **r_error_message)
       if (fd->filesdna) {
         blo_do_versions_dna(fd->filesdna, fd->fileversion, subversion);
         fd->compflags = DNA_struct_get_compareflags(fd->filesdna, fd->memsdna);
+        fd->reconstruct_info = DNA_reconstruct_info_create(
+            fd->filesdna, fd->memsdna, fd->compflags);
         /* used to retrieve ID names from (bhead+1) */
         fd->id_name_offs = DNA_elem_offset(fd->filesdna, "ID", "char", "name[]");
 
@@ -1610,6 +1612,9 @@ void blo_filedata_free(FileData *fd)
     if (fd->compflags) {
       MEM_freeN((void *)fd->compflags);
     }
+    if (fd->reconstruct_info) {
+      DNA_reconstruct_info_free(fd->reconstruct_info);
+    }
 
     if (fd->datamap) {
       oldnewmap_free(fd->datamap);
@@ -2177,8 +2182,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname)
           }
         }
 #endif
-        temp = DNA_struct_reconstruct(
-            fd->memsdna, fd->filesdna, fd->compflags, bh->SDNAnr, bh->nr, (bh + 1));
+        temp = DNA_struct_reconstruct(fd->reconstruct_info, bh->SDNAnr, bh->nr, (bh + 1));
       }
       else {
         /* SDNA_CMP_EQUAL */
@@ -9135,7 +9139,7 @@ static void convert_pointer_array_32_to_64(BlendDataReader *UNUSED(reader),
                                            const uint32_t *src,
                                            uint64_t *dst)
 {
-  /* Match pointer conversion rules from bh8_from_bh4 and cast_pointer. */
+  /* Match pointer conversion rules from bh8_from_bh4 and cast_pointer_32_to_64. */
   for (int i = 0; i < array_size; i++) {
     dst[i] = src[i];
   }

source/blender/blenloader/intern/readfile.h

@@ -106,6 +106,7 @@ typedef struct FileData {
   const struct SDNA *memsdna;
   /** Array of #eSDNA_StructCompare. */
   const char *compflags;
+  struct DNA_ReconstructInfo *reconstruct_info;
 
   int fileversion;
   /** Used to retrieve ID names from (bhead+1). */

source/blender/makesdna/DNA_genfile.h

@@ -78,6 +78,8 @@ enum eSDNA_StructCompare {
   /* Struct is different in some way
    * (needs to be copied/converted field by field) */
   SDNA_CMP_NOT_EQUAL = 2,
+  /* This is only used temporarily by #DNA_struct_get_compareflags. */
+  SDNA_CMP_UNKNOWN = 3,
 };
 
 struct SDNA *DNA_sdna_from_data(const void *data,
@@ -93,13 +95,17 @@ void DNA_sdna_current_init(void);
 const struct SDNA *DNA_sdna_current_get(void);
 void DNA_sdna_current_free(void);
 
+struct DNA_ReconstructInfo;
+struct DNA_ReconstructInfo *DNA_reconstruct_info_create(const struct SDNA *oldsdna,
+                                                        const struct SDNA *newsdna,
+                                                        const char *compare_flags);
+void DNA_reconstruct_info_free(struct DNA_ReconstructInfo *reconstruct_info);
+
 int DNA_struct_find_nr_ex(const struct SDNA *sdna, const char *str, unsigned int *index_last);
 int DNA_struct_find_nr(const struct SDNA *sdna, const char *str);
 void DNA_struct_switch_endian(const struct SDNA *oldsdna, int oldSDNAnr, char *data);
 const char *DNA_struct_get_compareflags(const struct SDNA *sdna, const struct SDNA *newsdna);
-void *DNA_struct_reconstruct(const struct SDNA *newsdna,
-                             const struct SDNA *oldsdna,
-                             const char *compflags,
+void *DNA_struct_reconstruct(const struct DNA_ReconstructInfo *reconstruct_info,
                              int oldSDNAnr,
                              int blocks,
                              const void *data);

source/blender/makesdna/DNA_sdna_types.h

@@ -108,13 +108,13 @@ typedef struct BHead {
 #
 typedef struct BHead4 {
   int code, len;
-  int old;
+  uint old;
   int SDNAnr, nr;
 } BHead4;
 #
 #
 typedef struct BHead8 {
   int code, len;
-  int64_t old;
+  uint64_t old;
   int SDNAnr, nr;
 } BHead8;