VSE: Disk cache

This patch implements dumping images from cache to HDD.
The main goal of this system is to provide a means to achieve consistent playback speed mainly for strips that are not possible to preview in real time.

How to use:
Disk cache has own settings in user preferences for path to storage, size limit and compression level.
To use disk cache, you need to check `Use Disk Cache` box, set `Disk Cache Directory`, `Disk Cache Limit` and save or open existing .blend file.
By default sequencer output will be cached only. Manual setting is possible in cache panel.

Uses:
 - Replacement or alternative for proxies. Disk cache will work with any strip type, supports float images as well.
 - Storage for strip thumbnails.
 - Less RAM needs to be allocated for preview cache

How it works:
Disk cache is extension of RAM cache. Every image, that is stored or deleted in RAM will be stored or deleted on HDD as well. Images can be compressed to save space and for use on slower drives. Compressed images are slower to write and read though.
Images are stored in bulk of 100 rendered frames per one file. This is to overcome slow file access time for large amount of files. Drawback is, that if one frame needs to be redrawn, all 100 frames are deleted.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D5524

release/datafiles/userdef/userdef_default.c

@@ -221,6 +221,11 @@ const UserDef U_default = {
             .temp_win_sizey = 600,
         },
 
+    .sequencer_disk_cache_dir = "",
+    .sequencer_disk_cache_compression = 0,
+    .sequencer_disk_cache_size_limit = 100,
+    .sequencer_disk_cache_flag = 0,
+
     .runtime =
         {
             .is_dirty = 0,

release/scripts/startup/bl_ui/space_userpref.py

@@ -628,6 +628,15 @@ class USERPREF_PT_system_memory(SystemPanel, CenterAlignMixIn, Panel):
 
         flow = layout.grid_flow(row_major=False, columns=0, even_columns=True, even_rows=False, align=False)
 
+        flow.prop(system, "use_sequencer_disk_cache")
+        flow.prop(system, "sequencer_disk_cache_dir")
+        flow.prop(system, "sequencer_disk_cache_size_limit")
+        flow.prop(system, "sequencer_disk_cache_compression")
+
+        layout.separator()
+
+        flow = layout.grid_flow(row_major=False, columns=0, even_columns=True, even_rows=False, align=False)
+
         flow.prop(system, "texture_time_out", text="Texture Time Out")
         flow.prop(system, "texture_collection_rate", text="Garbage Collection Rate")
 

source/blender/blenkernel/BKE_sequencer.h

@@ -312,19 +312,22 @@ void BKE_sequencer_proxy_set(struct Sequence *seq, bool value);
 struct ImBuf *BKE_sequencer_cache_get(const SeqRenderData *context,
                                       struct Sequence *seq,
                                       float cfra,
-                                      int type);
+                                      int type,
+                                      bool skip_disk_cache);
 void BKE_sequencer_cache_put(const SeqRenderData *context,
                              struct Sequence *seq,
                              float cfra,
                              int type,
                              struct ImBuf *nval,
-                             float cost);
+                             float cost,
+                             bool skip_disk_cache);
 bool BKE_sequencer_cache_put_if_possible(const SeqRenderData *context,
                                          struct Sequence *seq,
                                          float cfra,
                                          int type,
                                          struct ImBuf *nval,
-                                         float cost);
+                                         float cost,
+                                         bool skip_disk_cache);
 bool BKE_sequencer_cache_recycle_item(struct Scene *scene);
 void BKE_sequencer_cache_free_temp_cache(struct Scene *scene, short id, int cfra);
 void BKE_sequencer_cache_destruct(struct Scene *scene);

source/blender/blenkernel/intern/sequencer.c

@@ -3081,7 +3081,7 @@ static ImBuf *seq_render_image_strip(const SeqRenderData *context,
 
         if (i != context->view_id) {
           BKE_sequencer_cache_put(
-              &localcontext, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibufs_arr[i], 0);
+              &localcontext, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibufs_arr[i], 0, false);
         }
       }
     }
@@ -3201,7 +3201,7 @@ static ImBuf *seq_render_movie_strip(const SeqRenderData *context,
       }
       if (i != context->view_id) {
         BKE_sequencer_cache_put(
-            &localcontext, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibuf_arr[i], 0);
+            &localcontext, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibuf_arr[i], 0, false);
       }
     }
 
@@ -3600,7 +3600,8 @@ static ImBuf *seq_render_scene_strip(const SeqRenderData *context,
       }
 
       if (i != context->view_id) {
-        BKE_sequencer_cache_put(&localcontext, seq, cfra, SEQ_CACHE_STORE_RAW, ibufs_arr[i], 0);
+        BKE_sequencer_cache_put(
+            &localcontext, seq, cfra, SEQ_CACHE_STORE_RAW, ibufs_arr[i], 0, false);
       }
 
       RE_ReleaseResultImage(re);
@@ -3813,10 +3814,10 @@ static ImBuf *seq_render_strip(const SeqRenderData *context,
 
   clock_t begin = seq_estimate_render_cost_begin();
 
-  ibuf = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED);
+  ibuf = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, false);
 
   if (ibuf == NULL) {
-    ibuf = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_RAW);
+    ibuf = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_RAW, false);
     if (ibuf == NULL) {
       /* MOVIECLIPs have their own proxy management */
       if (seq->type != SEQ_TYPE_MOVIECLIP) {
@@ -3852,7 +3853,7 @@ static ImBuf *seq_render_strip(const SeqRenderData *context,
 
     if (use_preprocess) {
       float cost = seq_estimate_render_cost_end(context->scene, begin);
-      BKE_sequencer_cache_put(context, seq, cfra, SEQ_CACHE_STORE_RAW, ibuf, cost);
+      BKE_sequencer_cache_put(context, seq, cfra, SEQ_CACHE_STORE_RAW, ibuf, cost, false);
 
       /* reset timer so we can get partial render time */
       begin = seq_estimate_render_cost_begin();
@@ -3860,7 +3861,7 @@ static ImBuf *seq_render_strip(const SeqRenderData *context,
     }
 
     float cost = seq_estimate_render_cost_end(context->scene, begin);
-    BKE_sequencer_cache_put(context, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibuf, cost);
+    BKE_sequencer_cache_put(context, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibuf, cost, false);
   }
   return ibuf;
 }
@@ -3954,7 +3955,7 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context,
     int early_out;
     Sequence *seq = seq_arr[i];
 
-    out = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_COMPOSITE);
+    out = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_COMPOSITE, false);
 
     if (out) {
       break;
@@ -3986,7 +3987,8 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context,
           out = seq_render_strip_stack_apply_effect(context, seq, cfra, ibuf1, ibuf2);
 
           float cost = seq_estimate_render_cost_end(context->scene, begin);
-          BKE_sequencer_cache_put(context, seq_arr[i], cfra, SEQ_CACHE_STORE_COMPOSITE, out, cost);
+          BKE_sequencer_cache_put(
+              context, seq_arr[i], cfra, SEQ_CACHE_STORE_COMPOSITE, out, cost, false);
 
           IMB_freeImBuf(ibuf1);
           IMB_freeImBuf(ibuf2);
@@ -4014,7 +4016,8 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context,
     }
 
     float cost = seq_estimate_render_cost_end(context->scene, begin);
-    BKE_sequencer_cache_put(context, seq_arr[i], cfra, SEQ_CACHE_STORE_COMPOSITE, out, cost);
+    BKE_sequencer_cache_put(
+        context, seq_arr[i], cfra, SEQ_CACHE_STORE_COMPOSITE, out, cost, false);
   }
 
   return out;
@@ -4053,7 +4056,8 @@ ImBuf *BKE_sequencer_give_ibuf(const SeqRenderData *context, float cfra, int cha
   count = get_shown_sequences(seqbasep, cfra, chanshown, seq_arr);
 
   if (count) {
-    out = BKE_sequencer_cache_get(context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT);
+    out = BKE_sequencer_cache_get(
+        context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, false);
   }
 
   BKE_sequencer_cache_free_temp_cache(context->scene, context->task_id, cfra);
@@ -4068,11 +4072,11 @@ ImBuf *BKE_sequencer_give_ibuf(const SeqRenderData *context, float cfra, int cha
 
     if (context->is_prefetch_render) {
       BKE_sequencer_cache_put(
-          context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, out, cost);
+          context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, out, cost, false);
     }
     else {
       BKE_sequencer_cache_put_if_possible(
-          context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, out, cost);
+          context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, out, cost, false);
     }
     BLI_mutex_unlock(&seq_render_mutex);
   }
@@ -5294,7 +5298,7 @@ Sequence *BKE_sequence_alloc(ListBase *lb, int cfra, int machine, int type)
 
   seq->strip = seq_strip_alloc(type);
   seq->stereo3d_format = MEM_callocN(sizeof(Stereo3dFormat), "Sequence Stereo Format");
-  seq->cache_flag = SEQ_CACHE_ALL_TYPES;
+  seq->cache_flag = SEQ_CACHE_STORE_RAW | SEQ_CACHE_STORE_PREPROCESSED | SEQ_CACHE_STORE_COMPOSITE;
 
   return seq;
 }

source/blender/blenlib/BLI_fileops.h

@@ -153,7 +153,13 @@ int BLI_file_gzip(const char *from, const char *to) ATTR_WARN_UNUSED_RESULT ATTR
 #endif
 char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) ATTR_WARN_UNUSED_RESULT
     ATTR_NONNULL();
-
+size_t BLI_gzip_mem_to_file_at_pos(void *buf,
+                                   size_t len,
+                                   FILE *file,
+                                   size_t gz_stream_offset,
+                                   int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
+    ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
 size_t BLI_file_size(const char *file) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 

source/blender/blenlib/intern/fileops.c

@@ -158,6 +158,101 @@ char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size)
   return mem;
 }
 
+#define CHUNK 256 * 1024
+
+/* gzip byte array from memory and write it to file at certain position.
+ * return size of gzip stream.
+ */
+size_t BLI_gzip_mem_to_file_at_pos(
+    void *buf, size_t len, FILE *file, size_t gz_stream_offset, int compression_level)
+{
+  int ret, flush;
+  unsigned have;
+  z_stream strm;
+  unsigned char out[CHUNK];
+
+  fseek(file, gz_stream_offset, 0);
+
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+  ret = deflateInit(&strm, compression_level);
+  if (ret != Z_OK)
+    return 0;
+
+  strm.avail_in = len;
+  strm.next_in = (Bytef *)buf;
+  flush = Z_FINISH;
+
+  do {
+    strm.avail_out = CHUNK;
+    strm.next_out = out;
+    ret = deflate(&strm, flush);
+    if (ret == Z_STREAM_ERROR) {
+      return 0;
+    }
+    have = CHUNK - strm.avail_out;
+    if (fwrite(out, 1, have, file) != have || ferror(file)) {
+      deflateEnd(&strm);
+      return 0;
+    }
+  } while (strm.avail_out == 0);
+
+  if (strm.avail_in != 0 || ret != Z_STREAM_END) {
+    return 0;
+  }
+
+  deflateEnd(&strm);
+  return (size_t)strm.total_out;
+}
+
+/* read and decompress gzip stream from file at certain position to buffer.
+ * return size of decompressed data.
+ */
+size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
+{
+  int ret;
+  z_stream strm;
+  size_t chunk = 256 * 1024;
+  unsigned char in[CHUNK];
+
+  fseek(file, gz_stream_offset, 0);
+
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+  strm.avail_in = 0;
+  strm.next_in = Z_NULL;
+  ret = inflateInit(&strm);
+  if (ret != Z_OK)
+    return 0;
+
+  do {
+    strm.avail_in = fread(in, 1, chunk, file);
+    strm.next_in = in;
+    if (ferror(file)) {
+      inflateEnd(&strm);
+      return 0;
+    }
+
+    do {
+      strm.avail_out = len;
+      strm.next_out = (Bytef *)buf + strm.total_out;
+
+      ret = inflate(&strm, Z_NO_FLUSH);
+      if (ret == Z_STREAM_ERROR) {
+        return 0;
+      }
+    } while (strm.avail_out == 0);
+
+  } while (ret != Z_STREAM_END);
+
+  inflateEnd(&strm);
+  return (size_t)strm.total_out;
+}
+
+#undef CHUNK
+
 /**
  * Returns true if the file with the specified name can be written.
  * This implementation uses access(2), which makes the check according

source/blender/makesdna/DNA_sequence_types.h

@@ -276,6 +276,9 @@ typedef struct Editing {
   int cache_flag;
 
   struct PrefetchJob *prefetch_job;
+
+  /* Must be initialized only by BKE_sequencer_cache_create() */
+  int64_t disk_cache_timestamp;
 } Editing;
 
 /* ************* Effect Variable Structs ********* */
@@ -682,6 +685,7 @@ enum {
   SEQ_CACHE_VIEW_FINAL_OUT = (1 << 9),
 
   SEQ_CACHE_PREFETCH_ENABLE = (1 << 10),
+  SEQ_CACHE_DISK_CACHE_ENABLE = (1 << 11),
 };
 
 #ifdef __cplusplus

source/blender/makesdna/DNA_userdef_types.h

@@ -862,7 +862,13 @@ typedef struct UserDef {
 
   char render_display_type;      /* eUserpref_RenderDisplayType */
   char filebrowser_display_type; /* eUserpref_TempSpaceDisplayType */
-  char _pad5[4];
+
+  char sequencer_disk_cache_dir[1024];
+  int sequencer_disk_cache_compression; /* eUserpref_DiskCacheCompression */
+  int sequencer_disk_cache_size_limit;
+  short sequencer_disk_cache_flag;
+
+  char _pad5[2];
 
   struct WalkNavigation walk_navigation;
 
@@ -1286,6 +1292,12 @@ typedef enum eUserpref_EmulateMMBMod {
   USER_EMU_MMB_MOD_OSKEY = 1,
 } eUserpref_EmulateMMBMod;
 
+typedef enum eUserpref_DiskCacheCompression {
+  USER_SEQ_DISK_CACHE_COMPRESSION_NONE = 0,
+  USER_SEQ_DISK_CACHE_COMPRESSION_LOW = 1,
+  USER_SEQ_DISK_CACHE_COMPRESSION_HIGH = 2,
+} eUserpref_DiskCacheCompression;
+
 #ifdef __cplusplus
 }
 #endif

source/blender/makesrna/intern/rna_sequencer.c

@@ -1951,7 +1951,7 @@ static void rna_def_editor(BlenderRNA *brna)
   RNA_def_property_ui_range(prop, 0.0f, SEQ_CACHE_COST_MAX, 0.1f, 1);
   RNA_def_property_float_sdna(prop, NULL, "recycle_max_cost");
   RNA_def_property_ui_text(
-      prop, "Recycle Up to Cost", "Only frames with cost lower than this value will be recycled");
+      prop, "Recycle Up To Cost", "Only frames with cost lower than this value will be recycled");
 }
 
 static void rna_def_filter_video(StructRNA *srna)

source/blender/makesrna/intern/rna_userdef.c

@@ -192,6 +192,8 @@ static const EnumPropertyItem rna_enum_userdef_viewport_aa_items[] = {
 
 #  include "BLF_api.h"
 
+#  include "BLI_path_util.h"
+
 #  include "MEM_guardedalloc.h"
 #  include "MEM_CacheLimiterC-Api.h"
 
@@ -536,6 +538,19 @@ static void rna_Userdef_memcache_update(Main *UNUSED(bmain),
   USERDEF_TAG_DIRTY;
 }
 
+static void rna_Userdef_disk_cache_dir_update(Main *UNUSED(bmain),
+                                              Scene *UNUSED(scene),
+                                              PointerRNA *UNUSED(ptr))
+{
+  if (U.sequencer_disk_cache_dir[0] != '\0') {
+    BLI_path_abs(U.sequencer_disk_cache_dir, BKE_main_blendfile_path_from_global());
+    BLI_add_slash(U.sequencer_disk_cache_dir);
+    BLI_path_make_safe(U.sequencer_disk_cache_dir);
+  }
+
+  USERDEF_TAG_DIRTY;
+}
+
 static void rna_UserDef_weight_color_update(Main *bmain, Scene *scene, PointerRNA *ptr)
 {
   Object *ob;
@@ -5126,6 +5141,25 @@ static void rna_def_userdef_system(BlenderRNA *brna)
       {0, NULL, 0, NULL, NULL},
   };
 
+  static const EnumPropertyItem seq_disk_cache_compression_levels[] = {
+      {USER_SEQ_DISK_CACHE_COMPRESSION_NONE,
+       "NONE",
+       0,
+       "None",
+       "Requires fast storage, but uses minimum CPU resources"},
+      {USER_SEQ_DISK_CACHE_COMPRESSION_LOW,
+       "LOW",
+       0,
+       "Low",
+       "Doesn't require fast storage and uses less CPU resources"},
+      {USER_SEQ_DISK_CACHE_COMPRESSION_HIGH,
+       "HIGH",
+       0,
+       "High",
+       "Works on slower storage devices and uses most CPU resources"},
+      {0, NULL, 0, NULL, NULL},
+  };
+
   srna = RNA_def_struct(brna, "PreferencesSystem", NULL);
   RNA_def_struct_sdna(srna, "UserDef");
   RNA_def_struct_nested(brna, srna, "Preferences");
@@ -5168,6 +5202,31 @@ static void rna_def_userdef_system(BlenderRNA *brna)
   RNA_def_property_ui_text(prop, "Memory Cache Limit", "Memory cache limit (in megabytes)");
   RNA_def_property_update(prop, 0, "rna_Userdef_memcache_update");
 
+  /* Sequencer disk cache */
+
+  prop = RNA_def_property(srna, "use_sequencer_disk_cache", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(
+      prop, NULL, "sequencer_disk_cache_flag", SEQ_CACHE_DISK_CACHE_ENABLE);
+  RNA_def_property_ui_text(prop, "Use Disk Cache", "Store cached images to disk");
+
+  prop = RNA_def_property(srna, "sequencer_disk_cache_dir", PROP_STRING, PROP_DIRPATH);
+  RNA_def_property_string_sdna(prop, NULL, "sequencer_disk_cache_dir");
+  RNA_def_property_update(prop, 0, "rna_Userdef_disk_cache_dir_update");
+  RNA_def_property_ui_text(prop, "Disk Cache Directory", "Override default directory");
+
+  prop = RNA_def_property(srna, "sequencer_disk_cache_size_limit", PROP_INT, PROP_NONE);
+  RNA_def_property_int_sdna(prop, NULL, "sequencer_disk_cache_size_limit");
+  RNA_def_property_range(prop, 0, INT_MAX);
+  RNA_def_property_ui_text(prop, "Disk Cache Limit", "Disk cache limit (in gigabytes)");
+
+  prop = RNA_def_property(srna, "sequencer_disk_cache_compression", PROP_ENUM, PROP_NONE);
+  RNA_def_property_enum_items(prop, seq_disk_cache_compression_levels);
+  RNA_def_property_enum_sdna(prop, NULL, "sequencer_disk_cache_compression");
+  RNA_def_property_ui_text(
+      prop,
+      "Disk Cache Compression Level",
+      "Smaller compression will result in larger files, but less decoding overhead");
+
   prop = RNA_def_property(srna, "scrollback", PROP_INT, PROP_UNSIGNED);
   RNA_def_property_int_sdna(prop, NULL, "scrollback");
   RNA_def_property_range(prop, 32, 32768);