Alembic Procedural: basic cache control settings

This adds a setting to enable data caching, and another one to set the
maximum cache size in megabytes.

When caching is enabled we load the data for the entire animation in
memory, as we already do, however, if the data exceeds the memory limit,
render is aborted.

When caching is disabled, we simply load the data for the current frame
in memory.

Ref D10197

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D11163

intern/cycles/blender/blender_object.cpp

@@ -523,6 +523,9 @@ void BlenderSync::sync_procedural(BL::Object &b_ob,
 
   procedural->set_scale(cache_file.scale());
 
+  procedural->set_use_prefetch(cache_file.use_prefetch());
+  procedural->set_prefetch_cache_size(cache_file.prefetch_cache_size());
+
   /* create or update existing AlembicObjects */
   ustring object_path = ustring(b_mesh_cache.object_path());
 

intern/cycles/render/alembic.cpp

@@ -25,6 +25,7 @@
 #include "render/shader.h"
 
 #include "util/util_foreach.h"
+#include "util/util_logging.h"
 #include "util/util_progress.h"
 #include "util/util_transform.h"
 #include "util/util_vector.h"
@@ -211,6 +212,35 @@ void CachedData::set_time_sampling(TimeSampling time_sampling)
   }
 }
 
+size_t CachedData::memory_used() const
+{
+  size_t mem_used = 0;
+
+  mem_used += curve_first_key.memory_used();
+  mem_used += curve_keys.memory_used();
+  mem_used += curve_radius.memory_used();
+  mem_used += curve_shader.memory_used();
+  mem_used += num_ngons.memory_used();
+  mem_used += shader.memory_used();
+  mem_used += subd_creases_edge.memory_used();
+  mem_used += subd_creases_weight.memory_used();
+  mem_used += subd_face_corners.memory_used();
+  mem_used += subd_num_corners.memory_used();
+  mem_used += subd_ptex_offset.memory_used();
+  mem_used += subd_smooth.memory_used();
+  mem_used += subd_start_corner.memory_used();
+  mem_used += transforms.memory_used();
+  mem_used += triangles.memory_used();
+  mem_used += uv_loops.memory_used();
+  mem_used += vertices.memory_used();
+
+  for (const CachedAttribute &attr : attributes) {
+    mem_used += attr.data.memory_used();
+  }
+
+  return mem_used;
+}
+
 static M44d convert_yup_zup(const M44d &mtx, float scale_mult)
 {
   V3d scale, shear, rotation, translation;
@@ -706,6 +736,9 @@ NODE_DEFINE(AlembicProcedural)
 
   SOCKET_NODE_ARRAY(objects, "Objects", AlembicObject::get_node_type());
 
+  SOCKET_BOOLEAN(use_prefetch, "Use Prefetch", true);
+  SOCKET_INT(prefetch_cache_size, "Prefetch Cache Size", 4096);
+
   return type;
 }
 
@@ -823,6 +856,30 @@ void AlembicProcedural::generate(Scene *scene, Progress &progress)
     }
   }
 
+  if (use_prefetch_is_modified()) {
+    if (!use_prefetch) {
+      for (Node *node : objects) {
+        AlembicObject *object = static_cast<AlembicObject *>(node);
+        object->clear_cache();
+      }
+    }
+  }
+
+  if (prefetch_cache_size_is_modified()) {
+    /* Check whether the current memory usage fits in the new requested size,
+     * abort the render if it is any higher. */
+    size_t memory_used = 0ul;
+    for (Node *node : objects) {
+      AlembicObject *object = static_cast<AlembicObject *>(node);
+      memory_used += object->get_cached_data().memory_used();
+    }
+
+    if (memory_used > get_prefetch_cache_size_in_bytes()) {
+      progress.set_error("Error: Alembic Procedural memory limit reached");
+      return;
+    }
+  }
+
   build_caches(progress);
 
   foreach (Node *node, objects) {
@@ -1300,6 +1357,8 @@ void AlembicProcedural::walk_hierarchy(
 
 void AlembicProcedural::build_caches(Progress &progress)
 {
+  size_t memory_used = 0;
+
   for (Node *node : objects) {
     AlembicObject *object = static_cast<AlembicObject *>(node);
 
@@ -1353,7 +1412,18 @@ void AlembicProcedural::build_caches(Progress &progress)
     if (scale_is_modified() || object->get_cached_data().transforms.size() == 0) {
       object->setup_transform_cache(object->get_cached_data(), scale);
     }
+
+    memory_used += object->get_cached_data().memory_used();
+
+    if (use_prefetch) {
+      if (memory_used > get_prefetch_cache_size_in_bytes()) {
+        progress.set_error("Error: Alembic Procedural memory limit reached");
+        return;
+      }
+    }
   }
+
+  VLOG(1) << "AlembicProcedural memory usage : " << string_human_readable_size(memory_used);
 }
 
 CCL_NAMESPACE_END

intern/cycles/render/alembic.h

@@ -272,6 +272,21 @@ template<typename T> class DataStore {
     node->set(*socket, value);
   }
 
+  size_t memory_used() const
+  {
+    if constexpr (is_array<T>::value) {
+      size_t mem_used = 0;
+
+      for (const T &array : data) {
+        mem_used += array.size() * sizeof(array[0]);
+      }
+
+      return mem_used;
+    }
+
+    return data.size() * sizeof(T);
+  }
+
  private:
   const TimeIndexPair &get_index_for_time(double time) const
   {
@@ -332,6 +347,8 @@ struct CachedData {
   void invalidate_last_loaded_time(bool attributes_only = false);
 
   void set_time_sampling(Alembic::AbcCoreAbstract::TimeSampling time_sampling);
+
+  size_t memory_used() const;
 };
 
 /* Representation of an Alembic object for the AlembicProcedural.
@@ -482,6 +499,13 @@ class AlembicProcedural : public Procedural {
    * software. */
   NODE_SOCKET_API(float, scale)
 
+  /* Cache controls */
+  NODE_SOCKET_API(bool, use_prefetch)
+
+  /* Memory limit for the cache, if the data does not fit within this limit, rendering is aborted.
+   */
+  NODE_SOCKET_API(int, prefetch_cache_size)
+
   AlembicProcedural();
   ~AlembicProcedural();
 
@@ -531,6 +555,12 @@ class AlembicProcedural : public Procedural {
   void read_subd(AlembicObject *abc_object, Alembic::AbcGeom::Abc::chrono_t frame_time);
 
   void build_caches(Progress &progress);
+
+  size_t get_prefetch_cache_size_in_bytes() const
+  {
+    /* prefetch_cache_size is in megabytes, so convert to bytes. */
+    return static_cast<size_t>(prefetch_cache_size) * 1024 * 1024;
+  }
 };
 
 CCL_NAMESPACE_END

intern/cycles/render/alembic_read.cpp

@@ -44,9 +44,19 @@ static set<chrono_t> get_relevant_sample_times(AlembicProcedural *proc,
     return result;
   }
 
-  // load the data for the entire animation
-  const double start_frame = static_cast<double>(proc->get_start_frame());
-  const double end_frame = static_cast<double>(proc->get_end_frame());
+  double start_frame;
+  double end_frame;
+
+  if (proc->get_use_prefetch()) {
+    // load the data for the entire animation
+    start_frame = static_cast<double>(proc->get_start_frame());
+    end_frame = static_cast<double>(proc->get_end_frame());
+  }
+  else {
+    // load the data for the current frame
+    start_frame = static_cast<double>(proc->get_frame());
+    end_frame = start_frame;
+  }
 
   const double frame_rate = static_cast<double>(proc->get_frame_rate());
   const double start_time = start_frame / frame_rate;

source/blender/editors/interface/interface_templates.c

@@ -6489,6 +6489,17 @@ void uiTemplateCacheFile(uiLayout *layout,
   uiLayoutSetActive(row, engine_supports_procedural);
   uiItemR(row, &fileptr, "use_render_procedural", 0, NULL, ICON_NONE);
 
+  const bool use_render_procedural = RNA_boolean_get(&fileptr, "use_render_procedural");
+  const bool use_prefetch = RNA_boolean_get(&fileptr, "use_prefetch");
+
+  row = uiLayoutRow(layout, false);
+  uiLayoutSetEnabled(row, use_render_procedural);
+  uiItemR(row, &fileptr, "use_prefetch", 0, NULL, ICON_NONE);
+
+  sub = uiLayoutRow(layout, false);
+  uiLayoutSetEnabled(sub, use_prefetch && use_render_procedural);
+  uiItemR(sub, &fileptr, "prefetch_cache_size", 0, NULL, ICON_NONE);
+
   row = uiLayoutRowWithHeading(layout, true, IFACE_("Override Frame"));
   sub = uiLayoutRow(row, true);
   uiLayoutSetPropDecorate(sub, false);
@@ -6510,6 +6521,10 @@ void uiTemplateCacheFile(uiLayout *layout,
   uiItemR(layout, &fileptr, "velocity_name", 0, NULL, ICON_NONE);
   uiItemR(layout, &fileptr, "velocity_unit", 0, NULL, ICON_NONE);
 
+  row = uiLayoutRow(layout, false);
+  uiLayoutSetActive(row, engine_supports_procedural && use_render_procedural);
+  uiItemR(row, &fileptr, "default_radius", 0, NULL, ICON_NONE);
+
   /* TODO: unused for now, so no need to expose. */
 #if 0
   row = uiLayoutRow(layout, false);

source/blender/makesdna/DNA_cachefile_defaults.h

@@ -40,6 +40,8 @@
     .handle = NULL, \
     .handle_filepath[0] = '\0', \
     .handle_readers = NULL, \
+    .use_prefetch = 1, \
+    .prefetch_cache_size = 4096, \
   }
 
 /** \} */

source/blender/makesdna/DNA_cachefile_types.h

@@ -101,7 +101,15 @@ typedef struct CacheFile {
    */
   char use_render_procedural;
 
-  char _pad1[7];
+  char _pad1[3];
+
+  /** Enable data prefetching when using the Cycles Procedural. */
+  char use_prefetch;
+
+  /** Size in megabytes for the prefetch cache used by the Cycles Procedural. */
+  int prefetch_cache_size;
+
+  char _pad2[7];
 
   char velocity_unit;
   /* Name of the velocity property in the archive. */

source/blender/makesrna/intern/rna_cachefile.c

@@ -150,6 +150,23 @@ static void rna_def_cachefile(BlenderRNA *brna)
                            "determine which file to use in a file sequence");
   RNA_def_property_update(prop, 0, "rna_CacheFile_update");
 
+  /* ----------------- Cache controls ----------------- */
+
+  prop = RNA_def_property(srna, "use_prefetch", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_ui_text(
+      prop,
+      "Use Prefetch",
+      "When enabled, the Cycles Procedural will preload animation data for faster updates");
+  RNA_def_property_update(prop, 0, "rna_CacheFile_update");
+
+  prop = RNA_def_property(srna, "prefetch_cache_size", PROP_INT, PROP_UNSIGNED);
+  RNA_def_property_ui_text(
+      prop,
+      "Prefetch Cache Size",
+      "Memory usage limit in megabytes for the Cycles Procedural cache, if the data does not "
+      "fit within the limit, rendering is aborted");
+  RNA_def_property_update(prop, 0, "rna_CacheFile_update");
+
   /* ----------------- Axis Conversion ----------------- */
 
   prop = RNA_def_property(srna, "forward_axis", PROP_ENUM, PROP_NONE);