Fluid: Fixed slow cache loading for smoke data

Cache files are currently loaded via the Manta Python API. With very big caches this can slow down the viewport playback. Especially smoke simulations, which just load grids and no meshes, can suffer from this. This fix solves this problem by directly loading the cache files from disk (no Python). This fix has been in the works for some time. The developer of this patch is ready to handle any potential fall-out of this patch quickly.

intern/mantaflow/CMakeLists.txt

@@ -47,6 +47,35 @@ set(INC_SYS
   ${ZLIB_INCLUDE_DIRS}
 )
 
+if(WITH_TBB)
+  list(APPEND INC_SYS
+    ${TBB_INCLUDE_DIRS}
+  )
+  list(APPEND LIB
+    ${TBB_LIBRARIES}
+  )
+endif()
+
+if(WITH_OPENVDB)
+  list(APPEND INC_SYS
+    ${BOOST_INCLUDE_DIR}
+    ${OPENEXR_INCLUDE_DIRS}
+    ${OPENVDB_INCLUDE_DIRS}
+  )
+  list(APPEND LIB
+    ${OPENVDB_LIBRARIES}
+    ${OPENEXR_LIBRARIES}
+    ${ZLIB_LIBRARIES}
+    ${BOOST_LIBRARIES}
+  )
+  if(WITH_OPENVDB_BLOSC)
+    list(APPEND LIB
+      ${BLOSC_LIBRARIES}
+      ${ZLIB_LIBRARIES}
+    )
+  endif()
+endif()
+
 set(SRC
   intern/manta_python_API.cpp
   intern/manta_fluid_API.cpp

intern/mantaflow/extern/manta_fluid_API.h

@@ -55,6 +55,8 @@ int manta_update_mesh_structures(struct MANTA *fluid, struct FluidModifierData *
 int manta_update_particle_structures(struct MANTA *fluid,
                                      struct FluidModifierData *mmd,
                                      int framenr);
+int manta_update_smoke_structures(struct MANTA *fluid, struct FluidModifierData *mmd, int framenr);
+int manta_update_noise_structures(struct MANTA *fluid, struct FluidModifierData *mmd, int framenr);
 int manta_bake_data(struct MANTA *fluid, struct FluidModifierData *mmd, int framenr);
 int manta_bake_noise(struct MANTA *fluid, struct FluidModifierData *mmd, int framenr);
 int manta_bake_mesh(struct MANTA *fluid, struct FluidModifierData *mmd, int framenr);

intern/mantaflow/intern/MANTA_main.cpp

@@ -27,6 +27,10 @@
 #include <iomanip>
 #include <zlib.h>
 
+#if OPENVDB == 1
+#  include "openvdb/openvdb.h"
+#endif
+
 #include "MANTA_main.h"
 #include "manta.h"
 #include "Python.h"
@@ -1168,6 +1172,244 @@ int MANTA::updateParticleStructures(FluidModifierData *mmd, int framenr)
   return 1;
 }
 
+int MANTA::updateSmokeStructures(FluidModifierData *mmd, int framenr)
+{
+  if (MANTA::with_debug)
+    std::cout << "MANTA::updateGridStructures()" << std::endl;
+
+  mSmokeFromFile = false;
+
+  if (!mUsingSmoke)
+    return 0;
+  if (BLI_path_is_rel(mmd->domain->cache_directory))
+    return 0;
+
+  int result = 0;
+  int expected = 0; /* Expected number of read successes for this frame. */
+
+  std::ostringstream ss;
+  char cacheDir[FILE_MAX], targetFile[FILE_MAX];
+  cacheDir[0] = '\0';
+  targetFile[0] = '\0';
+
+  std::string dformat = getCacheFileEnding(mmd->domain->cache_data_format);
+  BLI_path_join(
+      cacheDir, sizeof(cacheDir), mmd->domain->cache_directory, FLUID_DOMAIN_DIR_DATA, nullptr);
+
+  expected += 1;
+  ss.str("");
+  ss << "density_####" << dformat;
+  BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str());
+  BLI_path_frame(targetFile, framenr, 0);
+  if (!BLI_exists(targetFile)) {
+    return 0;
+  }
+  result += updateGridFromFile(targetFile, mDensity);
+
+  expected += 1;
+  ss.str("");
+  ss << "shadow_####" << dformat;
+  BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str());
+  BLI_path_frame(targetFile, framenr, 0);
+  if (!BLI_exists(targetFile)) {
+    return 0;
+  }
+  result += updateGridFromFile(targetFile, mShadow);
+
+  if (mUsingHeat) {
+    expected += 1;
+    ss.str("");
+    ss << "heat_####" << dformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mHeat);
+  }
+
+  if (mUsingColors) {
+    expected += 3;
+    ss.str("");
+    ss << "color_r_####" << dformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mColorR);
+
+    ss.str("");
+    ss << "color_g_####" << dformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mColorG);
+
+    ss.str("");
+    ss << "color_b_####" << dformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mColorB);
+  }
+
+  if (mUsingFire) {
+    expected += 3;
+    ss.str("");
+    ss << "flame_####" << dformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mFlame);
+
+    ss.str("");
+    ss << "fuel_####" << dformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mFuel);
+
+    ss.str("");
+    ss << "react_####" << dformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDir, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mReact);
+  }
+
+  mSmokeFromFile = true;
+  return (result == expected) ? 1 : 0;
+}
+
+int MANTA::updateNoiseStructures(FluidModifierData *mmd, int framenr)
+{
+  if (MANTA::with_debug)
+    std::cout << "MANTA::updateNoiseStructures()" << std::endl;
+
+  mNoiseFromFile = false;
+
+  if (!mUsingSmoke || !mUsingNoise)
+    return 0;
+  if (BLI_path_is_rel(mmd->domain->cache_directory))
+    return 0;
+
+  int result = 0;
+  int expected = 0; /* Expected number of read successes for this frame. */
+
+  std::ostringstream ss;
+  char cacheDirData[FILE_MAX], cacheDirNoise[FILE_MAX], targetFile[FILE_MAX];
+  cacheDirData[0] = '\0';
+  cacheDirNoise[0] = '\0';
+  targetFile[0] = '\0';
+
+  std::string dformat = getCacheFileEnding(mmd->domain->cache_data_format);
+  std::string nformat = getCacheFileEnding(mmd->domain->cache_noise_format);
+  BLI_path_join(cacheDirData,
+                sizeof(cacheDirData),
+                mmd->domain->cache_directory,
+                FLUID_DOMAIN_DIR_DATA,
+                nullptr);
+  BLI_path_join(cacheDirNoise,
+                sizeof(cacheDirNoise),
+                mmd->domain->cache_directory,
+                FLUID_DOMAIN_DIR_NOISE,
+                nullptr);
+
+  expected += 1;
+  ss.str("");
+  ss << "density_noise_####" << nformat;
+  BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDirNoise, ss.str().c_str());
+  BLI_path_frame(targetFile, framenr, 0);
+  if (!BLI_exists(targetFile)) {
+    return 0;
+  }
+  result += updateGridFromFile(targetFile, mDensityHigh);
+
+  expected += 1;
+  ss.str("");
+  ss << "shadow_####" << dformat;
+  BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDirData, ss.str().c_str());
+  BLI_path_frame(targetFile, framenr, 0);
+  if (!BLI_exists(targetFile)) {
+    return 0;
+  }
+  result += updateGridFromFile(targetFile, mShadow);
+
+  if (mUsingColors) {
+    expected += 3;
+    ss.str("");
+    ss << "color_r_noise_####" << nformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDirNoise, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mColorRHigh);
+
+    ss.str("");
+    ss << "color_g_noise_####" << nformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDirNoise, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mColorGHigh);
+
+    ss.str("");
+    ss << "color_b_noise_####" << nformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDirNoise, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mColorBHigh);
+  }
+
+  if (mUsingFire) {
+    expected += 3;
+    ss.str("");
+    ss << "flame_noise_####" << nformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDirNoise, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mFlameHigh);
+
+    ss.str("");
+    ss << "fuel_noise_####" << nformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDirNoise, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mFuelHigh);
+
+    ss.str("");
+    ss << "react_noise_####" << nformat;
+    BLI_join_dirfile(targetFile, sizeof(targetFile), cacheDirNoise, ss.str().c_str());
+    BLI_path_frame(targetFile, framenr, 0);
+    if (!BLI_exists(targetFile)) {
+      return 0;
+    }
+    result += updateGridFromFile(targetFile, mReactHigh);
+  }
+
+  mNoiseFromFile = true;
+  return (result == expected) ? 1 : 0;
+}
+
 /* Dirty hack: Needed to format paths from python code that is run via PyRun_SimpleString */
 static std::string escapeSlashes(std::string const &s)
 {
@@ -1194,7 +1436,7 @@ int MANTA::writeConfiguration(FluidModifierData *mmd, int framenr)
   cacheDir[0] = '\0';
   targetFile[0] = '\0';
 
-  std::string dformat = getCacheFileEnding(mmd->domain->cache_data_format);
+  std::string dformat = ".uni";
 
   BLI_path_join(
       cacheDir, sizeof(cacheDir), mmd->domain->cache_directory, FLUID_DOMAIN_DIR_CONFIG, nullptr);
@@ -1287,7 +1529,7 @@ int MANTA::readConfiguration(FluidModifierData *mmd, int framenr)
   targetFile[0] = '\0';
   float dummy;
 
-  std::string dformat = getCacheFileEnding(mmd->domain->cache_data_format);
+  std::string dformat = ".uni";
 
   BLI_path_join(
       cacheDir, sizeof(cacheDir), mmd->domain->cache_directory, FLUID_DOMAIN_DIR_CONFIG, nullptr);
@@ -2533,6 +2775,174 @@ void MANTA::updateParticlesFromUni(const char *filename, bool isSecondarySys, bo
   gzclose(gzf);
 }
 
+int MANTA::updateGridFromFile(const char *filename, float *grid)
+{
+  if (with_debug)
+    std::cout << "MANTA::updateGridFromFile()" << std::endl;
+
+  if (!grid) {
+    std::cout << "MANTA::updateGridFromFile(): cannot read into uninitialized grid, grid is null"
+              << std::endl;
+    return 0;
+  }
+
+  std::string fname(filename);
+  std::string::size_type idx;
+
+  idx = fname.rfind('.');
+  if (idx != std::string::npos) {
+    std::string extension = fname.substr(idx + 1);
+
+    if (extension.compare("uni") == 0)
+      return updateGridFromUni(filename, grid);
+    else if (extension.compare("vdb") == 0)
+      return updateGridFromVDB(filename, grid);
+    else if (extension.compare("raw") == 0)
+      return updateGridFromRaw(filename, grid);
+    else
+      std::cerr << "MANTA::updateGridFromFile(): invalid file extension in file: " << filename
+                << std::endl;
+    return 0;
+  }
+  else {
+    std::cerr << "MANTA::updateGridFromFile(): unable to open file: " << filename << std::endl;
+    return 0;
+  }
+}
+
+int MANTA::updateGridFromUni(const char *filename, float *grid)
+{
+  if (with_debug)
+    std::cout << "MANTA::updateGridFromUni()" << std::endl;
+
+  gzFile gzf;
+  int ibuffer[4];
+
+  gzf = (gzFile)BLI_gzopen(filename, "rb1");
+  if (!gzf) {
+    std::cout << "MANTA::updateGridFromUni(): unable to open file" << std::endl;
+    return 0;
+  }
+
+  char ID[5] = {0, 0, 0, 0, 0};
+  gzread(gzf, ID, 4);
+
+  if (!strcmp(ID, "DDF2")) {
+    std::cout << "MANTA::updateGridFromUni(): grid uni file format DDF2 not supported anymore"
+              << std::endl;
+    return 0;
+  }
+  if (!strcmp(ID, "MNT1")) {
+    std::cout << "MANTA::updateGridFromUni(): grid uni file format MNT1 not supported anymore"
+              << std::endl;
+    return 0;
+  }
+  if (!strcmp(ID, "MNT2")) {
+    std::cout << "MANTA::updateGridFromUni(): grid uni file format MNT2 not supported anymore"
+              << std::endl;
+    return 0;
+  }
+
+  // grid uni header
+  const int STR_LEN_GRID = 252;
+  int elementType, bytesPerElement;  // data type info
+  char info[STR_LEN_GRID];           // mantaflow build information
+  int dimT;                          // optionally store forth dimension for 4d grids
+  unsigned long long timestamp;      // creation time
+
+  // read grid header
+  gzread(gzf, &ibuffer, sizeof(int) * 4);  // dimX, dimY, dimZ, gridType
+  gzread(gzf, &elementType, sizeof(int));
+  gzread(gzf, &bytesPerElement, sizeof(int));
+  gzread(gzf, &info, sizeof(info));
+  gzread(gzf, &dimT, sizeof(int));
+  gzread(gzf, &timestamp, sizeof(unsigned long long));
+
+  if (with_debug)
+    std::cout << "read " << ibuffer[3] << " grid type in file: " << filename << std::endl;
+
+  // Sanity checks
+  if (ibuffer[0] != mResX || ibuffer[1] != mResY || ibuffer[2] != mResZ) {
+    std::cout << "grid dim doesn't match, read: (" << ibuffer[0] << ", " << ibuffer[1] << ", "
+              << ibuffer[2] << ") vs setup: (" << mResX << ", " << mResY << ", " << mResZ << ")"
+              << std::endl;
+    return 0;
+  }
+
+  // Actual data reading
+  if (!strcmp(ID, "MNT3")) {
+    gzread(gzf, grid, sizeof(float) * ibuffer[0] * ibuffer[1] * ibuffer[2]);
+  }
+
+  if (with_debug)
+    std::cout << "read successfully: " << filename << std::endl;
+
+  gzclose(gzf);
+  return 1;
+}
+
+int MANTA::updateGridFromVDB(const char *filename, float *grid)
+{
+  if (with_debug)
+    std::cout << "MANTA::updateGridFromVDB()" << std::endl;
+
+  openvdb::initialize();
+  openvdb::io::File file(filename);
+  try {
+    file.open();
+  }
+  catch (const openvdb::v5_1::IoError) {
+    std::cout << "MANTA::updateGridFromVDB(): IOError, invalid OpenVDB file: " << filename
+              << std::endl;
+    return 0;
+  }
+
+  openvdb::GridBase::Ptr baseGrid;
+  for (openvdb::io::File::NameIterator nameIter = file.beginName(); nameIter != file.endName();
+       ++nameIter) {
+    baseGrid = file.readGrid(nameIter.gridName());
+    break;
+  }
+  file.close();
+  openvdb::FloatGrid::Ptr gridVDB = openvdb::gridPtrCast<openvdb::FloatGrid>(baseGrid);
+  openvdb::FloatGrid::Accessor accessor = gridVDB->getAccessor();
+
+  size_t index = 0;
+  for (int z = 0; z < mResZ; ++z) {
+    for (int y = 0; y < mResY; ++y) {
+      for (int x = 0; x < mResX; ++x, ++index) {
+        openvdb::Coord xyz(x, y, z);
+        float v = accessor.getValue(xyz);
+        grid[index] = v;
+      }
+    }
+  }
+  return 1;
+}
+
+int MANTA::updateGridFromRaw(const char *filename, float *grid)
+{
+  if (with_debug)
+    std::cout << "MANTA::updateGridFromRaw()" << std::endl;
+
+  gzFile gzf;
+  int expectedBytes, readBytes;
+
+  gzf = (gzFile)BLI_gzopen(filename, "rb");
+  if (!gzf) {
+    std::cout << "MANTA::updateGridFromRaw(): unable to open file" << std::endl;
+    return 0;
+  }
+
+  expectedBytes = sizeof(float) * mResX * mResY * mResZ;
+  readBytes = gzread(gzf, grid, expectedBytes);
+
+  assert(expectedBytes == readBytes);
+
+  gzclose(gzf);
+  return 1;
+}
+
 void MANTA::updatePointers()
 {
   if (with_debug)
@@ -2660,7 +3070,9 @@ void MANTA::updatePointers()
         callPythonFunction("pLifeSnd" + parts_ext, func));
   }
 
-  mFlipFromFile = true;
+  mFlipFromFile = false;
   mMeshFromFile = false;
   mParticlesFromFile = false;
+  mSmokeFromFile = false;
+  mNoiseFromFile = false;
 }

intern/mantaflow/intern/MANTA_main.h

@@ -93,6 +93,8 @@ struct MANTA {
   int updateMeshStructures(FluidModifierData *mmd, int framenr);
   int updateFlipStructures(FluidModifierData *mmd, int framenr);
   int updateParticleStructures(FluidModifierData *mmd, int framenr);
+  int updateSmokeStructures(FluidModifierData *mmd, int framenr);
+  int updateNoiseStructures(FluidModifierData *mmd, int framenr);
   void updateVariables(FluidModifierData *mmd);
 
   // Bake cache
@@ -742,6 +744,8 @@ struct MANTA {
   bool mFlipFromFile;
   bool mMeshFromFile;
   bool mParticlesFromFile;
+  bool mSmokeFromFile;
+  bool mNoiseFromFile;
 
   int mResX;
   int mResY;
@@ -852,8 +856,12 @@ struct MANTA {
   void updateMeshFromObj(const char *filename);
   void updateMeshFromUni(const char *filename);
   void updateParticlesFromUni(const char *filename, bool isSecondarySys, bool isVelData);
+  int updateGridFromUni(const char *filename, float *grid);
+  int updateGridFromVDB(const char *filename, float *grid);
+  int updateGridFromRaw(const char *filename, float *grid);
   void updateMeshFromFile(const char *filename);
   void updateParticlesFromFile(const char *filename, bool isSecondarySys, bool isVelData);
+  int updateGridFromFile(const char *filename, float *grid);
 };
 
 #endif

intern/mantaflow/intern/manta_fluid_API.cpp

@@ -143,6 +143,20 @@ int manta_update_particle_structures(MANTA *fluid, FluidModifierData *mmd, int f
   return fluid->updateParticleStructures(mmd, framenr);
 }
 
+int manta_update_smoke_structures(MANTA *fluid, FluidModifierData *mmd, int framenr)
+{
+  if (!fluid || !mmd)
+    return 0;
+  return fluid->updateSmokeStructures(mmd, framenr);
+}
+
+int manta_update_noise_structures(MANTA *fluid, FluidModifierData *mmd, int framenr)
+{
+  if (!fluid || !mmd)
+    return 0;
+  return fluid->updateNoiseStructures(mmd, framenr);
+}
+
 int manta_bake_data(MANTA *fluid, FluidModifierData *mmd, int framenr)
 {
   if (!fluid || !mmd)

source/blender/blenkernel/intern/fluid.c

@@ -3505,10 +3505,14 @@ static void BKE_fluid_modifier_processDomain(FluidModifierData *mmd,
           manta_needs_realloc(mds->fluid, mmd)) {
         BKE_fluid_reallocate_fluid(mds, mds->res, 1);
       }
-      has_noise = manta_read_noise(mds->fluid, mmd, noise_frame);
+      if (!baking_data && !baking_noise && !mode_replay) {
+        has_data = manta_update_noise_structures(mds->fluid, mmd, noise_frame);
+      }
+      else {
+        has_noise = manta_read_noise(mds->fluid, mmd, noise_frame);
+      }
 
-      /* In case of using the adaptive domain, copy all data that was read to a new fluid object.
-       */
+      /* When using the adaptive domain, copy all data that was read to a new fluid object. */
       if (with_adaptive && baking_noise) {
         /* Adaptive domain needs to know about current state, so save it, then copy. */
         copy_v3_v3_int(o_res, mds->res);
@@ -3521,7 +3525,13 @@ static void BKE_fluid_modifier_processDomain(FluidModifierData *mmd,
               mds, o_res, mds->res, o_min, mds->res_min, o_max, o_shift, mds->shift);
         }
       }
-      has_data = manta_read_data(mds->fluid, mmd, data_frame);
+      if (!baking_data && !baking_noise && !mode_replay) {
+        /* There is no need to call manta_update_smoke_structures() here.
+         * The noise cache has already been read with manta_update_noise_structures(). */
+      }
+      else {
+        has_data = manta_read_data(mds->fluid, mmd, data_frame);
+      }
     }
     /* Read data cache only */
     else {
@@ -3532,7 +3542,12 @@ static void BKE_fluid_modifier_processDomain(FluidModifierData *mmd,
           BKE_fluid_reallocate_fluid(mds, mds->res, 1);
         }
         /* Read data cache */
-        has_data = manta_read_data(mds->fluid, mmd, data_frame);
+        if (!baking_data && !baking_particles && !baking_mesh && !mode_replay) {
+          has_data = manta_update_smoke_structures(mds->fluid, mmd, data_frame);
+        }
+        else {
+          has_data = manta_read_data(mds->fluid, mmd, data_frame);
+        }
       }
       if (with_liquid) {
         if (!baking_data && !baking_particles && !baking_mesh && !mode_replay) {