Build: upgrade OpenImageDenoise to 1.2.1

This requires ISPC for building OpenImageDenoise, so that is now added as a dependency as well. Blender itself does not need ISPC for building so it is not included as part of the precompiled libraries. Differential Revision: https://developer.blender.org/D7641
2020-06-25 12:44:39 +02:00 · 2020-06-25 12:44:39 +02:00 · d3283ef121
parent 6431b11381
commit d3283ef121
10 changed files with 149 additions and 130 deletions
--- a/build_files/build_environment/CMakeLists.txt
+++ b/build_files/build_environment/CMakeLists.txt
@ -97,6 +97,7 @@ if(UNIX)
 else()
  include(cmake/pugixml.cmake)
 endif()
+include(cmake/ispc.cmake)
 include(cmake/openimagedenoise.cmake)
 include(cmake/embree.cmake)
 include(cmake/xr_openxr.cmake)
--- a/build_files/build_environment/cmake/ispc.cmake
+++ b/build_files/build_environment/cmake/ispc.cmake
@ -0,0 +1,62 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ***** END GPL LICENSE BLOCK *****
+
+if(WIN32)
+  set(ISPC_EXTRA_ARGS_WIN
+    -DFLEX_EXECUTABLE=${LIBDIR}/flexbison/win_flex.exe
+    -DBISON_EXECUTABLE=${LIBDIR}/flexbison/win_bison.exe
+    -DM4_EXECUTABLE=${DOWNLOAD_DIR}/mingw/mingw64/msys/1.0/bin/m4.exe
+  )
+endif()
+
+set(ISPC_EXTRA_ARGS
+    -DARM_ENABLED=Off
+    -DISPC_NO_DUMPS=On
+    -DISPC_INCLUDE_EXAMPLES=Off
+    -DISPC_INCLUDE_TESTS=Off
+    -DLLVM_ROOT=${LIBDIR}/llvm/lib/cmake/llvm
+    -DCLANG_EXECUTABLE=${LIBDIR}/clang/bin/clang
+    -DISPC_INCLUDE_TESTS=Off
+    -DCLANG_LIBRARY_DIR=${LIBDIR}/clang/lib
+    -DCLANG_INCLUDE_DIRS=${LIBDIR}/clang/include
+    ${ISPC_EXTRA_ARGS_WIN}
+)
+
+ExternalProject_Add(external_ispc
+  URL ${ISPC_URI}
+  DOWNLOAD_DIR ${DOWNLOAD_DIR}
+  URL_HASH MD5=${ISPC_HASH}
+  PREFIX ${BUILD_DIR}/ispc
+  PATCH_COMMAND ${PATCH_CMD} -p 1 -d ${BUILD_DIR}/ispc/src/external_ispc < ${PATCH_DIR}/ispc.diff
+  CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/ispc -Wno-dev ${DEFAULT_CMAKE_FLAGS} ${ISPC_EXTRA_ARGS} ${BUILD_DIR}/ispc/src/external_ispc
+  INSTALL_DIR ${LIBDIR}/ispc
+)
+
+add_dependencies(
+  external_ispc
+  ll
+  external_clang
+)
+
+if(WIN32)
+  add_dependencies(
+    external_ispc
+    external_flexbison
+  )
+endif()
+
--- a/build_files/build_environment/cmake/openimagedenoise.cmake
+++ b/build_files/build_environment/cmake/openimagedenoise.cmake
@ -18,26 +18,41 @@


 set(OIDN_EXTRA_ARGS
-  -DWITH_EXAMPLE=OFF
-  -DWITH_TEST=OFF
+  -DOIDN_APPS=OFF
  -DTBB_ROOT=${LIBDIR}/tbb
  -DTBB_STATIC_LIB=${TBB_STATIC_LIBRARY}
  -DOIDN_STATIC_LIB=ON
+  -DOIDN_STATIC_RUNTIME=OFF
+  -DISPC_EXECUTABLE=${LIBDIR}/ispc/bin/ispc
 )

+if(WIN32)
+  set(OIDN_EXTRA_ARGS
+    ${OIDN_EXTRA_ARGS}
+    -DTBB_DEBUG_LIBRARY=${LIBDIR}/tbb/lib/tbb.lib
+    -DTBB_DEBUG_LIBRARY_MALLOC=${LIBDIR}/tbb/lib/tbbmalloc.lib
+  )
+else()
+  set(OIDN_EXTRA_ARGS
+    ${OIDN_EXTRA_ARGS}
+    -Dtbb_LIBRARY_RELEASE=${LIBDIR}/tbb/lib/tbb_static.a
+    -Dtbbmalloc_LIBRARY_RELEASE=${LIBDIR}/tbb/lib/tbbmalloc_static.a
+  )
+endif()
+
 ExternalProject_Add(external_openimagedenoise
  URL ${OIDN_URI}
  DOWNLOAD_DIR ${DOWNLOAD_DIR}
  URL_HASH MD5=${OIDN_HASH}
  PREFIX ${BUILD_DIR}/openimagedenoise
  CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/openimagedenoise ${DEFAULT_CMAKE_FLAGS} ${OIDN_EXTRA_ARGS}
-  PATCH_COMMAND ${PATCH_CMD} --verbose -p 1 -N -d ${BUILD_DIR}/openimagedenoise/src/external_openimagedenoise < ${PATCH_DIR}/openimagedenoise.diff
  INSTALL_DIR ${LIBDIR}/openimagedenoise
 )

 add_dependencies(
  external_openimagedenoise
  external_tbb
+  external_ispc
 )

 if(WIN32)
@ -46,7 +61,7 @@ if(WIN32)
      COMMAND ${CMAKE_COMMAND} -E copy_directory ${LIBDIR}/openimagedenoise/include ${HARVEST_TARGET}/openimagedenoise/include
      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/openimagedenoise/lib/openimagedenoise.lib ${HARVEST_TARGET}/openimagedenoise/lib/openimagedenoise.lib
      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/openimagedenoise/lib/common.lib ${HARVEST_TARGET}/openimagedenoise/lib/common.lib
-      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/openimagedenoise/lib/mkldnn.lib ${HARVEST_TARGET}/openimagedenoise/lib/mkldnn.lib
+      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/openimagedenoise/lib/dnnl.lib ${HARVEST_TARGET}/openimagedenoise/lib/dnnl.lib
      DEPENDEES install
    )
  endif()
@ -54,7 +69,7 @@ if(WIN32)
    ExternalProject_Add_Step(external_openimagedenoise after_install
      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/openimagedenoise/lib/openimagedenoise.lib ${HARVEST_TARGET}/openimagedenoise/lib/openimagedenoise_d.lib
      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/openimagedenoise/lib/common.lib ${HARVEST_TARGET}/openimagedenoise/lib/common_d.lib
-      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/openimagedenoise/lib/mkldnn.lib ${HARVEST_TARGET}/openimagedenoise/lib/mkldnn_d.lib
+      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/openimagedenoise/lib/dnnl.lib ${HARVEST_TARGET}/openimagedenoise/lib/dnnl_d.lib
      DEPENDEES install
    )
  endif()
--- a/build_files/build_environment/cmake/setup_mingw64.cmake
+++ b/build_files/build_environment/cmake/setup_mingw64.cmake
@ -128,6 +128,14 @@ if((EXISTS "${DOWNLOAD_DIR}/mingw/mingw64/bin/mingw-get.exe") AND (NOT EXISTS "$
  )
 endif()

+if((EXISTS "${DOWNLOAD_DIR}/mingw/mingw64/bin/mingw-get.exe") AND (NOT EXISTS "${DOWNLOAD_DIR}/mingw/mingw64/msys/1.0/bin/m4.exe"))
+  message("Installing m4")
+  execute_process(
+    COMMAND ${DOWNLOAD_DIR}/mingw/mingw64/bin/mingw-get install msys msys-m4
+    WORKING_DIRECTORY  ${DOWNLOAD_DIR}/mingw/mingw64/bin/
+  )
+endif()
+
 message("Checking for CoreUtils")
 # download old core_utils for pr.exe (ffmpeg needs it to build)
 if(NOT EXISTS "${DOWNLOAD_DIR}/coreutils-5.97-MSYS-1.0.11-snapshot.tar.bz2")
--- a/build_files/build_environment/cmake/tbb.cmake
+++ b/build_files/build_environment/cmake/tbb.cmake
@ -50,6 +50,13 @@ ExternalProject_Add(external_tbb
 if(WIN32)
  if(BUILD_MODE STREQUAL Release)
    ExternalProject_Add_Step(external_tbb after_install
+      # findtbb.cmake in some deps *NEEDS* to find tbb_debug.lib even if they are not going to use it
+      # to make that test pass, we place a copy with the right name in the lib folder.
+      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/tbb/lib/tbb.lib ${HARVEST_TARGET}/tbb/lib/tbb_debug.lib
+      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/tbb/lib/tbbmalloc.lib ${HARVEST_TARGET}/tbb/lib/tbbmalloc_debug.lib
+      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/tbb/lib/tbb.dll ${HARVEST_TARGET}/tbb/lib/tbb_debug.dll
+      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/tbb/lib/tbbmalloc.dll ${HARVEST_TARGET}/tbb/lib/tbbmalloc_debug.dll
+      # Normal collection of build artifacts
      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/tbb/lib/tbb.lib ${HARVEST_TARGET}/tbb/lib/tbb.lib
      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/tbb/lib/tbb.dll ${HARVEST_TARGET}/tbb/lib/tbb.dll
      COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/tbb/lib/tbbmalloc.lib ${HARVEST_TARGET}/tbb/lib/tbbmalloc.lib
--- a/build_files/build_environment/cmake/versions.cmake
+++ b/build_files/build_environment/cmake/versions.cmake
@ -297,9 +297,9 @@ set(USD_VERSION 20.05)
 set(USD_URI https://github.com/PixarAnimationStudios/USD/archive/v${USD_VERSION}.tar.gz)
 set(USD_HASH 6d679e739e7f65725d9c029e37dda9fc)

-set(OIDN_VERSION 1.0.0)
-set(OIDN_URI https://github.com/OpenImageDenoise/oidn/releases/download/v${OIDN_VERSION}/oidn-${OIDN_VERSION}.src.zip)
-set(OIDN_HASH 19fe67b0164e8f020ac8a4f520defe60)
+set(OIDN_VERSION 1.2.1)
+set(OIDN_URI https://github.com/OpenImageDenoise/oidn/releases/download/v${OIDN_VERSION}/oidn-${OIDN_VERSION}.src.tar.gz)
+set(OIDN_HASH cbebc1a25eb6de62af3a59e943063608)

 set(LIBGLU_VERSION 9.0.1)
 set(LIBGLU_URI ftp://ftp.freedesktop.org/pub/mesa/glu/glu-${LIBGLU_VERSION}.tar.xz)
@ -312,3 +312,6 @@ set(MESA_HASH d60828056d77bfdbae0970f9b15fb1be)
 set(XR_OPENXR_SDK_VERSION 1.0.8)
 set(XR_OPENXR_SDK_URI https://github.com/KhronosGroup/OpenXR-SDK/archive/release-${XR_OPENXR_SDK_VERSION}.tar.gz)
 set(XR_OPENXR_SDK_HASH c6de63d2e0f9029aa58dfa97cad8ce07)
+set(ISPC_VERSION v1.13.0)
+set(ISPC_URI https://github.com/ispc/ispc/archive/${ISPC_VERSION}.tar.gz)
+set(ISPC_HASH 4bf5e8d0020c4b9980faa702c1a6f25f)
--- a/build_files/build_environment/patches/ispc.diff
+++ b/build_files/build_environment/patches/ispc.diff
@ -0,0 +1,36 @@
+diff -Naur external_ispc/CMakeLists.txt external_ispc_fixed/CMakeLists.txt
+--- external_ispc/CMakeLists.txt	2020-04-23 17:29:06 -0600
+++ external_ispc_fixed/CMakeLists.txt	2020-05-05 09:01:09 -0600
+@@ -389,7 +389,7 @@
+ 
+ # Link against Clang libraries
+ foreach(clangLib ${CLANG_LIBRARY_LIST})
+-    find_library(${clangLib}Path NAMES ${clangLib} HINTS ${LLVM_LIBRARY_DIRS})
+    find_library(${clangLib}Path NAMES ${clangLib} HINTS ${LLVM_LIBRARY_DIRS} ${CLANG_LIBRARY_DIR})
+     list(APPEND CLANG_LIBRARY_FULL_PATH_LIST ${${clangLib}Path})
+ endforeach()
+ target_link_libraries(${PROJECT_NAME} ${CLANG_LIBRARY_FULL_PATH_LIST})
+diff -Naur orig/CMakeLists.txt external_ispc/CMakeLists.txt
+--- orig/CMakeLists.txt 2020-05-05 09:19:11 -0600
+++ external_ispc/CMakeLists.txt        2020-05-05 09:26:44 -0600
+@@ -333,7 +333,7 @@
+
+ # Include directories
+ target_include_directories(${PROJECT_NAME} PRIVATE
+-                           ${LLVM_INCLUDE_DIRS}
+                           ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS}
+                            ${CMAKE_CURRENT_SOURCE_DIR}/src
+                            ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR})
+ # Compile options
+diff -Naur orig/cmake/GenerateBuiltins.cmake.txt external_ispc/cmake/GenerateBuiltins.cmake.txt
+++ orig/cmake/GenerateBuiltins.cmake	2020-05-25 13:32:40.830803821 +0200
+++ external_ispc/cmake/GenerateBuiltins.cmake	2020-05-25 13:32:40.830803821 +0200
+@@ -97,6 +97,8 @@
+ 
+     if ("${bit}" STREQUAL "32" AND ${arch} STREQUAL "x86")
+         set(target_arch "i386")
+        # Blender: disable 32bit due to build issues on Linux and being unnecessary.
+        set(SKIP ON)
+     elseif ("${bit}" STREQUAL "64" AND ${arch} STREQUAL "x86")
+         set(target_arch "x86_64")
+     elseif ("${bit}" STREQUAL "32" AND ${arch} STREQUAL "arm")
--- a/build_files/build_environment/patches/openimagedenoise.diff
+++ b/build_files/build_environment/patches/openimagedenoise.diff
@ -1,120 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 70ec895..e616b63 100644
--- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -178,7 +178,9 @@ set_property(TARGET ${PROJECT_NAME} PROPERTY SOVERSION "0")
- ## Open Image Denoise examples
- ## ----------------------------------------------------------------------------
- 
-add_subdirectory(examples)
-+if(WITH_EXAMPLE)
-+  add_subdirectory(examples)
-+endif()
- 
- ## ----------------------------------------------------------------------------
- ## Open Image Denoise install and packaging
-Submodule mkl-dnn contains modified content
-diff --git a/mkl-dnn/cmake/TBB.cmake b/mkl-dnn/cmake/TBB.cmake
-index 0711e699..c14210b6 100644
--- a/mkl-dnn/cmake/TBB.cmake
-+++ b/mkl-dnn/cmake/TBB.cmake
-@@ -138,13 +138,13 @@ else()
-         set(TBB_LIBRARY_MALLOC TBB_LIBRARY_MALLOC-NOTFOUND)
-         if(APPLE)
-             find_path(TBB_INCLUDE_DIR tbb/task_scheduler_init.h PATHS ${TBB_ROOT}/include NO_DEFAULT_PATH)
-            find_library(TBB_LIBRARY tbb PATHS ${TBB_ROOT}/lib NO_DEFAULT_PATH)
-            find_library(TBB_LIBRARY_MALLOC tbbmalloc PATHS ${TBB_ROOT}/lib NO_DEFAULT_PATH)
-+            find_library(TBB_LIBRARY tbb_static PATHS ${TBB_ROOT}/lib NO_DEFAULT_PATH)
-+            find_library(TBB_LIBRARY_MALLOC tbbmalloc_static PATHS ${TBB_ROOT}/lib NO_DEFAULT_PATH)
-         else()
-             find_path(TBB_INCLUDE_DIR tbb/task_scheduler_init.h PATHS ${TBB_ROOT}/include NO_DEFAULT_PATH)
-             set(TBB_HINTS HINTS ${TBB_ROOT}/lib/intel64/gcc4.4 ${TBB_ROOT}/lib ${TBB_ROOT}/lib64 PATHS /usr/libx86_64-linux-gnu/)
-            find_library(TBB_LIBRARY tbb ${TBB_HINTS})
-            find_library(TBB_LIBRARY_MALLOC tbbmalloc ${TBB_HINTS})
-+            find_library(TBB_LIBRARY tbb_static ${TBB_HINTS})
-+            find_library(TBB_LIBRARY_MALLOC tbbmalloc_static ${TBB_HINTS})
-         endif()
-     endif()
- 
-diff '--ignore-matching-lines=:' -ur '--exclude=*.svn*' -u -r
--- a/cmake/install.cmake	2019-08-12 18:02:20.794402575 +0200
-+++ b/cmake/install.cmake	2019-08-12 18:06:07.470045703 +0200
-@@ -18,6 +18,13 @@
- ## Install library
- ## ----------------------------------------------------------------------------
- 
-+if(UNIX)
-+install(FILES
-+  ${CMAKE_BINARY_DIR}/libOpenImageDenoise.a
-+  ${CMAKE_BINARY_DIR}/libmkldnn.a
-+  ${CMAKE_BINARY_DIR}/libcommon.a
-+  DESTINATION ${CMAKE_INSTALL_LIBDIR})
-+else()
- install(TARGETS ${PROJECT_NAME}
-   EXPORT
-     ${PROJECT_NAME}_Export
-@@ -38,6 +45,7 @@
-       DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel
-   )
- endif()
-+endif()
- 
- ## ----------------------------------------------------------------------------
- ## Install headers
-@@ -78,6 +86,7 @@
- ## Install CMake configuration files
- ## ----------------------------------------------------------------------------
- 
-+if(NOT UNIX)
- install(EXPORT ${PROJECT_NAME}_Export
-   DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
-   #NAMESPACE ${PROJECT_NAME}::
-@@ -92,3 +101,4 @@
-   DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
-   COMPONENT devel
- )
-+endif()
-diff '--ignore-matching-lines=:' -ur '--exclude=*.svn*' -u -r
--- a/CMakeLists.txt	2019-08-12 14:22:00.974078598 +0200
-+++ b/CMakeLists.txt	2019-08-12 18:05:05.949057375 +0200
-@@ -14,7 +14,11 @@
- ## limitations under the License.                                           ##
- ## ======================================================================== ##
- 
-cmake_minimum_required(VERSION 3.1)
-+if(UNIX)
-+  cmake_minimum_required(VERSION 3.1)
-+else()
-+  cmake_minimum_required(VERSION 3.13)
-+endif()
- 
- set(OIDN_VERSION_MAJOR 1)
- set(OIDN_VERSION_MINOR 0)
-@@ -32,13 +36,8 @@
- set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake")
- 
- # Build as shared or static library
-if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0")
-  option(OIDN_STATIC_LIB "Build Open Image Denoise as a static library.")
-  mark_as_advanced(CLEAR OIDN_STATIC_LIB)
-else()
-  set(OIDN_STATIC_LIB OFF CACHE BOOL "Build Open Image Denoise as a static library." FORCE)
-  mark_as_advanced(OIDN_STATIC_LIB)
-endif()
-+option(OIDN_STATIC_LIB "Build Open Image Denoise as a static library.")
-+mark_as_advanced(CLEAR OIDN_STATIC_LIB)
- if(OIDN_STATIC_LIB)
-   set(OIDN_LIB_TYPE STATIC)
- else()
-diff -Naur orig/core/api.cpp external_openimagedenoise/core/api.cpp
--- orig/core/api.cpp	2019-07-19 08:37:04 -0600
-+++ external_openimagedenoise/core/api.cpp	2020-01-21 15:10:56 -0700
-@@ -15,7 +15,7 @@
- // ======================================================================== //
- 
- #ifdef _WIN32
-#  define OIDN_API extern "C" __declspec(dllexport)
-+#  define OIDN_API extern "C" 
- #else
- #  define OIDN_API extern "C" __attribute__ ((visibility ("default")))
- #endif
--- a/build_files/cmake/Modules/FindOpenImageDenoise.cmake
+++ b/build_files/cmake/Modules/FindOpenImageDenoise.cmake
@ -48,7 +48,14 @@ SET(_openimagedenoise_FIND_COMPONENTS
 # These are needed when building statically
 SET(_openimagedenoise_FIND_STATIC_COMPONENTS
  common
+
+  # These additional library names change between versions, we list all of them
+  # so builds work with multiple versions. Missing libraries are skipped.
+  dnnl_cpu
+  dnnl_common
+  dnnl_cpu # Second time because of circular dependency
  mkldnn
+  dnnl
 )

 SET(_openimagedenoise_LIBRARIES)
--- a/build_files/cmake/platform/platform_win32.cmake
+++ b/build_files/cmake/platform/platform_win32.cmake
@ -539,10 +539,10 @@ if(WITH_OPENIMAGEDENOISE)
  set(OPENIMAGEDENOISE_LIBRARIES
    optimized ${OPENIMAGEDENOISE_LIBPATH}/OpenImageDenoise.lib
    optimized ${OPENIMAGEDENOISE_LIBPATH}/common.lib
-    optimized ${OPENIMAGEDENOISE_LIBPATH}/mkldnn.lib
+    optimized ${OPENIMAGEDENOISE_LIBPATH}/dnnl.lib
    debug ${OPENIMAGEDENOISE_LIBPATH}/OpenImageDenoise_d.lib
    debug ${OPENIMAGEDENOISE_LIBPATH}/common_d.lib
-    debug ${OPENIMAGEDENOISE_LIBPATH}/mkldnn_d.lib)
+    debug ${OPENIMAGEDENOISE_LIBPATH}/dnnl_d.lib)
  set(OPENIMAGEDENOISE_DEFINITIONS)
 endif()