Deps builder: OIIO/OSL/ISPC/OIDN/LLVM/Flex updates

This diff somewhat snowballed out of updating OIDN to 1.4.1 it had some
changes that allowed us to remove the arm hacks we had in place and
revert to using identical versions for a whole bunch of deps. But that
required an update to ISPC which needed a newer LLVM and if we're
updating LLVM we may as well update OSL, and when we update OSL, OIIO
may as well be dragged in soo......anyhow...

This diff updates:

LLVM 9.0.0 (11.0.1 for mac/arm) -> 12.0.0
OIIO 2.1.15.0 -> 2.2.15.1
OSL 1.11.10.0 -> 1.11.14.1
winflex_bison 2.5.5-> 2.5.24 (ispc needed newer bison, windows only dep)
OIDN 1.4.0 -> 1.4.1
ISPC v1.14.1(random hash for mac/arm) -> v1.16.0
Flex 2.6.4 (ISPC needed newer Flex than available on CentOS 7)

and removes most of the "special arm/mac" versions. I think just ssl and
embree are left with special versions.

notable changes:
@LazyDodo included some clang headers in the linux/mac harvest which are
needed to start writing custom clang based tooling like D9465 these were
already shipping on windows, but not the other platforms.

[macOS] Change the `LC_ID_DYLIB` of OpenMP  for {D11997}. This changes
where the executables look for dylibs.

Reviewed By: sebbas, LazyDodo

Differential Revision: https://developer.blender.org/D11748
This commit is contained in:
Sybren A. Stüvel 2021-07-27 10:15:31 +02:00
parent 5a07174ce3
commit c5e5ac4a17
13 changed files with 134 additions and 128 deletions

View File

@ -82,7 +82,11 @@ if(UNIX)
endif()
include(cmake/openimageio.cmake)
include(cmake/tiff.cmake)
include(cmake/flexbison.cmake)
if(WIN32)
include(cmake/flexbison.cmake)
else()
include(cmake/flex.cmake)
endif()
include(cmake/osl.cmake)
include(cmake/tbb.cmake)
include(cmake/openvdb.cmake)

View File

@ -93,3 +93,4 @@ download_source(GMP)
download_source(POTRACE)
download_source(HARU)
download_source(ZSTD)
download_source(FLEX)

View File

@ -0,0 +1,28 @@
# ***** BEGIN GPL LICENSE BLOCK *****
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ***** END GPL LICENSE BLOCK *****
ExternalProject_Add(external_flex
URL file://${PACKAGE_DIR}/${FLEX_FILE}
URL_HASH ${FLEX_HASH_TYPE}=${FLEX_HASH}
DOWNLOAD_DIR ${DOWNLOAD_DIR}
PREFIX ${BUILD_DIR}/flex
CONFIGURE_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/flex/src/external_flex/ && ${CONFIGURE_COMMAND} --prefix=${LIBDIR}/flex
BUILD_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/flex/src/external_flex/ && make -j${MAKE_THREADS}
INSTALL_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/flex/src/external_flex/ && make install
INSTALL_DIR ${LIBDIR}/flex
)

View File

@ -106,6 +106,7 @@ harvest(llvm/include llvm/include "*")
harvest(llvm/bin llvm/bin "llvm-config")
harvest(llvm/lib llvm/lib "libLLVM*.a")
harvest(llvm/lib llvm/lib "libclang*.a")
harvest(llvm/lib/clang llvm/lib/clang "*.h")
if(APPLE)
harvest(openmp/lib openmp/lib "*")
harvest(openmp/include openmp/include "*.h")

View File

@ -29,12 +29,13 @@ elseif(APPLE)
if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "arm64")
set(ISPC_EXTRA_ARGS_APPLE
-DBISON_EXECUTABLE=/opt/homebrew/opt/bison/bin/bison
-DFLEX_EXECUTABLE=/opt/homebrew/opt/flex/bin/flex
-DFLEX_EXECUTABLE=${LIBDIR}/flex/bin/flex
-DARM_ENABLED=On
)
else()
set(ISPC_EXTRA_ARGS_APPLE
-DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison
-DFLEX_EXECUTABLE=${LIBDIR}/flex/bin/flex
-DARM_ENABLED=Off
)
endif()
@ -43,6 +44,7 @@ elseif(UNIX)
-DCMAKE_C_COMPILER=${LIBDIR}/llvm/bin/clang
-DCMAKE_CXX_COMPILER=${LIBDIR}/llvm/bin/clang++
-DARM_ENABLED=Off
-DFLEX_EXECUTABLE=${LIBDIR}/flex/bin/flex
)
endif()
@ -82,4 +84,9 @@ if(WIN32)
external_ispc
external_flexbison
)
else()
add_dependencies(
external_ispc
external_flex
)
endif()

View File

@ -45,7 +45,6 @@ ExternalProject_Add(external_openimagedenoise
DOWNLOAD_DIR ${DOWNLOAD_DIR}
URL_HASH ${OIDN_HASH_TYPE}=${OIDN_HASH}
PREFIX ${BUILD_DIR}/openimagedenoise
PATCH_COMMAND ${PATCH_CMD} -p 1 -N -d ${BUILD_DIR}/openimagedenoise/src/external_openimagedenoise < ${PATCH_DIR}/oidn.diff
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/openimagedenoise ${DEFAULT_CMAKE_FLAGS} ${OIDN_EXTRA_ARGS}
INSTALL_DIR ${LIBDIR}/openimagedenoise
)

View File

@ -22,9 +22,8 @@ ExternalProject_Add(external_openmp
DOWNLOAD_DIR ${DOWNLOAD_DIR}
URL_HASH ${OPENMP_HASH_TYPE}=${OPENMP_HASH}
PREFIX ${BUILD_DIR}/openmp
PATCH_COMMAND ${PATCH_CMD} -p 1 -d ${BUILD_DIR}/openmp/src/external_openmp < ${PATCH_DIR}/openmp.diff
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/openmp ${DEFAULT_CMAKE_FLAGS}
INSTALL_COMMAND cd ${BUILD_DIR}/openmp/src/external_openmp-build && install_name_tool -id @executable_path/../Resources/lib/libomp.dylib runtime/src/libomp.dylib && make install
INSTALL_COMMAND cd ${BUILD_DIR}/openmp/src/external_openmp-build && install_name_tool -id @rpath/libomp.dylib runtime/src/libomp.dylib && make install
INSTALL_DIR ${LIBDIR}/openmp
)

View File

@ -51,6 +51,7 @@ set(OSL_EXTRA_ARGS
-DOpenImageIO_ROOT=${LIBDIR}/openimageio/
-DOSL_BUILD_TESTS=OFF
-DOSL_BUILD_MATERIALX=OFF
-DPNG_ROOT=${LIBDIR}/png
-DZLIB_LIBRARY=${LIBDIR}/zlib/lib/${ZLIB_LIBRARY}
-DZLIB_INCLUDE_DIR=${LIBDIR}/zlib/include/
${OSL_FLEX_BISON}
@ -69,13 +70,9 @@ set(OSL_EXTRA_ARGS
${OSL_SIMD_FLAGS}
-Dpugixml_ROOT=${LIBDIR}/pugixml
-DUSE_PYTHON=OFF
-DCMAKE_CXX_STANDARD=14
)
# Apple arm64 uses LLVM 11, LLVM 10+ requires C++14
if (APPLE AND "${CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")
list(APPEND OSL_EXTRA_ARGS -DCMAKE_CXX_STANDARD=14)
endif()
ExternalProject_Add(external_osl
URL file://${PACKAGE_DIR}/${OSL_FILE}
DOWNLOAD_DIR ${DOWNLOAD_DIR}
@ -93,10 +90,20 @@ add_dependencies(
ll
external_openexr
external_zlib
external_flexbison
external_openimageio
external_pugixml
)
if(WIN32)
add_dependencies(
external_osl
external_flexbison
)
else()
add_dependencies(
external_osl
external_flex
)
endif()
if(WIN32)
if(BUILD_MODE STREQUAL Release)

View File

@ -152,35 +152,20 @@ set(OPENCOLORIO_HASH 1a2e3478b6cd9a1549f24e1b2205e3f0)
set(OPENCOLORIO_HASH_TYPE MD5)
set(OPENCOLORIO_FILE OpenColorIO-${OPENCOLORIO_VERSION}.tar.gz)
if(BLENDER_PLATFORM_ARM)
# Newer version required by ISPC with arm support.
set(LLVM_VERSION 11.0.1)
set(LLVM_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz)
set(LLVM_HASH e700af40ab83463e4e9ab0ba3708312e)
set(LLVM_HASH_TYPE MD5)
set(LLVM_FILE llvm-project-${LLVM_VERSION}.src.tar.xz)
set(LLVM_VERSION 12.0.0)
set(LLVM_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz)
set(LLVM_HASH 5a4fab4d7fc84aefffb118ac2c8a4fc0)
set(LLVM_HASH_TYPE MD5)
set(LLVM_FILE llvm-project-${LLVM_VERSION}.src.tar.xz)
set(OPENMP_VERSION 9.0.1)
set(OPENMP_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${OPENMP_VERSION}/openmp-${OPENMP_VERSION}.src.tar.xz)
set(OPENMP_HASH 6eade16057edbdecb3c4eef9daa2bfcf)
set(OPENMP_HASH_TYPE MD5)
set(OPENMP_FILE openmp-${OPENMP_VERSION}.src.tar.xz)
else()
set(LLVM_VERSION 9.0.1)
set(LLVM_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.tar.xz)
set(LLVM_HASH b4268e733dfe352960140dc07ef2efcb)
set(LLVM_HASH_TYPE MD5)
set(LLVM_FILE llvm-project-${LLVM_VERSION}.tar.xz)
set(OPENMP_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/openmp-${LLVM_VERSION}.src.tar.xz)
set(OPENMP_HASH ac48ce3e4582ccb82f81ab59eb3fc9dc)
set(OPENMP_HASH_TYPE MD5)
set(OPENMP_FILE openmp-${LLVM_VERSION}.src.tar.xz)
set(OPENMP_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/openmp-${LLVM_VERSION}.src.tar.xz)
set(OPENMP_HASH 6eade16057edbdecb3c4eef9daa2bfcf)
set(OPENMP_HASH_TYPE MD5)
set(OPENMP_FILE openmp-${LLVM_VERSION}.src.tar.xz)
endif()
set(OPENIMAGEIO_VERSION 2.1.15.0)
set(OPENIMAGEIO_VERSION 2.2.15.1)
set(OPENIMAGEIO_URI https://github.com/OpenImageIO/oiio/archive/Release-${OPENIMAGEIO_VERSION}.tar.gz)
set(OPENIMAGEIO_HASH f03aa5e3ac4795af04771ee4146e9832)
set(OPENIMAGEIO_HASH 3db5c5f0b3dc91597c75e5df09eb9072)
set(OPENIMAGEIO_HASH_TYPE MD5)
set(OPENIMAGEIO_FILE OpenImageIO-${OPENIMAGEIO_VERSION}.tar.gz)
@ -190,9 +175,9 @@ set(TIFF_HASH 2165e7aba557463acc0664e71a3ed424)
set(TIFF_HASH_TYPE MD5)
set(TIFF_FILE tiff-${TIFF_VERSION}.tar.gz)
set(OSL_VERSION 1.11.10.0)
set(OSL_VERSION 1.11.14.1)
set(OSL_URI https://github.com/imageworks/OpenShadingLanguage/archive/Release-${OSL_VERSION}.tar.gz)
set(OSL_HASH dfdc23597aeef083832cbada62211756)
set(OSL_HASH 1abd7ce40481771a9fa937f19595d2f2)
set(OSL_HASH_TYPE MD5)
set(OSL_FILE OpenShadingLanguage-${OSL_VERSION}.tar.gz)
@ -370,12 +355,18 @@ set(PUGIXML_HASH 0c208b0664c7fb822bf1b49ad035e8fd)
set(PUGIXML_HASH_TYPE MD5)
set(PUGIXML_FILE pugixml-${PUGIXML_VERSION}.tar.gz)
set(FLEXBISON_VERSION 2.5.5)
set(FLEXBISON_VERSION 2.5.24)
set(FLEXBISON_URI http://prdownloads.sourceforge.net/winflexbison/win_flex_bison-${FLEXBISON_VERSION}.zip)
set(FLEXBISON_HASH d87a3938194520d904013abef3df10ce)
set(FLEXBISON_HASH 6b549d43e34ece0e8ed05af92daa31c4)
set(FLEXBISON_HASH_TYPE MD5)
set(FLEXBISON_FILE win_flex_bison-${FLEXBISON_VERSION}.zip)
set(FLEX_VERSION 2.6.4)
set(FLEX_URI https://github.com/westes/flex/releases/download/v${FLEX_VERSION}/flex-${FLEX_VERSION}.tar.gz)
set(FLEX_HASH 2882e3179748cc9f9c23ec593d6adc8d)
set(FLEX_HASH_TYPE MD5)
set(FLEX_FILE flex-${FLEX_VERSION}.tar.gz)
# Libraries to keep Python modules static on Linux.
# NOTE: bzip.org domain does no longer belong to BZip 2 project, so we download
@ -432,9 +423,9 @@ set(USD_HASH 1dd1e2092d085ed393c1f7c450a4155a)
set(USD_HASH_TYPE MD5)
set(USD_FILE usd-v${USD_VERSION}.tar.gz)
set(OIDN_VERSION 1.4.0)
set(OIDN_VERSION 1.4.1)
set(OIDN_URI https://github.com/OpenImageDenoise/oidn/releases/download/v${OIDN_VERSION}/oidn-${OIDN_VERSION}.src.tar.gz)
set(OIDN_HASH 421824019becc5b664a22a2b98332bc5)
set(OIDN_HASH df4007b0ab93b1c41cdf223b075d01c0)
set(OIDN_HASH_TYPE MD5)
set(OIDN_FILE oidn-${OIDN_VERSION}.src.tar.gz)
@ -444,10 +435,10 @@ set(LIBGLU_HASH 151aef599b8259efe9acd599c96ea2a3)
set(LIBGLU_HASH_TYPE MD5)
set(LIBGLU_FILE glu-${LIBGLU_VERSION}.tar.xz)
set(MESA_VERSION 20.3.4)
set(MESA_VERSION 21.1.5)
set(MESA_URI ftp://ftp.freedesktop.org/pub/mesa/mesa-${MESA_VERSION}.tar.xz)
set(MESA_HASH 556338446aef8ae947a789b3e0b5e056)
set(MESA_HASH_TYPE MD5)
set(MESA_HASH 022c7293074aeeced2278c872db4fa693147c70f8595b076cf3f1ef81520766d)
set(MESA_HASH_TYPE SHA256)
set(MESA_FILE mesa-${MESA_VERSION}.tar.xz)
set(NASM_VERSION 2.15.02)
@ -468,19 +459,11 @@ set(WL_PROTOCOLS_URI https://gitlab.freedesktop.org/wayland/wayland-protocols/-/
set(WL_PROTOCOLS_HASH af5ca07e13517cdbab33504492cef54a)
set(WL_PROTOCOLS_HASH_TYPE MD5)
if(BLENDER_PLATFORM_ARM)
# Unreleased version with macOS arm support.
set(ISPC_URI https://github.com/ispc/ispc/archive/f5949c055eb9eeb93696978a3da4bfb3a6a30b35.zip)
set(ISPC_HASH d382fea18d01dbd0cd05d9e1ede36d7d)
set(ISPC_HASH_TYPE MD5)
set(ISPC_FILE f5949c055eb9eeb93696978a3da4bfb3a6a30b35.zip)
else()
set(ISPC_VERSION v1.14.1)
set(ISPC_URI https://github.com/ispc/ispc/archive/${ISPC_VERSION}.tar.gz)
set(ISPC_HASH 968fbc8dfd16a60ba4e32d2e0e03ea7a)
set(ISPC_HASH_TYPE MD5)
set(ISPC_FILE ispc-${ISPC_VERSION}.tar.gz)
endif()
set(ISPC_VERSION v1.16.0)
set(ISPC_URI https://github.com/ispc/ispc/archive/${ISPC_VERSION}.tar.gz)
set(ISPC_HASH 2e3abedbc0ea9aaec17d6562c632454d)
set(ISPC_HASH_TYPE MD5)
set(ISPC_FILE ispc-${ISPC_VERSION}.tar.gz)
set(GMP_VERSION 6.2.0)
set(GMP_URI https://gmplib.org/download/gmp/gmp-${GMP_VERSION}.tar.xz)

View File

@ -1,10 +0,0 @@
--- external_openimagedenoise/cmake/oidn_ispc.cmake 2021-02-15 17:29:34.000000000 +0100
+++ external_openimagedenoise/cmake/oidn_ispc.cmake2 2021-02-15 17:29:28.000000000 +0100
@@ -98,7 +98,7 @@
elseif(OIDN_ARCH STREQUAL "ARM64")
set(ISPC_ARCHITECTURE "aarch64")
if(APPLE)
- set(ISPC_TARGET_OS "--target-os=ios")
+ set(ISPC_TARGET_OS "--target-os=macos")
endif()
endif()

View File

@ -34,24 +34,3 @@ diff -Naur orig/src/include/OpenImageIO/platform.h external_openimageio/src/incl
# include <windows.h>
#endif
diff -Naur orig/src/libutil/ustring.cpp external_openimageio/src/libutil/ustring.cpp
--- orig/src/libutil/ustring.cpp 2020-05-11 05:43:52.000000000 +0200
+++ external_openimageio/src/libutil/ustring.cpp 2020-11-26 12:06:08.000000000 +0100
@@ -337,6 +337,8 @@
// the std::string to make it point to our chars! In such a case, the
// destructor will be careful not to allow a deallocation.
+ // Disable internal std::string for Apple silicon based Macs
+#if !(defined(__APPLE__) && defined(__arm64__))
#if defined(__GNUC__) && !defined(_LIBCPP_VERSION) \
&& defined(_GLIBCXX_USE_CXX11_ABI) && _GLIBCXX_USE_CXX11_ABI
// NEW gcc ABI
@@ -382,7 +384,7 @@
return;
}
#endif
-
+#endif
// Remaining cases - just assign the internal string. This may result
// in double allocation for the chars. If you care about that, do
// something special for your platform, much like we did for gcc and

View File

@ -1,23 +0,0 @@
diff --git a/runtime/src/z_Linux_asm.S b/runtime/src/z_Linux_asm.S
index 0d8885e..42aa5ad 100644
--- a/runtime/src/z_Linux_asm.S
+++ b/runtime/src/z_Linux_asm.S
@@ -1540,10 +1540,12 @@ __kmp_unnamed_critical_addr:
.comm .gomp_critical_user_,32,8
.data
.align 8
- .global __kmp_unnamed_critical_addr
-__kmp_unnamed_critical_addr:
+ .global ___kmp_unnamed_critical_addr
+___kmp_unnamed_critical_addr:
.8byte .gomp_critical_user_
- .size __kmp_unnamed_critical_addr,8
+# if !(KMP_OS_DARWIN)
+ .size ___kmp_unnamed_critical_addr,8
+# endif
#endif /* KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 */
#if KMP_OS_LINUX

View File

@ -63,19 +63,50 @@ diff -Naur org/CMakeLists.txt external_osl/CMakeLists.txt
set (OSL_NO_DEFAULT_TEXTURESYSTEM OFF CACHE BOOL "Do not use create a raw OIIO::TextureSystem")
if (OSL_NO_DEFAULT_TEXTURESYSTEM)
diff --git a/src/liboslexec/llvm_util.cpp b/src/liboslexec/llvm_util.cpp
index 445f6400..3d468de2 100644
--- a/src/liboslexec/llvm_util.cpp
+++ b/src/liboslexec/llvm_util.cpp
@@ -3430,8 +3430,9 @@ LLVM_Util::call_function (llvm::Value *func, cspan<llvm::Value *> args)
#endif
//llvm_gen_debug_printf (std::string("start ") + std::string(name));
#if OSL_LLVM_VERSION >= 110
- OSL_DASSERT(llvm::isa<llvm::Function>(func));
- llvm::Value *r = builder().CreateCall(llvm::cast<llvm::Function>(func), llvm::ArrayRef<llvm::Value *>(args.data(), args.size()));
+ llvm::Value* r = builder().CreateCall(
+ llvm::cast<llvm::FunctionType>(func->getType()->getPointerElementType()), func,
+ llvm::ArrayRef<llvm::Value*>(args.data(), args.size()));
#else
llvm::Value *r = builder().CreateCall (func, llvm::ArrayRef<llvm::Value *>(args.data(), args.size()));
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 990f50d69..46ef7351d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -252,11 +252,9 @@ install (EXPORT OSL_EXPORTED_TARGETS
FILE ${OSL_TARGETS_EXPORT_NAME}
NAMESPACE ${PROJECT_NAME}::)
-
-
-
-osl_add_all_tests()
-
+if (${PROJECT_NAME}_BUILD_TESTS AND NOT ${PROJECT_NAME}_IS_SUBPROJECT)
+ osl_add_all_tests()
+endif ()
if (NOT ${PROJECT_NAME}_IS_SUBPROJECT)
include (packaging)
diff -Naur external_osl_orig/src/cmake/externalpackages.cmake external_osl/src/cmake/externalpackages.cmake
--- external_osl_orig/src/cmake/externalpackages.cmake 2021-06-01 13:44:18 -0600
+++ external_osl/src/cmake/externalpackages.cmake 2021-06-28 07:44:32 -0600
@@ -80,6 +80,7 @@
checked_find_package (ZLIB REQUIRED) # Needed by several packages
+checked_find_package (PNG REQUIRED) # Needed since OIIO needs it
# IlmBase & OpenEXR
checked_find_package (OpenEXR REQUIRED
diff -Naur external_osl_orig/src/liboslcomp/oslcomp.cpp external_osl/src/liboslcomp/oslcomp.cpp
--- external_osl_orig/src/liboslcomp/oslcomp.cpp 2021-06-01 13:44:18 -0600
+++ external_osl/src/liboslcomp/oslcomp.cpp 2021-06-28 09:11:06 -0600
@@ -21,6 +21,13 @@
#if !defined(__STDC_CONSTANT_MACROS)
# define __STDC_CONSTANT_MACROS 1
#endif
+
+// clang uses CALLBACK in its templates which causes issues if it is already defined
+#ifdef _WIN32 && defined(CALLBACK)
+# undef CALLBACK
+#endif
+
+//
#include <clang/Basic/TargetInfo.h>
#include <clang/Frontend/CompilerInstance.h>
#include <clang/Frontend/TextDiagnosticPrinter.h>