Buildbot: refactor all build slave code
* Move common code into buildbot_utils.py * Remove legacy code from removed builders * Split code into smaller functions Differential Revision: https://developer.blender.org/D5642
This commit is contained in:
parent
a565096109
commit
283d96de11
Notes:
blender-bot
2023-02-14 00:57:36 +01:00
Referenced by issue #69423, Crash interacting with object with Vertex Weight Proximity modifier using geometry edges
|
|
@ -0,0 +1,92 @@
|
|||
# ##### BEGIN GPL LICENSE BLOCK #####
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#
|
||||
# ##### END GPL LICENSE BLOCK #####
|
||||
|
||||
# <pep8 compliant>
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
class Builder:
|
||||
def __init__(self, name, branch):
|
||||
self.name = name
|
||||
self.branch = branch
|
||||
|
||||
# Buildbot runs from build/ directory
|
||||
self.blender_dir = os.path.abspath(os.path.join('..', 'blender.git'))
|
||||
self.build_dir = os.path.abspath(os.path.join('..', 'build', name))
|
||||
self.install_dir = os.path.abspath(os.path.join('..', 'install', name))
|
||||
self.upload_dir = os.path.abspath(os.path.join('..', 'install'))
|
||||
|
||||
# Detect platform
|
||||
if name.startswith('mac'):
|
||||
self.platform = 'mac'
|
||||
self.command_prefix = []
|
||||
elif name.startswith('linux'):
|
||||
self.platform = 'linux'
|
||||
self.command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
|
||||
elif name.startswith('win'):
|
||||
self.platform = 'win'
|
||||
self.command_prefix = []
|
||||
else:
|
||||
raise ValueError('Unkonw platform for builder ' + self.platform)
|
||||
|
||||
# Always 64 bit now
|
||||
self.bits = 64
|
||||
|
||||
def create_builder_from_arguments():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('builder_name')
|
||||
parser.add_argument('branch', default='master', nargs='?')
|
||||
args = parser.parse_args()
|
||||
return Builder(args.builder_name, args.branch)
|
||||
|
||||
|
||||
class VersionInfo:
|
||||
def __init__(self, builder):
|
||||
# Get version information
|
||||
buildinfo_h = os.path.join(builder.build_dir, "source", "creator", "buildinfo.h")
|
||||
blender_h = os.path.join(builder.blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
|
||||
|
||||
version_number = int(self._parse_header_file(blender_h, 'BLENDER_VERSION'))
|
||||
self.version = "%d.%d" % (version_number // 100, version_number % 100)
|
||||
self.hash = self._parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
|
||||
|
||||
def _parse_header_file(self, filename, define):
|
||||
import re
|
||||
regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
|
||||
with open(filename, "r") as file:
|
||||
for l in file:
|
||||
match = regex.match(l)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def call(cmd, env=None, exit_on_error=True):
|
||||
print(' '.join(cmd))
|
||||
|
||||
# Flush to ensure correct order output on Windows.
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
|
||||
retcode = subprocess.call(cmd, env=env)
|
||||
if exit_on_error and retcode != 0:
|
||||
sys.exit(retcode)
|
||||
return retcode
|
||||
|
|
@ -18,149 +18,80 @@
|
|||
|
||||
# <pep8 compliant>
|
||||
|
||||
import buildbot_utils
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import shutil
|
||||
|
||||
# get builder name
|
||||
if len(sys.argv) < 2:
|
||||
sys.stderr.write("Not enough arguments, expecting builder name\n")
|
||||
sys.exit(1)
|
||||
def get_cmake_options(builder):
|
||||
config_file = "build_files/cmake/config/blender_release.cmake"
|
||||
options = ['-DCMAKE_BUILD_TYPE:STRING=Release', '-DWITH_GTESTS=ON']
|
||||
|
||||
builder = sys.argv[1]
|
||||
if builder.platform == 'mac':
|
||||
options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
|
||||
options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
|
||||
elif builder.platform == 'win':
|
||||
options.extend(['-G', 'Visual Studio 15 2017 Win64'])
|
||||
elif builder.platform == 'linux':
|
||||
config_file = "build_files/buildbot/config/blender_linux.cmake"
|
||||
|
||||
# we run from build/ directory
|
||||
blender_dir = os.path.join('..', 'blender.git')
|
||||
options.append("-C" + os.path.join(builder.blender_dir, config_file))
|
||||
options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir))
|
||||
|
||||
return options
|
||||
|
||||
def parse_header_file(filename, define):
|
||||
import re
|
||||
regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
|
||||
with open(filename, "r") as file:
|
||||
for l in file:
|
||||
match = regex.match(l)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
def update_git(builder):
|
||||
# Do extra git fetch because not all platform/git/buildbot combinations
|
||||
# update the origin remote, causing buildinfo to detect local changes.
|
||||
os.chdir(builder.blender_dir)
|
||||
|
||||
if 'cmake' in builder:
|
||||
# cmake
|
||||
|
||||
# Some fine-tuning configuration
|
||||
blender_dir = os.path.abspath(blender_dir)
|
||||
build_dir = os.path.abspath(os.path.join('..', 'build', builder))
|
||||
install_dir = os.path.abspath(os.path.join('..', 'install', builder))
|
||||
targets = ['blender']
|
||||
command_prefix = []
|
||||
|
||||
bits = 64
|
||||
|
||||
# Config file to be used (relative to blender's sources root)
|
||||
cmake_config_file = "build_files/cmake/config/blender_release.cmake"
|
||||
|
||||
# Set build options.
|
||||
cmake_options = []
|
||||
cmake_extra_options = ['-DCMAKE_BUILD_TYPE:STRING=Release',
|
||||
'-DWITH_GTESTS=ON']
|
||||
|
||||
if builder.startswith('mac'):
|
||||
# Set up OSX architecture
|
||||
if builder.endswith('x86_64_10_9_cmake'):
|
||||
cmake_extra_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
|
||||
cmake_extra_options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9')
|
||||
|
||||
elif builder.startswith('win'):
|
||||
if builder.startswith('win64'):
|
||||
cmake_options.extend(['-G', 'Visual Studio 15 2017 Win64'])
|
||||
elif builder.startswith('win32'):
|
||||
bits = 32
|
||||
cmake_options.extend(['-G', 'Visual Studio 15 2017'])
|
||||
|
||||
elif builder.startswith('linux'):
|
||||
cmake_config_file = "build_files/buildbot/config/blender_linux.cmake"
|
||||
tokens = builder.split("_")
|
||||
glibc = tokens[1]
|
||||
if glibc == 'glibc224':
|
||||
deb_name = "stretch"
|
||||
if builder.endswith('x86_64_cmake'):
|
||||
chroot_name = 'buildbot_' + deb_name + '_x86_64'
|
||||
elif builder.endswith('i686_cmake'):
|
||||
bits = 32
|
||||
chroot_name = 'buildbot_' + deb_name + '_i686'
|
||||
command_prefix = ['schroot', '-c', chroot_name, '--']
|
||||
elif glibc == 'glibc217':
|
||||
command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
|
||||
|
||||
cmake_options.append("-C" + os.path.join(blender_dir, cmake_config_file))
|
||||
|
||||
# Prepare CMake options needed to configure cuda binaries compilation, 64bit only.
|
||||
if bits == 64:
|
||||
cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=ON")
|
||||
cmake_options.append("-DCUDA_64_BIT_DEVICE_CODE=ON")
|
||||
else:
|
||||
cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=OFF")
|
||||
|
||||
cmake_options.append("-DCMAKE_INSTALL_PREFIX=%s" % (install_dir))
|
||||
|
||||
cmake_options += cmake_extra_options
|
||||
print("Fetching remotes")
|
||||
command = ['git', 'fetch', '--all']
|
||||
buildbot_utils.call(builder.command_prefix + command)
|
||||
|
||||
def clean_directories(builder):
|
||||
# Make sure no garbage remained from the previous run
|
||||
if os.path.isdir(install_dir):
|
||||
shutil.rmtree(install_dir)
|
||||
if os.path.isdir(builder.install_dir):
|
||||
shutil.rmtree(builder.install_dir)
|
||||
|
||||
for target in targets:
|
||||
print("Building target %s" % (target))
|
||||
# Construct build directory name based on the target
|
||||
target_build_dir = build_dir
|
||||
target_command_prefix = command_prefix[:]
|
||||
if target != 'blender':
|
||||
target_build_dir += '_' + target
|
||||
target_name = 'install'
|
||||
# Tweaking CMake options to respect the target
|
||||
target_cmake_options = cmake_options[:]
|
||||
# Do extra git fetch because not all platform/git/buildbot combinations
|
||||
# update the origin remote, causing buildinfo to detect local changes.
|
||||
os.chdir(blender_dir)
|
||||
print("Fetching remotes")
|
||||
command = ['git', 'fetch', '--all']
|
||||
print(command)
|
||||
retcode = subprocess.call(target_command_prefix + command)
|
||||
if retcode != 0:
|
||||
sys.exit(retcode)
|
||||
# Make sure build directory exists and enter it
|
||||
if not os.path.isdir(target_build_dir):
|
||||
os.mkdir(target_build_dir)
|
||||
os.chdir(target_build_dir)
|
||||
# Configure the build
|
||||
print("CMake options:")
|
||||
print(target_cmake_options)
|
||||
if os.path.exists('CMakeCache.txt'):
|
||||
print("Removing CMake cache")
|
||||
os.remove('CMakeCache.txt')
|
||||
# Remove buildinfo files to force buildbot to re-generate them.
|
||||
for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ):
|
||||
full_path = os.path.join('source', 'creator', buildinfo)
|
||||
if os.path.exists(full_path):
|
||||
print("Removing {}" . format(buildinfo))
|
||||
os.remove(full_path)
|
||||
retcode = subprocess.call(target_command_prefix + ['cmake', blender_dir] + target_cmake_options)
|
||||
if retcode != 0:
|
||||
print('Configuration FAILED!')
|
||||
sys.exit(retcode)
|
||||
# Make sure build directory exists and enter it
|
||||
os.makedirs(builder.build_dir, exist_ok=True)
|
||||
|
||||
if 'win32' in builder or 'win64' in builder:
|
||||
command = ['cmake', '--build', '.', '--target', target_name, '--config', 'Release']
|
||||
else:
|
||||
command = ['make', '-s', '-j2', target_name]
|
||||
# Remove buildinfo files to force buildbot to re-generate them.
|
||||
for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ):
|
||||
full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo)
|
||||
if os.path.exists(full_path):
|
||||
print("Removing {}" . format(buildinfo))
|
||||
os.remove(full_path)
|
||||
|
||||
print("Executing command:")
|
||||
print(command)
|
||||
retcode = subprocess.call(target_command_prefix + command)
|
||||
def cmake_configure(builder):
|
||||
# CMake configuration
|
||||
os.chdir(builder.build_dir)
|
||||
|
||||
if retcode != 0:
|
||||
sys.exit(retcode)
|
||||
cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt')
|
||||
if os.path.exists(cmake_cache):
|
||||
print("Removing CMake cache")
|
||||
os.remove(cmake_cache)
|
||||
|
||||
else:
|
||||
print("Unknown building system")
|
||||
sys.exit(1)
|
||||
print("CMake configure:")
|
||||
cmake_options = get_cmake_options(builder)
|
||||
command = ['cmake', builder.blender_dir] + cmake_options
|
||||
buildbot_utils.call(builder.command_prefix + command)
|
||||
|
||||
def cmake_build(builder):
|
||||
# CMake build
|
||||
os.chdir(builder.build_dir)
|
||||
|
||||
if builder.platform == 'win':
|
||||
command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release']
|
||||
else:
|
||||
command = ['make', '-s', '-j2', 'install']
|
||||
|
||||
print("CMake build:")
|
||||
buildbot_utils.call(builder.command_prefix + command)
|
||||
|
||||
if __name__ == "__main__":
|
||||
builder = buildbot_utils.create_builder_from_arguments()
|
||||
update_git(builder)
|
||||
clean_directories(builder)
|
||||
cmake_configure(builder)
|
||||
cmake_build(builder)
|
||||
|
|
|
|||
|
|
@ -22,47 +22,35 @@
|
|||
# system and zipping it into buildbot_upload.zip. This is then uploaded
|
||||
# to the master in the next buildbot step.
|
||||
|
||||
import buildbot_utils
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
# get builder name
|
||||
if len(sys.argv) < 2:
|
||||
sys.stderr.write("Not enough arguments, expecting builder name\n")
|
||||
sys.exit(1)
|
||||
def get_package_name(builder, platform=None):
|
||||
info = buildbot_utils.VersionInfo(builder)
|
||||
|
||||
builder = sys.argv[1]
|
||||
# Never write branch if it is master.
|
||||
branch = sys.argv[2] if (len(sys.argv) >= 3 and sys.argv[2] != 'master') else ''
|
||||
package_name = 'blender-' + info.version + '-' + info.hash
|
||||
if platform:
|
||||
package_name += '-' + platform
|
||||
if builder.branch != 'master':
|
||||
package_name = builder.branch + "-" + package_name
|
||||
|
||||
blender_dir = os.path.join('..', 'blender.git')
|
||||
build_dir = os.path.join('..', 'build', builder)
|
||||
install_dir = os.path.join('..', 'install', builder)
|
||||
buildbot_upload_zip = os.path.abspath(os.path.join(os.path.dirname(install_dir), "buildbot_upload.zip"))
|
||||
return package_name
|
||||
|
||||
upload_filename = None # Name of the archive to be uploaded
|
||||
# (this is the name of archive which will appear on the
|
||||
# download page)
|
||||
upload_filepath = None # Filepath to be uploaded to the server
|
||||
# (this folder will be packed)
|
||||
def create_buildbot_upload_zip(builder, package_filepath, package_filename):
|
||||
import zipfile
|
||||
|
||||
buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip")
|
||||
if os.path.exists(buildbot_upload_zip):
|
||||
os.remove(buildbot_upload_zip)
|
||||
|
||||
def parse_header_file(filename, define):
|
||||
import re
|
||||
regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
|
||||
with open(filename, "r") as file:
|
||||
for l in file:
|
||||
match = regex.match(l)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
# Make sure install directory always exists
|
||||
if not os.path.exists(install_dir):
|
||||
os.makedirs(install_dir)
|
||||
|
||||
try:
|
||||
z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
|
||||
z.write(package_filepath, arcname=package_filename)
|
||||
z.close()
|
||||
except Exception as ex:
|
||||
sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
def create_tar_bz2(src, dest, package_name):
|
||||
# One extra to remove leading os.sep when cleaning root for package_root
|
||||
|
|
@ -80,163 +68,97 @@ def create_tar_bz2(src, dest, package_name):
|
|||
package.add(entry[0], entry[1], recursive=False)
|
||||
package.close()
|
||||
|
||||
def cleanup_files(dirpath, extension):
|
||||
for f in os.listdir(dirpath):
|
||||
filepath = os.path.join(dirpath, f)
|
||||
if os.path.isfile(filepath) and f.endswith(extension):
|
||||
os.remove(filepath)
|
||||
|
||||
if builder.find('cmake') != -1:
|
||||
# CMake
|
||||
if 'win' in builder or 'mac' in builder:
|
||||
os.chdir(build_dir)
|
||||
|
||||
files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')]
|
||||
for f in files:
|
||||
os.remove(f)
|
||||
retcode = subprocess.call(['cpack', '-G', 'ZIP'])
|
||||
result_file = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')][0]
|
||||
|
||||
# TODO(sergey): Such magic usually happens in SCon's packaging but we don't have it
|
||||
# in the CMake yet. For until then we do some magic here.
|
||||
tokens = result_file.split('-')
|
||||
blender_version = tokens[1].split('.')
|
||||
blender_full_version = '.'.join(blender_version[0:2])
|
||||
git_hash = tokens[2].split('.')[1]
|
||||
platform = builder.split('_')[0]
|
||||
if platform == 'mac':
|
||||
# Special exception for OSX
|
||||
platform = 'OSX-10.9-'
|
||||
if builder.endswith('x86_64_10_9_cmake'):
|
||||
platform += 'x86_64'
|
||||
if builder.endswith('vc2015'):
|
||||
platform += "-vc14"
|
||||
builderified_name = 'blender-{}-{}-{}'.format(blender_full_version, git_hash, platform)
|
||||
# NOTE: Blender 2.7 is already respected by blender_full_version.
|
||||
if branch != '' and branch != 'blender2.7':
|
||||
builderified_name = branch + "-" + builderified_name
|
||||
|
||||
os.rename(result_file, "{}.zip".format(builderified_name))
|
||||
# create zip file
|
||||
try:
|
||||
if os.path.exists(buildbot_upload_zip):
|
||||
os.remove(buildbot_upload_zip)
|
||||
z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
|
||||
z.write("{}.zip".format(builderified_name))
|
||||
z.close()
|
||||
sys.exit(retcode)
|
||||
except Exception as ex:
|
||||
sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
elif builder.startswith('linux_'):
|
||||
blender = os.path.join(install_dir, 'blender')
|
||||
|
||||
buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h")
|
||||
blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
|
||||
|
||||
# Get version information
|
||||
blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION'))
|
||||
blender_version = "%d.%d" % (blender_version // 100, blender_version % 100)
|
||||
blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
|
||||
blender_glibc = builder.split('_')[1]
|
||||
command_prefix = []
|
||||
bits = 64
|
||||
blender_arch = 'x86_64'
|
||||
|
||||
if blender_glibc == 'glibc224':
|
||||
if builder.endswith('x86_64_cmake'):
|
||||
chroot_name = 'buildbot_stretch_x86_64'
|
||||
elif builder.endswith('i686_cmake'):
|
||||
chroot_name = 'buildbot_stretch_i686'
|
||||
bits = 32
|
||||
blender_arch = 'i686'
|
||||
command_prefix = ['schroot', '-c', chroot_name, '--']
|
||||
elif blender_glibc == 'glibc217':
|
||||
command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
|
||||
|
||||
# Strip all unused symbols from the binaries
|
||||
print("Stripping binaries...")
|
||||
subprocess.call(command_prefix + ['strip', '--strip-all', blender])
|
||||
|
||||
print("Stripping python...")
|
||||
py_target = os.path.join(install_dir, blender_version)
|
||||
subprocess.call(command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';'])
|
||||
|
||||
# Copy all specific files which are too specific to be copied by
|
||||
# the CMake rules themselves
|
||||
print("Copying extra scripts and libs...")
|
||||
|
||||
extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
|
||||
mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2')
|
||||
software_gl = os.path.join(blender_dir, 'release', 'bin', 'blender-softwaregl')
|
||||
icons = os.path.join(blender_dir, 'release', 'freedesktop', 'icons')
|
||||
|
||||
os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
|
||||
os.system('cp %s %s' % (software_gl, install_dir))
|
||||
os.system('cp -r %s %s' % (icons, install_dir))
|
||||
os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl')))
|
||||
|
||||
# Construct archive name
|
||||
package_name = 'blender-%s-%s-linux-%s-%s' % (blender_version,
|
||||
blender_hash,
|
||||
blender_glibc,
|
||||
blender_arch)
|
||||
# NOTE: Blender 2.7 is already respected by blender_full_version.
|
||||
if branch != '' and branch != 'blender2.7':
|
||||
package_name = branch + "-" + package_name
|
||||
|
||||
upload_filename = package_name + ".tar.bz2"
|
||||
|
||||
print("Creating .tar.bz2 archive")
|
||||
upload_filepath = install_dir + '.tar.bz2'
|
||||
create_tar_bz2(install_dir, upload_filepath, package_name)
|
||||
else:
|
||||
print("Unknown building system")
|
||||
sys.exit(1)
|
||||
def find_file(dirpath, extension):
|
||||
for f in os.listdir(dirpath):
|
||||
filepath = os.path.join(dirpath, f)
|
||||
if os.path.isfile(filepath) and f.endswith(extension):
|
||||
return f
|
||||
return None
|
||||
|
||||
|
||||
if upload_filepath is None:
|
||||
# clean release directory if it already exists
|
||||
release_dir = 'release'
|
||||
def pack_mac(builder):
|
||||
os.chdir(builder.build_dir)
|
||||
cleanup_files(builder.build_dir, '.zip')
|
||||
|
||||
if os.path.exists(release_dir):
|
||||
for f in os.listdir(release_dir):
|
||||
if os.path.isfile(os.path.join(release_dir, f)):
|
||||
os.remove(os.path.join(release_dir, f))
|
||||
package_name = get_package_name(builder, 'OSX-10.9-x86_64')
|
||||
package_filename = package_name + '.zip'
|
||||
|
||||
# create release package
|
||||
try:
|
||||
subprocess.call(['make', 'package_archive'])
|
||||
except Exception as ex:
|
||||
sys.stderr.write('Make package release failed' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
buildbot_utils.call(['cpack', '-G', 'ZIP'])
|
||||
package_filepath = find_file(builder.build_dir, '.zip')
|
||||
|
||||
# find release directory, must exist this time
|
||||
if not os.path.exists(release_dir):
|
||||
sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
|
||||
sys.exit(1)
|
||||
create_buildbot_upload_zip(builder, package_filepath, package_filename)
|
||||
|
||||
# find release package
|
||||
file = None
|
||||
filepath = None
|
||||
|
||||
for f in os.listdir(release_dir):
|
||||
rf = os.path.join(release_dir, f)
|
||||
if os.path.isfile(rf) and f.startswith('blender'):
|
||||
file = f
|
||||
filepath = rf
|
||||
def pack_win(builder):
|
||||
os.chdir(builder.build_dir)
|
||||
cleanup_files(builder.build_dir, '.zip')
|
||||
|
||||
if not file:
|
||||
sys.stderr.write("Failed to find release package.\n")
|
||||
sys.exit(1)
|
||||
package_name = get_package_name(builder, 'win' + str(builder.bits))
|
||||
package_filename = package_name + '.zip'
|
||||
|
||||
upload_filename = file
|
||||
upload_filepath = filepath
|
||||
buildbot_utils.call(['cpack', '-G', 'ZIP'])
|
||||
package_filepath = find_file(builder.build_dir, '.zip')
|
||||
|
||||
# create zip file
|
||||
try:
|
||||
upload_zip = os.path.join(buildbot_upload_zip)
|
||||
if os.path.exists(upload_zip):
|
||||
os.remove(upload_zip)
|
||||
z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
|
||||
z.write(upload_filepath, arcname=upload_filename)
|
||||
z.close()
|
||||
except Exception as ex:
|
||||
sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
|
||||
sys.exit(1)
|
||||
create_buildbot_upload_zip(builder, package_filepath, package_filename)
|
||||
|
||||
|
||||
def pack_linux(builder):
|
||||
blender_executable = os.path.join(builder.install_dir, 'blender')
|
||||
|
||||
info = buildbot_utils.VersionInfo(builder)
|
||||
blender_glibc = builder.name.split('_')[1]
|
||||
blender_arch = 'x86_64'
|
||||
|
||||
# Strip all unused symbols from the binaries
|
||||
print("Stripping binaries...")
|
||||
buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable])
|
||||
|
||||
print("Stripping python...")
|
||||
py_target = os.path.join(builder.install_dir, info.version)
|
||||
buildbot_utils.call(builder.command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';'])
|
||||
|
||||
# Copy all specific files which are too specific to be copied by
|
||||
# the CMake rules themselves
|
||||
print("Copying extra scripts and libs...")
|
||||
|
||||
extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
|
||||
mesalibs = os.path.join(extra, 'mesalibs' + str(builder.bits) + '.tar.bz2')
|
||||
software_gl = os.path.join(builder.blender_dir, 'release', 'bin', 'blender-softwaregl')
|
||||
icons = os.path.join(builder.blender_dir, 'release', 'freedesktop', 'icons')
|
||||
|
||||
os.system('tar -xpf %s -C %s' % (mesalibs, builder.install_dir))
|
||||
os.system('cp %s %s' % (software_gl, builder.install_dir))
|
||||
os.system('cp -r %s %s' % (icons, builder.install_dir))
|
||||
os.system('chmod 755 %s' % (os.path.join(builder.install_dir, 'blender-softwaregl')))
|
||||
|
||||
# Construct package name
|
||||
platform_name = 'linux-' + blender_glibc + '-' + blender_arch
|
||||
package_name = get_package_name(builder, platform_name)
|
||||
package_filename = package_name + ".tar.bz2"
|
||||
|
||||
print("Creating .tar.bz2 archive")
|
||||
package_filepath = builder.install_dir + '.tar.bz2'
|
||||
create_tar_bz2(builder.install_dir, package_filepath, package_name)
|
||||
|
||||
# Create buildbot_upload.zip
|
||||
create_buildbot_upload_zip(builder, package_filepath, package_filename)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
builder = buildbot_utils.create_builder_from_arguments()
|
||||
|
||||
# Make sure install directory always exists
|
||||
os.makedirs(builder.install_dir, exist_ok=True)
|
||||
|
||||
if builder.platform == 'mac':
|
||||
pack_mac(builder)
|
||||
elif builder.platform == 'win':
|
||||
pack_win(builder)
|
||||
elif builder.platform == 'linux':
|
||||
pack_linux(builder)
|
||||
|
|
|
|||
|
|
@ -21,23 +21,17 @@
|
|||
# Runs on buildbot slave, rsync zip directly to buildbot server rather
|
||||
# than using upload which is much slower
|
||||
|
||||
import buildbot_utils
|
||||
import os
|
||||
import sys
|
||||
|
||||
# get builder name
|
||||
if len(sys.argv) < 2:
|
||||
sys.stderr.write("Not enough arguments, expecting builder name\n")
|
||||
sys.exit(1)
|
||||
if __name__ == "__main__":
|
||||
builder = buildbot_utils.create_builder_from_arguments()
|
||||
|
||||
builder = sys.argv[1]
|
||||
# rsync, this assumes ssh keys are setup so no password is needed
|
||||
local_zip = "buildbot_upload.zip"
|
||||
remote_folder = "builder.blender.org:/data/buildbot-master/uploaded/"
|
||||
remote_zip = remote_folder + "buildbot_upload_" + builder.name + ".zip"
|
||||
|
||||
# rsync, this assumes ssh keys are setup so no password is needed
|
||||
local_zip = "buildbot_upload.zip"
|
||||
remote_folder = "builder.blender.org:/data/buildbot-master/uploaded/"
|
||||
remote_zip = remote_folder + "buildbot_upload_" + builder + ".zip"
|
||||
command = "rsync -avz %s %s" % (local_zip, remote_zip)
|
||||
|
||||
print(command)
|
||||
|
||||
ret = os.system(command)
|
||||
sys.exit(ret)
|
||||
command = ["rsync", "-avz", local_zip, remote_zip]
|
||||
buildbot_utils.call(command)
|
||||
|
|
|
|||
|
|
@ -18,59 +18,39 @@
|
|||
|
||||
# <pep8 compliant>
|
||||
|
||||
import subprocess
|
||||
import buildbot_utils
|
||||
import os
|
||||
import sys
|
||||
|
||||
# get builder name
|
||||
if len(sys.argv) < 2:
|
||||
sys.stderr.write("Not enough arguments, expecting builder name\n")
|
||||
sys.exit(1)
|
||||
def get_ctest_environment(builder):
|
||||
info = buildbot_utils.VersionInfo(builder)
|
||||
blender_version_dir = os.path.join(builder.install_dir, info.version)
|
||||
|
||||
builder = sys.argv[1]
|
||||
env = os.environ.copy()
|
||||
env['BLENDER_SYSTEM_SCRIPTS'] = os.path.join(blender_version_dir, 'scripts')
|
||||
env['BLENDER_SYSTEM_DATAFILES'] = os.path.join(blender_version_dir, 'datafiles')
|
||||
return env
|
||||
|
||||
# we run from build/ directory
|
||||
blender_dir = '../blender.git'
|
||||
def get_ctest_arguments(builder):
|
||||
args = ['--output-on-failure']
|
||||
if builder.platform == 'win':
|
||||
args += ['-C', 'Release']
|
||||
return args
|
||||
|
||||
if "cmake" in builder:
|
||||
def test(builder):
|
||||
os.chdir(builder.build_dir)
|
||||
|
||||
command = builder.command_prefix + ['ctest'] + get_ctest_arguments(builder)
|
||||
ctest_env = get_ctest_environment(builder)
|
||||
buildbot_utils.call(command, env=ctest_env, exit_on_error=False)
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Automated tests are still DISABLED!")
|
||||
sys.exit(0)
|
||||
|
||||
build_dir = os.path.abspath(os.path.join('..', 'build', builder))
|
||||
install_dir = os.path.abspath(os.path.join('..', 'install', builder))
|
||||
# NOTE: For quick test only to see if the approach work.
|
||||
# n the future must be replaced with an actual blender version.
|
||||
blender_version = '2.80'
|
||||
blender_version_dir = os.path.join(install_dir, blender_version)
|
||||
command_prefix = []
|
||||
extra_ctest_args = []
|
||||
|
||||
if builder.startswith('win'):
|
||||
extra_ctest_args += ['-C', 'Release']
|
||||
elif builder.startswith('linux'):
|
||||
tokens = builder.split("_")
|
||||
glibc = tokens[1]
|
||||
if glibc == 'glibc224':
|
||||
deb_name = "stretch"
|
||||
if builder.endswith('x86_64_cmake'):
|
||||
chroot_name = 'buildbot_' + deb_name + '_x86_64'
|
||||
elif builder.endswith('i686_cmake'):
|
||||
chroot_name = 'buildbot_' + deb_name + '_i686'
|
||||
command_prefix = ['schroot', '--preserve-environment', '-c', chroot_name, '--']
|
||||
elif glibc == 'glibc217':
|
||||
command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
|
||||
|
||||
ctest_env = os.environ.copy()
|
||||
ctest_env['BLENDER_SYSTEM_SCRIPTS'] = os.path.join(blender_version_dir, 'scripts')
|
||||
ctest_env['BLENDER_SYSTEM_DATAFILES'] = os.path.join(blender_version_dir, 'datafiles')
|
||||
|
||||
os.chdir(build_dir)
|
||||
retcode = subprocess.call(command_prefix + ['ctest', '--output-on-failure'] + extra_ctest_args,
|
||||
env=ctest_env)
|
||||
builder = buildbot_utils.create_builder_from_arguments()
|
||||
test(builder)
|
||||
|
||||
# Always exit with a success, for until we know all the tests are passing
|
||||
# on all builders.
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Unknown building system")
|
||||
sys.exit(1)
|
||||
|
|
|
|||
|
|
@ -18,14 +18,13 @@
|
|||
|
||||
# <pep8 compliant>
|
||||
|
||||
import buildbot_utils
|
||||
import os
|
||||
import runpy
|
||||
|
||||
# We run from build/ directory.
|
||||
blender_dir = os.path.join('..', 'blender.git')
|
||||
blender_dir = os.path.abspath(blender_dir)
|
||||
os.chdir(blender_dir)
|
||||
if __name__ == "__main__":
|
||||
builder = buildbot_utils.create_builder_from_arguments()
|
||||
os.chdir(builder.blender_dir)
|
||||
|
||||
# Run make update which handles all libraries and submodules.
|
||||
make_update = os.path.join(blender_dir, "build_files", "utils", "make_update.py")
|
||||
runpy.run_path(make_update)
|
||||
# Run make update which handles all libraries and submodules.
|
||||
make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py")
|
||||
buildbot_utils.call(['python3', make_update])
|
||||
|
|
|
|||
Loading…
Reference in New Issue