blender
/
blender
129
398
Fork 572
Code Issues 6.2k Pull Requests 747 Projects 19 Wiki Activity

Tests: speed up render tests by running multiple in the same process 

Blender startup time and shader compilation is a big factor when running
hundreds of tests, so now all renders in the same ctest run in the same
process. If a test crashes, the remaining tests in the same category will
be marked as skipped.

Benchmarked on a quad core with ctest -j8.

cycles: 118.1s -> 94.3s
eevee: 66.2s -> 29.2s
workbench: 31.7s -> 8.6s
This commit is contained in:
Brecht Van Lommel 2019-05-10 23:00:35 +02:00
parent 79b9596c66
commit 93901e7f0a
Notes: blender-bot 2024-03-22 15:57:27 +01:00 
Referenced by commit 3b51260387, Revert "Tests: speed up render tests by running multiple in the same process"
5 changed files with 199 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
tests/python/cycles_render_tests.py
View File

@ -9,77 +9,76 @@ import subprocess
import sys
def render_file(filepath, output_filepath):
dirname = os.path.dirname(filepath)
basedir = os.path.dirname(dirname)
subject = os.path.basename(dirname)
frame_filepath = output_filepath + '0001.png'
common_args = [
"-noaudio",
"--factory-startup",
"--enable-autoexec",
filepath,
"-E", "CYCLES",
"-o", output_filepath,
"-F", "PNG"]
def render_files(filepaths, output_filepaths):
command = [BLENDER, "--background"]
# OSL and GPU examples
# custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
# custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
custom_args = os.getenv('CYCLESTEST_ARGS')
custom_args = shlex.split(custom_args) if custom_args else []
common_args += custom_args
if subject == 'opengl':
command = [BLENDER, "--window-geometry", "0", "0", "1", "1"]
command += common_args
command += ['--python', os.path.join(basedir, "util", "render_opengl.py")]
elif subject == 'bake':
command = [BLENDER, "--background"]
command += common_args
command += ['--python', os.path.join(basedir, "util", "render_bake.py")]
elif subject == 'denoise_animation':
command = [BLENDER, "--background"]
command += common_args
command += ['--python', os.path.join(basedir, "util", "render_denoise.py")]
else:
command = [BLENDER, "--background"]
command += common_args
command += ["-f", "1"]
for filepath, output_filepath in zip(filepaths, output_filepaths):
dirname = os.path.dirname(filepath)
basedir = os.path.dirname(dirname)
subject = os.path.basename(dirname)
frame_filepath = output_filepath + '0001.png'
common_args = [
"-noaudio",
"--factory-startup",
"--enable-autoexec",
filepath,
"-E", "CYCLES",
"-o", output_filepath,
"-F", "PNG"]
common_args += custom_args
if subject == 'bake':
command.extend(common_args)
command.extend(['--python', os.path.join(basedir, "util", "render_bake.py")])
elif subject == 'denoise_animation':
command.extend(common_args)
command.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")])
else:
command.extend(common_args)
command.extend(["-f", "1"])
error = None
try:
# Success
output = subprocess.check_output(command)
if os.path.exists(frame_filepath):
shutil.copy(frame_filepath, output_filepath)
os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(output.decode("utf-8"))
return None
except subprocess.CalledProcessError as e:
# Error
if os.path.exists(frame_filepath):
os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(e.output.decode("utf-8"))
if b"Error: engine not found" in e.output:
return "NO_ENGINE"
elif b"blender probably wont start" in e.output:
return "NO_START"
return "CRASH"
error = "CRASH"
except BaseException as e:
# Crash
if os.path.exists(frame_filepath):
os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(e)
return "CRASH"
print(e.decode("utf-8"))
error = "CRASH"
# Detect missing filepaths and consider those errors
errors = []
for output_filepath in output_filepaths:
frame_filepath = output_filepath + '0001.png'
if os.path.exists(frame_filepath):
shutil.copy(frame_filepath, output_filepath)
os.remove(frame_filepath)
errors.append(None)
else:
errors.append(error)
error = 'SKIPPED'
return errors
def create_argparse():
parser = argparse.ArgumentParser()
@ -108,7 +107,7 @@ def main():
report.set_pixelated(True)
report.set_reference_dir("cycles_renders")
report.set_compare_engines('cycles', 'eevee')
ok = report.run(test_dir, render_file)
ok = report.run(test_dir, render_files)
sys.exit(not ok)

66
tests/python/eevee_render_tests.py
View File

@ -49,57 +49,61 @@ if inside_blender:
sys.exit(1)
def render_file(filepath, output_filepath):
dirname = os.path.dirname(filepath)
basedir = os.path.dirname(dirname)
subject = os.path.basename(dirname)
frame_filepath = output_filepath + '0001.png'
def render_files(filepaths, output_filepaths):
command = [
BLENDER,
"--background",
"-noaudio",
"--factory-startup",
"--enable-autoexec",
filepath,
"-E", "BLENDER_EEVEE",
"-P",
os.path.realpath(__file__),
"-o", output_filepath,
"-F", "PNG",
"-f", "1"]
"--enable-autoexec"]
for filepath, output_filepath in zip(filepaths, output_filepaths):
frame_filepath = output_filepath + '0001.png'
if os.path.exists(frame_filepath):
os.remove(frame_filepath)
command.extend([
filepath,
"-E", "BLENDER_EEVEE",
"-P",
os.path.realpath(__file__),
"-o", output_filepath,
"-F", "PNG",
"-f", "1"])
error = None
try:
# Success
output = subprocess.check_output(command)
if os.path.exists(frame_filepath):
shutil.copy(frame_filepath, output_filepath)
os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(output.decode("utf-8"))
return None
except subprocess.CalledProcessError as e:
# Error
if os.path.exists(frame_filepath):
os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(e.output.decode("utf-8"))
if b"Error: engine not found" in e.output:
return "NO_ENGINE"
elif b"blender probably wont start" in e.output:
return "NO_START"
return "CRASH"
error = "CRASH"
except BaseException as e:
# Crash
if os.path.exists(frame_filepath):
os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(e)
return "CRASH"
print(e.decode("utf-8"))
error = "CRASH"
# Detect missing filepaths and consider those errors
errors = []
for output_filepath in output_filepaths:
frame_filepath = output_filepath + '0001.png'
if os.path.exists(frame_filepath):
shutil.copy(frame_filepath, output_filepath)
os.remove(frame_filepath)
errors.append(None)
else:
errors.append(error)
error = 'SKIPPED'
return errors
def create_argparse():
@ -129,7 +133,7 @@ def main():
report.set_pixelated(True)
report.set_reference_dir("eevee_renders")
report.set_compare_engines('eevee', 'cycles')
ok = report.run(test_dir, render_file)
ok = report.run(test_dir, render_files)
sys.exit(not ok)

73
tests/python/modules/render_report.py
View File

@ -374,41 +374,48 @@ class Report:
return not failed
def _run_test(self, filepath, render_cb):
testname = test_get_name(filepath)
print_message(testname, 'SUCCESS', 'RUN')
time_start = time.time()
tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname)
def _run_tests(self, filepaths, render_cb):
# Run all tests together for performance, since Blender
# startup time is a significant factor.
tmp_filepaths = []
for filepath in filepaths:
testname = test_get_name(filepath)
print_message(testname, 'SUCCESS', 'RUN')
tmp_filepaths.append(os.path.join(self.output_dir, "tmp_" + testname))
error = render_cb(filepath, tmp_filepath)
status = "FAIL"
if not error:
if not self._diff_output(filepath, tmp_filepath):
error = "VERIFY"
run_errors = render_cb(filepaths, tmp_filepaths)
errors = []
if os.path.exists(tmp_filepath):
os.remove(tmp_filepath)
for error, filepath, tmp_filepath in zip(run_errors, filepaths, tmp_filepaths):
if not error:
if os.path.getsize(tmp_filepath) == 0:
error = "VERIFY"
elif not self._diff_output(filepath, tmp_filepath):
error = "VERIFY"
time_end = time.time()
elapsed_ms = int((time_end - time_start) * 1000)
if not error:
print_message("{} ({} ms)" . format(testname, elapsed_ms),
'SUCCESS', 'OK')
else:
if error == "NO_ENGINE":
print_message("Can't perform tests because the render engine failed to load!")
return error
elif error == "NO_START":
print_message('Can not perform tests because blender fails to start.',
'Make sure INSTALL target was run.')
return error
elif error == 'VERIFY':
print_message("Rendered result is different from reference image")
if os.path.exists(tmp_filepath):
os.remove(tmp_filepath)
errors.append(error)
testname = test_get_name(filepath)
if not error:
print_message(testname, 'SUCCESS', 'OK')
else:
print_message("Unknown error %r" % error)
print_message("{} ({} ms)" . format(testname, elapsed_ms),
'FAILURE', 'FAILED')
return error
if error == "SKIPPED":
print_message("Skipped after previous render caused error")
elif error == "NO_ENGINE":
print_message("Can't perform tests because the render engine failed to load!")
elif error == "NO_START":
print_message('Can not perform tests because blender fails to start.',
'Make sure INSTALL target was run.')
elif error == 'VERIFY':
print_message("Rendered result is different from reference image")
else:
print_message("Unknown error %r" % error)
print_message(testname, 'FAILURE', 'FAILED')
return errors
def _run_all_tests(self, dirname, dirpath, render_cb):
passed_tests = []
@ -419,8 +426,8 @@ class Report:
format(len(all_files)),
'SUCCESS', "==========")
time_start = time.time()
for filepath in all_files:
error = self._run_test(filepath, render_cb)
errors = self._run_tests(all_files, render_cb)
for filepath, error in zip(all_files, errors):
testname = test_get_name(filepath)
if error:
if error == "NO_ENGINE":

77
tests/python/opengl_draw_tests.py
View File

@ -31,41 +31,48 @@ if inside_blender:
sys.exit(0)
def render_file(filepath, output_filepath):
command = (
BLENDER,
"--no-window-focus",
"--window-geometry",
"0", "0", "1024", "768",
"-noaudio",
"--factory-startup",
"--enable-autoexec",
filepath,
"-P",
os.path.realpath(__file__),
"--",
output_filepath)
def render_files(filepaths, output_filepaths):
errors = []
try:
# Success
output = subprocess.check_output(command)
if VERBOSE:
print(output.decode("utf-8"))
return None
except subprocess.CalledProcessError as e:
# Error
if os.path.exists(output_filepath):
os.remove(output_filepath)
if VERBOSE:
print(e.output.decode("utf-8"))
return "CRASH"
except BaseException as e:
# Crash
if os.path.exists(output_filepath):
os.remove(output_filepath)
if VERBOSE:
print(e)
return "CRASH"
for filepath, output_filepath in zip(filepaths, output_filepaths):
command = (
BLENDER,
"--no-window-focus",
"--window-geometry",
"0", "0", "1024", "768",
"-noaudio",
"--factory-startup",
"--enable-autoexec",
filepath,
"-P",
os.path.realpath(__file__),
"--",
output_filepath)
error = None
try:
# Success
output = subprocess.check_output(command)
if VERBOSE:
print(output.decode("utf-8"))
except subprocess.CalledProcessError as e:
# Error
if os.path.exists(output_filepath):
os.remove(output_filepath)
if VERBOSE:
print(e.output.decode("utf-8"))
error = "CRASH"
except BaseException as e:
# Crash
if os.path.exists(output_filepath):
os.remove(output_filepath)
if VERBOSE:
print(e)
error = "CRASH"
errors.append(error)
return errors
def create_argparse():
@ -92,7 +99,7 @@ def main():
from modules import render_report
report = render_report.Report("OpenGL Draw Test Report", output_dir, idiff)
ok = report.run(test_dir, render_file)
ok = report.run(test_dir, render_files)
sys.exit(not ok)

66
tests/python/workbench_render_tests.py
View File

@ -34,57 +34,61 @@ if inside_blender:
sys.exit(1)
def render_file(filepath, output_filepath):
dirname = os.path.dirname(filepath)
basedir = os.path.dirname(dirname)
subject = os.path.basename(dirname)
frame_filepath = output_filepath + '0001.png'
def render_files(filepaths, output_filepaths):
command = [
BLENDER,
"--background",
"-noaudio",
"--factory-startup",
"--enable-autoexec",
filepath,
"-E", "BLENDER_WORKBENCH",
"-P",
os.path.realpath(__file__),
"-o", output_filepath,
"-F", "PNG",
"-f", "1"]
"--enable-autoexec"]
for filepath, output_filepath in zip(filepaths, output_filepaths):
frame_filepath = output_filepath + '0001.png'
if os.path.exists(frame_filepath):
os.remove(frame_filepath)
command.extend([
filepath,
"-E", "BLENDER_WORKBENCH",
"-P",
os.path.realpath(__file__),
"-o", output_filepath,
"-F", "PNG",
"-f", "1"])
error = None
try:
# Success
output = subprocess.check_output(command)
if os.path.exists(frame_filepath):
shutil.copy(frame_filepath, output_filepath)
os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(output.decode("utf-8"))
return None
except subprocess.CalledProcessError as e:
# Error
if os.path.exists(frame_filepath):
os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(e.output.decode("utf-8"))
if b"Error: engine not found" in e.output:
return "NO_ENGINE"
elif b"blender probably wont start" in e.output:
return "NO_START"
return "CRASH"
error = "CRASH"
except BaseException as e:
# Crash
if os.path.exists(frame_filepath):
os.remove(frame_filepath)
if VERBOSE:
print(" ".join(command))
print(e)
return "CRASH"
print(e.decode("utf-8"))
error = "CRASH"
# Detect missing filepaths and consider those errors
errors = []
for output_filepath in output_filepaths:
frame_filepath = output_filepath + '0001.png'
if os.path.exists(frame_filepath):
shutil.copy(frame_filepath, output_filepath)
os.remove(frame_filepath)
errors.append(None)
else:
errors.append(error)
error = 'SKIPPED'
return errors
def create_argparse():
@ -114,7 +118,7 @@ def main():
report.set_pixelated(True)
report.set_reference_dir("workbench_renders")
report.set_compare_engines('workbench', 'eevee')
ok = report.run(test_dir, render_file)
ok = report.run(test_dir, render_files)
sys.exit(not ok)