Tests: continue running benchmarks if some tests fail to build or run

Convenient when testing many revisions where some might be broken.

tests/performance/api/config.py

@@ -31,6 +31,7 @@ class TestEntry:
     device_id: str = 'CPU'
     device_name: str = 'Unknown CPU'
     status: str = 'queued'
+    error_msg: str = ''
     output: Dict = field(default_factory=dict)
     benchmark_type: str = 'comparison'
 
@@ -42,7 +43,8 @@ class TestEntry:
 
     def from_json(self, json_dict):
         for field in self.__dataclass_fields__:
-            setattr(self, field, json_dict[field])
+            if field in json_dict:
+                setattr(self, field, json_dict[field])
 
 
 class TestQueue:

tests/performance/api/environment.py

@@ -88,16 +88,21 @@ class TestEnvironment:
         self.call([self.git_executable, 'reset', '--hard', 'HEAD'], self.blender_dir)
         self.call([self.git_executable, 'checkout', '--detach', git_hash], self.blender_dir)
 
-    def build(self) -> None:
+    def build(self) -> bool:
         # Build Blender revision
         if not self.build_dir.exists():
             sys.stderr.write('\n\nError: no build set up, run `./benchmark init --build` first\n')
             sys.exit(1)
 
         jobs = str(multiprocessing.cpu_count())
-        self.call([self.cmake_executable, '.'] + self.cmake_options, self.build_dir)
-        self.call([self.cmake_executable, '--build', '.', '-j', jobs, '--target', 'install'], self.build_dir)
+        try:
+            self.call([self.cmake_executable, '.'] + self.cmake_options, self.build_dir)
+            self.call([self.cmake_executable, '--build', '.', '-j', jobs, '--target', 'install'], self.build_dir)
+        except:
+            return False
+
         self._init_default_blender_executable()
+        return True
 
     def set_blender_executable(self, executable_path: pathlib.Path) -> None:
         # Run all Blender commands with this executable.
@@ -277,8 +282,10 @@ class TestEnvironment:
 
     def resolve_git_hash(self, revision):
         # Get git hash for a tag or branch.
-        return self.call([self.git_executable, 'rev-parse', revision], self.blender_git_dir)[0].strip()
+        lines = self.call([self.git_executable, 'rev-parse', revision], self.blender_git_dir)
+        return lines[0].strip() if len(lines) else revision
 
     def git_hash_date(self, git_hash):
         # Get commit data for a git hash.
-        return int(self.call([self.git_executable, 'log', '-n1', git_hash, '--format=%at'], self.blender_git_dir)[0].strip())
+        lines = self.call([self.git_executable, 'log', '-n1', git_hash, '--format=%at'], self.blender_git_dir)
+        return int(lines[0].strip()) if len(lines) else 0

tests/performance/api/graph.py

@@ -30,6 +30,7 @@ class TestGraph:
 
         data = []
         for device_name, device_entries in devices.items():
+
             # Gather used categories.
             categories = {}
             for entry in device_entries:
@@ -57,6 +58,8 @@ class TestGraph:
         self.json = json.dumps(data, indent=2)
 
     def chart(self, device_name: str, chart_name: str, entries: List, chart_type: str, output: str) -> Dict:
+        entries = sorted(entries, key=lambda entry: entry.date)
+
         # Gather used tests.
         tests = {}
         for entry in entries:

tests/performance/benchmark

@@ -66,6 +66,8 @@ def print_row(config: api.TestConfig, entries: List, end='\n') -> None:
 
             if status == 'outdated':
                 result += " (outdated)"
+        elif status == 'failed':
+            result = "failed: " + entry.error_msg
         else:
             result = status
 
@@ -105,20 +107,37 @@ def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry
         logname += '_' + device_id
     env.set_log_file(config.logs_dir / (logname + '.log'), clear=True)
 
+    # Clear output
+    entry.output = None
+    entry.error_msg = ''
+
     # Build revision, or just set path to existing executable.
     entry.status = 'building'
     print_row(config, row, end='\r')
+    executable_ok = True
     if len(entry.executable):
         env.set_blender_executable(pathlib.Path(entry.executable))
     else:
         env.checkout(git_hash)
-        env.build()
+        executable_ok = env.build()
+        if not executable_ok:
+            entry.status = 'failed'
+            entry.error_msg = 'Failed to build'
 
     # Run test and update output and status.
     entry.status = 'running'
     print_row(config, row, end='\r')
-    entry.output = test.run(env, device_id)
-    entry.status = 'done' if entry.output else 'failed'
+
+    if executable_ok:
+        try:
+            entry.output = test.run(env, device_id)
+            if not entry.output:
+                raise Exception("Test produced no output")
+            entry.status = 'done'
+        except Exception as e:
+            entry.status = 'failed'
+            entry.error_msg = str(e)
+
     print_row(config, row, end='\r')
 
     # Update device name in case the device changed since the entry was created.