feat(eval): filter out failed evals and report total evaluation failures

Signed-off-by: Raito Bezarius <raito@lix.systems>

buildbot_nix/__init__.py

@@ -371,14 +371,26 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
                 if not system or system in self.supported_systems:  # report eval errors
                     filtered_jobs.append(job)
 
+            eval_show_log: Log = yield self.getLog("stdio")
+            for job in filtered_jobs:
+                # This is an evaluation error, let's warn about it.
+                if job.get('error') is not None:
+                    eval_show_log.addStdout("failed to eval `{}`: {}".format(job.get("attr"), job.get("error")))
+
+            # Filter out failed evaluations
+            succeeded_jobs = [job for job in filtered_jobs if job.get('error') is None]
+            if not succeeded_jobs:
+                eval_show_log.addStdout("all jobs failed to evaluate, failing this step")
+                raise BuildStepFailed()
+
             drv_show_log: Log = yield self.getLog("stdio")
-            drv_show_log.addStdout(f"getting derivation infos\n")
+            drv_show_log.addStdout(f"getting derivation infos for valid derivations\n")
             cmd = yield self.makeRemoteShellCommand(
                 stdioLogName=None,
                 collectStdout=True,
                 command=(
                     ["nix", "derivation", "show", "--recursive"]
-                    + [ drv for drv in (job.get("drvPath") for job in filtered_jobs) if drv ]
+                    + [ drv for drv in (job.get("drvPath") for job in succeeded_jobs) if drv ]
                 ),
             )
             yield self.runCommand(cmd)
@@ -399,7 +411,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
                     r.update(*[ deps[k] for k in r ])
                 return r.difference([key])
 
-            job_set = set(( drv for drv in ( job.get("drvPath") for job in filtered_jobs ) if drv ))
+            job_set = set(( drv for drv in ( job.get("drvPath") for job in succeeded_jobs ) if drv ))
             all_deps = { k: list(closure_of(k, all_deps).intersection(job_set)) for k in job_set }
 
             self.build.addStepsAfterCurrentStep(
@@ -407,7 +419,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
                     BuildTrigger(
                         builds_scheduler_group=f"{project_name}-nix-build",
                         name="build derivations",
-                        jobs=filtered_jobs,
+                        jobs=succeeded_jobs,
                         all_deps=all_deps,
                     ),
                 ],