diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py
index aa66e54..fdfcecc 100644
--- a/buildbot_nix/__init__.py
+++ b/buildbot_nix/__init__.py
@@ -381,6 +381,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
         if result == util.SUCCESS:
             # create a ShellCommand for each stage and add them to the build
             jobs = []
+            eval_errors: list[tuple[str, str]] = []
 
             for line in self.observer.getStdout().split("\n"):
                 if line != "":
@@ -392,11 +393,22 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
                     jobs.append(job)
             filtered_jobs = []
             for job in jobs:
+                if err := job.get("error"):
+                    eval_errors.append((job.get('attr'), err))
+
                 system = job.get("system")
-                if not system or system in self.supported_systems:  # report eval errors
+                if not system or system in self.supported_systems:
                     filtered_jobs.append(job)
 
             drv_show_log: Log = yield self.getLog("stdio")
+
+            if eval_errors:
+                msg = "Failing job due to evaluation errors!\n"
+                msg += "\n".join(
+                    f"- {attr}: {failure}" for (attr, failure) in eval_errors)
+                drv_show_log.addStdout(msg)
+                raise BuildbotNixError("Evaluation error in attributes: " + ", ".join(attr for (attr, _) in eval_errors))
+
             drv_show_log.addStdout(f"getting derivation infos\n")
             cmd = yield self.makeRemoteShellCommand(
                 stdioLogName=None,