diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py
index 22a77f6..6110346 100644
--- a/buildbot_nix/__init__.py
+++ b/buildbot_nix/__init__.py
@@ -404,6 +404,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
         if result == util.SUCCESS:
             # create a ShellCommand for each stage and add them to the build
             jobs = []
+            eval_errors: list[tuple[str, str]] = []
 
             for line in self.observer.getStdout().split("\n"):
                 if line != "":
@@ -415,14 +416,25 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
                     jobs.append(job)
             filtered_jobs = []
             for job in jobs:
+                if err := job.get("error"):
+                    eval_errors.append((job.get('attr'), err))
+
                 system = job.get("system")
-                if not system or system in self.supported_systems:  # report eval errors
+                if not system or system in self.supported_systems:
                     filtered_jobs.append(job)
 
             # Filter out failed evaluations
             succeeded_jobs = [job for job in filtered_jobs if job.get('error') is None]
 
             drv_show_log: StreamLog = yield self.getLog("stdio")
+
+            if eval_errors:
+                msg = "Failing job due to evaluation errors!\n"
+                msg += "\n".join(
+                    f"- {attr}: {failure}" for (attr, failure) in eval_errors)
+                drv_show_log.addStdout(msg)
+                raise BuildbotNixError("Evaluation error in attributes: " + ", ".join(attr for (attr, _) in eval_errors))
+
             all_deps = dict()
 
             def closure_of(key, deps):