diff --git a/t/evaluator/evaluate-constituents-broken.t b/t/evaluator/evaluate-constituents-broken.t
new file mode 100644
index 00000000..ed25d192
--- /dev/null
+++ b/t/evaluator/evaluate-constituents-broken.t
@@ -0,0 +1,32 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use Hydra::Helper::Exec;
+
+my $ctx = test_context();
+
+my $jobsetCtx = $ctx->makeJobset(
+    expression => 'constituents-broken.nix',
+);
+my $jobset = $jobsetCtx->{"jobset"};
+
+my ($res, $stdout, $stderr) = captureStdoutStderr(60,
+    ("hydra-eval-jobset", $jobsetCtx->{"project"}->name, $jobset->name)
+);
+isnt($res, 0, "hydra-eval-jobset exits non-zero");
+ok(utf8::decode($stderr), "Stderr output is UTF8-clean");
+like(
+    $stderr,
+    qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
+    "The stderr record includes a relevant error message"
+);
+
+$jobset->discard_changes;  # refresh from DB
+like(
+    $jobset->errormsg,
+    qr/aggregate job ‘mixed_aggregate’ failed with the error: constituentA: does not exist/,
+    "The jobset records a relevant error message"
+);
+
+done_testing;
diff --git a/t/jobs/constituents-broken.nix b/t/jobs/constituents-broken.nix
new file mode 100644
index 00000000..0445a990
--- /dev/null
+++ b/t/jobs/constituents-broken.nix
@@ -0,0 +1,19 @@
+with import ./config.nix;
+rec {
+  constituentA = null;
+
+  constituentB = mkDerivation {
+    name = "empty-dir-B";
+    builder = ./empty-dir-builder.sh;
+  };
+
+  mixed_aggregate = mkDerivation {
+    name = "mixed_aggregate";
+    _hydraAggregate = true;
+    constituents = [
+      "constituentA"
+      constituentB
+    ];
+    builder = ./empty-dir-builder.sh;
+  };
+}