diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm
index 0d51b9fb..32dea879 100644
--- a/src/lib/Hydra/Helper/AddBuilds.pm
+++ b/src/lib/Hydra/Helper/AddBuilds.pm
@@ -282,7 +282,16 @@ sub evalJobs {
         SuppressEmpty => '')
         or die "cannot parse XML output";
 
-    return ($jobs, $nixExprInput);
+    my %jobNames;
+    my $errors;
+    foreach my $job (@{$jobs->{job}}) {
+        $jobNames{$job->{jobName}}++;
+        if ($jobNames{$job->{jobName}} == 2) {
+            $errors .= "warning: there are multiple jobs named ‘$job->{jobName}’; support for this will go away soon!\n\n";
+        }
+    }
+
+    return ($jobs, $nixExprInput, $errors);
 }
 
 
diff --git a/src/script/hydra-evaluator b/src/script/hydra-evaluator
index 646407a2..52192c74 100755
--- a/src/script/hydra-evaluator
+++ b/src/script/hydra-evaluator
@@ -133,7 +133,7 @@ sub checkJobsetWrapped {
 
     # Evaluate the job expression.
     my $evalStart = time;
-    my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
+    my ($jobs, $nixExprInput, $msg) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
     my $evalStop = time;
 
     my $jobOutPathMap = {};
@@ -238,7 +238,6 @@ sub checkJobsetWrapped {
     });
 
     # Store the error messages for jobs that failed to evaluate.
-    my $msg = "";
     foreach my $error (@{$jobs->{error}}) {
         my $bindings = "";
         foreach my $arg (@{$error->{arg}}) {