t: create a evalFails helper

t/lib/CliRunners.pm

@@ -4,8 +4,12 @@ use strict;
 package CliRunners;
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
-                 evalSucceeds runBuild sendNotifications
-                 captureStdoutStderr);
+    captureStdoutStderr
+    evalFails
+    evalSucceeds
+    runBuild
+    sendNotifications
+);
 
 
 sub captureStdoutStderr {
@@ -30,6 +34,19 @@ sub evalSucceeds {
     return !$res;
 }
 
+sub evalFails {
+    my ($jobset) = @_;
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
+    $jobset->discard_changes;  # refresh from DB
+    if (!$res) {
+        chomp $stdout; chomp $stderr;
+        print STDERR "Evaluation errors for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
+        print STDERR "STDOUT: $stdout\n" if $stdout ne "";
+        print STDERR "STDERR: $stderr\n" if $stderr ne "";
+    }
+    return !!$res;
+}
+
 sub runBuild {
     my ($build) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id));

t/lib/Setup.pm

@@ -11,10 +11,23 @@ use Cwd qw(abs_path getcwd);
 use CliRunners;
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(test_context test_init hydra_setup write_file nrBuildsForJobset queuedBuildsForJobset
-                 nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput
-                 evalSucceeds runBuild sendNotifications updateRepository
-                 captureStdoutStderr);
+our @EXPORT = qw(
+    captureStdoutStderr
+    createBaseJobset
+    createJobsetWithOneInput
+    evalFails
+    evalSucceeds
+    hydra_setup
+    nrBuildsForJobset
+    nrQueuedBuildsForJobset
+    queuedBuildsForJobset
+    runBuild
+    sendNotifications
+    test_context
+    test_init
+    updateRepository
+    write_file
+);
 
 # Set up the environment for running tests.
 #