From 40e556aee557652ea0a6230629aff12473120684 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 10 Jan 2022 16:46:19 -0500
Subject: [PATCH 1/5] tests: evalSucceeds: don't print stdout/stderr unless it
 fails

---
 t/lib/CliRunners.pm | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/t/lib/CliRunners.pm b/t/lib/CliRunners.pm
index f857d468..82f17403 100644
--- a/t/lib/CliRunners.pm
+++ b/t/lib/CliRunners.pm
@@ -21,10 +21,12 @@ sub evalSucceeds {
     my ($jobset) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
     $jobset->discard_changes;  # refresh from DB
-    chomp $stdout; chomp $stderr;
-    print STDERR "Evaluation errors for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
-    print STDERR "STDOUT: $stdout\n" if $stdout ne "";
-    print STDERR "STDERR: $stderr\n" if $stderr ne "";
+    if ($res) {
+        chomp $stdout; chomp $stderr;
+        print STDERR "Evaluation errors for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
+        print STDERR "STDOUT: $stdout\n" if $stdout ne "";
+        print STDERR "STDERR: $stderr\n" if $stderr ne "";
+    }
     return !$res;
 }
 

From 8d4c448e9298eca4a37f8dd103c588069ff4be0e Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 10 Jan 2022 16:47:04 -0500
Subject: [PATCH 2/5] t: create a evalFails helper

---
 t/lib/CliRunners.pm | 21 +++++++++++++++++++--
 t/lib/Setup.pm      | 21 +++++++++++++++++----
 2 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/t/lib/CliRunners.pm b/t/lib/CliRunners.pm
index 82f17403..21235837 100644
--- a/t/lib/CliRunners.pm
+++ b/t/lib/CliRunners.pm
@@ -4,8 +4,12 @@ use strict;
 package CliRunners;
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
-                 evalSucceeds runBuild sendNotifications
-                 captureStdoutStderr);
+    captureStdoutStderr
+    evalFails
+    evalSucceeds
+    runBuild
+    sendNotifications
+);
 
 
 sub captureStdoutStderr {
@@ -30,6 +34,19 @@ sub evalSucceeds {
     return !$res;
 }
 
+sub evalFails {
+    my ($jobset) = @_;
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
+    $jobset->discard_changes;  # refresh from DB
+    if (!$res) {
+        chomp $stdout; chomp $stderr;
+        print STDERR "Evaluation errors for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
+        print STDERR "STDOUT: $stdout\n" if $stdout ne "";
+        print STDERR "STDERR: $stderr\n" if $stderr ne "";
+    }
+    return !!$res;
+}
+
 sub runBuild {
     my ($build) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id));
diff --git a/t/lib/Setup.pm b/t/lib/Setup.pm
index d0f0230b..d7772731 100644
--- a/t/lib/Setup.pm
+++ b/t/lib/Setup.pm
@@ -11,10 +11,23 @@ use Cwd qw(abs_path getcwd);
 use CliRunners;
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(test_context test_init hydra_setup write_file nrBuildsForJobset queuedBuildsForJobset
-                 nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput
-                 evalSucceeds runBuild sendNotifications updateRepository
-                 captureStdoutStderr);
+our @EXPORT = qw(
+    captureStdoutStderr
+    createBaseJobset
+    createJobsetWithOneInput
+    evalFails
+    evalSucceeds
+    hydra_setup
+    nrBuildsForJobset
+    nrQueuedBuildsForJobset
+    queuedBuildsForJobset
+    runBuild
+    sendNotifications
+    test_context
+    test_init
+    updateRepository
+    write_file
+);
 
 # Set up the environment for running tests.
 #

From ae38cc5d040d668ab5d55bf8e9e0077d04778fd8 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 10 Jan 2022 16:47:22 -0500
Subject: [PATCH 3/5] test_context: support an override on jobsdir when
 creating a jobset

---
 t/lib/HydraTestContext.pm | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/t/lib/HydraTestContext.pm b/t/lib/HydraTestContext.pm
index 43806c6f..9f9db4f2 100644
--- a/t/lib/HydraTestContext.pm
+++ b/t/lib/HydraTestContext.pm
@@ -125,12 +125,14 @@ sub jobsdir {
 # Hash Parameters:
 #
 #  * expression: The file in the jobsdir directory to evaluate
+#  * jobsdir: An alternative jobsdir to source the expression from
 #  * build: Bool. Attempt to build all the resulting jobs. Default: false.
 sub makeAndEvaluateJobset {
     my ($self, %opts) = @_;
 
     my $expression = $opts{'expression'} || die "Mandatory 'expression' option not passed to makeAndEValuateJobset.";
     my $should_build = $opts{'build'} // 0;
+    my $jobsdir = $opts{'jobsdir'} // $self->jobsdir;
 
 
     # Create a new user for this test
@@ -155,7 +157,7 @@ sub makeAndEvaluateJobset {
         emailoverride => ""
     });
     my $jobsetinput = $jobset->jobsetinputs->create({name => "jobs", type => "path"});
-    $jobsetinput->jobsetinputalts->create({altnr => 0, value => $self->jobsdir});
+    $jobsetinput->jobsetinputalts->create({altnr => 0, value => $jobsdir});
 
     evalSucceeds($jobset) or die "Evaluating jobs/$expression should exit with return code 0";
 

From 0ada412979c3b199c5005473fb892788c232be1b Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 10 Jan 2022 16:49:35 -0500
Subject: [PATCH 4/5] hydra-eval-jobset: write a test validating the events
 that comes out

---
 t/jobs/hydra-eval-notifications.nix         | 30 +++++++
 t/scripts/hydra-eval-jobset/notifications.t | 91 +++++++++++++++++++++
 2 files changed, 121 insertions(+)
 create mode 100644 t/jobs/hydra-eval-notifications.nix
 create mode 100644 t/scripts/hydra-eval-jobset/notifications.t

diff --git a/t/jobs/hydra-eval-notifications.nix b/t/jobs/hydra-eval-notifications.nix
new file mode 100644
index 00000000..0be7a5a4
--- /dev/null
+++ b/t/jobs/hydra-eval-notifications.nix
@@ -0,0 +1,30 @@
+# This file gets copied around, and intentionally does not refer to
+# anything but itself as "default.nix".
+
+let
+  simpleDerivation = name: builderText: derivation {
+    inherit name;
+    system = builtins.currentSystem;
+    builder = "/bin/sh";
+    args = [
+      (builtins.toFile "builder.sh" builderText)
+    ];
+  };
+in
+{
+  stable-job-queued = simpleDerivation "stable-job-queued" ''
+    echo "here is a stable job that passes every time" > $out
+  '';
+
+  stable-job-passing = simpleDerivation "stable-job-passing" ''
+    echo "here is a stable job that passes every time" > $out
+  '';
+
+  stable-job-failing = simpleDerivation "stable-job-failing" ''
+    echo "this job is a stable job that fails every time" > $out
+  '';
+
+  variable-job = simpleDerivation "variable-job" ''
+    echo ${builtins.toFile "default.nix" (builtins.readFile ./default.nix)} > $out
+  '';
+}
diff --git a/t/scripts/hydra-eval-jobset/notifications.t b/t/scripts/hydra-eval-jobset/notifications.t
new file mode 100644
index 00000000..cd716779
--- /dev/null
+++ b/t/scripts/hydra-eval-jobset/notifications.t
@@ -0,0 +1,91 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use File::Copy;
+use Hydra::PostgresListener;
+
+my $ctx = test_context(
+    hydra_config => q|
+# No caching for PathInput plugin, otherwise we get wrong values
+# (as it has a 30s window where no changes to the file are considered).
+path_input_cache_validity_seconds = 0
+|
+);
+
+my $dbh = $ctx->db()->storage->dbh;
+my $listener = Hydra::PostgresListener->new($dbh);
+
+$listener->subscribe("cached_build_finished");
+$listener->subscribe("cached_build_queued");
+$listener->subscribe("build_queued");
+$listener->subscribe("eval_failed");
+$listener->subscribe("eval_cached");
+$listener->subscribe("eval_added");
+$listener->subscribe("eval_started");
+$listener->subscribe("builds_added");
+
+
+my $jobsetdir = $ctx->tmpdir . '/jobset';
+mkdir($jobsetdir);
+copy($ctx->jobsdir . '/hydra-eval-notifications.nix', "$jobsetdir/default.nix");
+
+my $builds = $ctx->makeAndEvaluateJobset(
+    expression => "default.nix",
+    jobsdir => $jobsetdir,
+    build => 0
+);
+
+subtest "on the initial evaluation" => sub {
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "every eval starts with a notification");
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 1/4 builds being queued");
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 2/4 builds being queued");
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 3/4 builds being queued");
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 4/4 builds being queued");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_added", "the evaluation has completed");
+    is($listener->block_for_messages(0)->()->{"channel"}, "builds_added", "new builds have been scheduled");
+    is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
+};
+
+subtest "on a subsequent, totally cached / unchanged evaluation" => sub {
+    ok(evalSucceeds($builds->{"variable-job"}->jobset), "evaluating for the second time");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "an evaluation has started");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_cached", "the evaluation finished and nothing changed");
+    is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
+};
+
+subtest "on a fresh evaluation with changed sources" => sub {
+    open(my $fh, ">>", "${jobsetdir}/default.nix") or die "didn't open?";
+    say $fh "\n";
+    close $fh;
+
+    ok(runBuild($builds->{"stable-job-passing"}), "building the stable passing job");
+    $builds->{"stable-job-passing"}->discard_changes();
+
+    ok(runBuild($builds->{"stable-job-failing"}), "building the stable failing job");
+    $builds->{"stable-job-failing"}->discard_changes();
+
+    ok(evalSucceeds($builds->{"variable-job"}->jobset), "evaluating for the third time");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "the evaluation started");
+
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect only one new build being queued");
+
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_added", "a new evaluation was added");
+    is($listener->block_for_messages(0)->()->{"channel"}, "builds_added", "a new build was added");
+    is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
+};
+
+subtest "on a fresh evaluation with corrupted sources" => sub {
+    open(my $fh, ">>", "${jobsetdir}/default.nix") or die "didn't open?";
+    say $fh "this is not valid nix code!\n";
+    close $fh;
+
+    ok(evalFails($builds->{"variable-job"}->jobset), "evaluating the corrupted job");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "the evaluation started");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_failed", "the evaluation failed");
+    is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
+
+};
+
+done_testing;

From 2e195c524de09b52472cb29bde270448d8a1f83d Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 11 Jan 2022 13:10:43 -0500
Subject: [PATCH 5/5] evalSucceds/evalFails: log that the success or failure
 was not expected

---
 t/lib/CliRunners.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/t/lib/CliRunners.pm b/t/lib/CliRunners.pm
index 21235837..e693eeb7 100644
--- a/t/lib/CliRunners.pm
+++ b/t/lib/CliRunners.pm
@@ -27,7 +27,7 @@ sub evalSucceeds {
     $jobset->discard_changes;  # refresh from DB
     if ($res) {
         chomp $stdout; chomp $stderr;
-        print STDERR "Evaluation errors for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
+        print STDERR "Evaluation unexpectedly failed for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
         print STDERR "STDOUT: $stdout\n" if $stdout ne "";
         print STDERR "STDERR: $stderr\n" if $stderr ne "";
     }
@@ -40,7 +40,7 @@ sub evalFails {
     $jobset->discard_changes;  # refresh from DB
     if (!$res) {
         chomp $stdout; chomp $stderr;
-        print STDERR "Evaluation errors for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
+        print STDERR "Evaluation unexpectedly succeeded for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
         print STDERR "STDOUT: $stdout\n" if $stdout ne "";
         print STDERR "STDERR: $stderr\n" if $stderr ne "";
     }