Merge pull request #1101 from DeterminateSystems/hydra-eval-jobset-notifications

hydra-eval-jobset: test emitted notifications

t/jobs/hydra-eval-notifications.nix

@@ -0,0 +1,30 @@
+# This file gets copied around, and intentionally does not refer to
+# anything but itself as "default.nix".
+
+let
+  simpleDerivation = name: builderText: derivation {
+    inherit name;
+    system = builtins.currentSystem;
+    builder = "/bin/sh";
+    args = [
+      (builtins.toFile "builder.sh" builderText)
+    ];
+  };
+in
+{
+  stable-job-queued = simpleDerivation "stable-job-queued" ''
+    echo "here is a stable job that passes every time" > $out
+  '';
+
+  stable-job-passing = simpleDerivation "stable-job-passing" ''
+    echo "here is a stable job that passes every time" > $out
+  '';
+
+  stable-job-failing = simpleDerivation "stable-job-failing" ''
+    echo "this job is a stable job that fails every time" > $out
+  '';
+
+  variable-job = simpleDerivation "variable-job" ''
+    echo ${builtins.toFile "default.nix" (builtins.readFile ./default.nix)} > $out
+  '';
+}

t/lib/CliRunners.pm

@@ -4,8 +4,12 @@ use strict;
 package CliRunners;
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
-                 evalSucceeds runBuild sendNotifications
+    captureStdoutStderr
-                 captureStdoutStderr);
+    evalFails
+    evalSucceeds
+    runBuild
+    sendNotifications
+);
 
 
 sub captureStdoutStderr {
@@ -21,13 +25,28 @@ sub evalSucceeds {
     my ($jobset) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
     $jobset->discard_changes;  # refresh from DB
+    if ($res) {
         chomp $stdout; chomp $stderr;
-    print STDERR "Evaluation errors for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
+        print STDERR "Evaluation unexpectedly failed for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
         print STDERR "STDOUT: $stdout\n" if $stdout ne "";
         print STDERR "STDERR: $stderr\n" if $stderr ne "";
+    }
     return !$res;
 }
 
+sub evalFails {
+    my ($jobset) = @_;
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
+    $jobset->discard_changes;  # refresh from DB
+    if (!$res) {
+        chomp $stdout; chomp $stderr;
+        print STDERR "Evaluation unexpectedly succeeded for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
+        print STDERR "STDOUT: $stdout\n" if $stdout ne "";
+        print STDERR "STDERR: $stderr\n" if $stderr ne "";
+    }
+    return !!$res;
+}
+
 sub runBuild {
     my ($build) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id));

t/lib/HydraTestContext.pm

@@ -125,12 +125,14 @@ sub jobsdir {
 # Hash Parameters:
 #
 #  * expression: The file in the jobsdir directory to evaluate
+#  * jobsdir: An alternative jobsdir to source the expression from
 #  * build: Bool. Attempt to build all the resulting jobs. Default: false.
 sub makeAndEvaluateJobset {
     my ($self, %opts) = @_;
 
     my $expression = $opts{'expression'} || die "Mandatory 'expression' option not passed to makeAndEValuateJobset.";
     my $should_build = $opts{'build'} // 0;
+    my $jobsdir = $opts{'jobsdir'} // $self->jobsdir;
 
 
     # Create a new user for this test
@@ -155,7 +157,7 @@ sub makeAndEvaluateJobset {
         emailoverride => ""
     });
     my $jobsetinput = $jobset->jobsetinputs->create({name => "jobs", type => "path"});
-    $jobsetinput->jobsetinputalts->create({altnr => 0, value => $self->jobsdir});
+    $jobsetinput->jobsetinputalts->create({altnr => 0, value => $jobsdir});
 
     evalSucceeds($jobset) or die "Evaluating jobs/$expression should exit with return code 0";
 

t/lib/Setup.pm

@@ -11,10 +11,23 @@ use Cwd qw(abs_path getcwd);
 use CliRunners;
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(test_context test_init hydra_setup write_file nrBuildsForJobset queuedBuildsForJobset
+our @EXPORT = qw(
-                 nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput
+    captureStdoutStderr
-                 evalSucceeds runBuild sendNotifications updateRepository
+    createBaseJobset
-                 captureStdoutStderr);
+    createJobsetWithOneInput
+    evalFails
+    evalSucceeds
+    hydra_setup
+    nrBuildsForJobset
+    nrQueuedBuildsForJobset
+    queuedBuildsForJobset
+    runBuild
+    sendNotifications
+    test_context
+    test_init
+    updateRepository
+    write_file
+);
 
 # Set up the environment for running tests.
 #

t/scripts/hydra-eval-jobset/notifications.t

@@ -0,0 +1,91 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use File::Copy;
+use Hydra::PostgresListener;
+
+my $ctx = test_context(
+    hydra_config => q|
+# No caching for PathInput plugin, otherwise we get wrong values
+# (as it has a 30s window where no changes to the file are considered).
+path_input_cache_validity_seconds = 0
+|
+);
+
+my $dbh = $ctx->db()->storage->dbh;
+my $listener = Hydra::PostgresListener->new($dbh);
+
+$listener->subscribe("cached_build_finished");
+$listener->subscribe("cached_build_queued");
+$listener->subscribe("build_queued");
+$listener->subscribe("eval_failed");
+$listener->subscribe("eval_cached");
+$listener->subscribe("eval_added");
+$listener->subscribe("eval_started");
+$listener->subscribe("builds_added");
+
+
+my $jobsetdir = $ctx->tmpdir . '/jobset';
+mkdir($jobsetdir);
+copy($ctx->jobsdir . '/hydra-eval-notifications.nix', "$jobsetdir/default.nix");
+
+my $builds = $ctx->makeAndEvaluateJobset(
+    expression => "default.nix",
+    jobsdir => $jobsetdir,
+    build => 0
+);
+
+subtest "on the initial evaluation" => sub {
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "every eval starts with a notification");
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 1/4 builds being queued");
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 2/4 builds being queued");
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 3/4 builds being queued");
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 4/4 builds being queued");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_added", "the evaluation has completed");
+    is($listener->block_for_messages(0)->()->{"channel"}, "builds_added", "new builds have been scheduled");
+    is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
+};
+
+subtest "on a subsequent, totally cached / unchanged evaluation" => sub {
+    ok(evalSucceeds($builds->{"variable-job"}->jobset), "evaluating for the second time");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "an evaluation has started");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_cached", "the evaluation finished and nothing changed");
+    is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
+};
+
+subtest "on a fresh evaluation with changed sources" => sub {
+    open(my $fh, ">>", "${jobsetdir}/default.nix") or die "didn't open?";
+    say $fh "\n";
+    close $fh;
+
+    ok(runBuild($builds->{"stable-job-passing"}), "building the stable passing job");
+    $builds->{"stable-job-passing"}->discard_changes();
+
+    ok(runBuild($builds->{"stable-job-failing"}), "building the stable failing job");
+    $builds->{"stable-job-failing"}->discard_changes();
+
+    ok(evalSucceeds($builds->{"variable-job"}->jobset), "evaluating for the third time");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "the evaluation started");
+
+    is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect only one new build being queued");
+
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_added", "a new evaluation was added");
+    is($listener->block_for_messages(0)->()->{"channel"}, "builds_added", "a new build was added");
+    is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
+};
+
+subtest "on a fresh evaluation with corrupted sources" => sub {
+    open(my $fh, ">>", "${jobsetdir}/default.nix") or die "didn't open?";
+    say $fh "this is not valid nix code!\n";
+    close $fh;
+
+    ok(evalFails($builds->{"variable-job"}->jobset), "evaluating the corrupted job");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "the evaluation started");
+    is($listener->block_for_messages(0)->()->{"channel"}, "eval_failed", "the evaluation failed");
+    is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
+
+};
+
+done_testing;