Merge pull request #1101 from DeterminateSystems/hydra-eval-jobset-notifications
hydra-eval-jobset: test emitted notifications
This commit is contained in:
commit
769f56f793
30
t/jobs/hydra-eval-notifications.nix
Normal file
30
t/jobs/hydra-eval-notifications.nix
Normal file
|
@ -0,0 +1,30 @@
|
|||
# This file gets copied around, and intentionally does not refer to
|
||||
# anything but itself as "default.nix".
|
||||
|
||||
let
|
||||
simpleDerivation = name: builderText: derivation {
|
||||
inherit name;
|
||||
system = builtins.currentSystem;
|
||||
builder = "/bin/sh";
|
||||
args = [
|
||||
(builtins.toFile "builder.sh" builderText)
|
||||
];
|
||||
};
|
||||
in
|
||||
{
|
||||
stable-job-queued = simpleDerivation "stable-job-queued" ''
|
||||
echo "here is a stable job that passes every time" > $out
|
||||
'';
|
||||
|
||||
stable-job-passing = simpleDerivation "stable-job-passing" ''
|
||||
echo "here is a stable job that passes every time" > $out
|
||||
'';
|
||||
|
||||
stable-job-failing = simpleDerivation "stable-job-failing" ''
|
||||
echo "this job is a stable job that fails every time" > $out
|
||||
'';
|
||||
|
||||
variable-job = simpleDerivation "variable-job" ''
|
||||
echo ${builtins.toFile "default.nix" (builtins.readFile ./default.nix)} > $out
|
||||
'';
|
||||
}
|
|
@ -4,8 +4,12 @@ use strict;
|
|||
package CliRunners;
|
||||
our @ISA = qw(Exporter);
|
||||
our @EXPORT = qw(
|
||||
evalSucceeds runBuild sendNotifications
|
||||
captureStdoutStderr);
|
||||
captureStdoutStderr
|
||||
evalFails
|
||||
evalSucceeds
|
||||
runBuild
|
||||
sendNotifications
|
||||
);
|
||||
|
||||
|
||||
sub captureStdoutStderr {
|
||||
|
@ -21,13 +25,28 @@ sub evalSucceeds {
|
|||
my ($jobset) = @_;
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
|
||||
$jobset->discard_changes; # refresh from DB
|
||||
if ($res) {
|
||||
chomp $stdout; chomp $stderr;
|
||||
print STDERR "Evaluation errors for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
|
||||
print STDERR "Evaluation unexpectedly failed for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
|
||||
print STDERR "STDOUT: $stdout\n" if $stdout ne "";
|
||||
print STDERR "STDERR: $stderr\n" if $stderr ne "";
|
||||
}
|
||||
return !$res;
|
||||
}
|
||||
|
||||
sub evalFails {
|
||||
my ($jobset) = @_;
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-eval-jobset", $jobset->project->name, $jobset->name));
|
||||
$jobset->discard_changes; # refresh from DB
|
||||
if (!$res) {
|
||||
chomp $stdout; chomp $stderr;
|
||||
print STDERR "Evaluation unexpectedly succeeded for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
|
||||
print STDERR "STDOUT: $stdout\n" if $stdout ne "";
|
||||
print STDERR "STDERR: $stderr\n" if $stderr ne "";
|
||||
}
|
||||
return !!$res;
|
||||
}
|
||||
|
||||
sub runBuild {
|
||||
my ($build) = @_;
|
||||
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id));
|
||||
|
|
|
@ -125,12 +125,14 @@ sub jobsdir {
|
|||
# Hash Parameters:
|
||||
#
|
||||
# * expression: The file in the jobsdir directory to evaluate
|
||||
# * jobsdir: An alternative jobsdir to source the expression from
|
||||
# * build: Bool. Attempt to build all the resulting jobs. Default: false.
|
||||
sub makeAndEvaluateJobset {
|
||||
my ($self, %opts) = @_;
|
||||
|
||||
my $expression = $opts{'expression'} || die "Mandatory 'expression' option not passed to makeAndEValuateJobset.";
|
||||
my $should_build = $opts{'build'} // 0;
|
||||
my $jobsdir = $opts{'jobsdir'} // $self->jobsdir;
|
||||
|
||||
|
||||
# Create a new user for this test
|
||||
|
@ -155,7 +157,7 @@ sub makeAndEvaluateJobset {
|
|||
emailoverride => ""
|
||||
});
|
||||
my $jobsetinput = $jobset->jobsetinputs->create({name => "jobs", type => "path"});
|
||||
$jobsetinput->jobsetinputalts->create({altnr => 0, value => $self->jobsdir});
|
||||
$jobsetinput->jobsetinputalts->create({altnr => 0, value => $jobsdir});
|
||||
|
||||
evalSucceeds($jobset) or die "Evaluating jobs/$expression should exit with return code 0";
|
||||
|
||||
|
|
|
@ -11,10 +11,23 @@ use Cwd qw(abs_path getcwd);
|
|||
use CliRunners;
|
||||
|
||||
our @ISA = qw(Exporter);
|
||||
our @EXPORT = qw(test_context test_init hydra_setup write_file nrBuildsForJobset queuedBuildsForJobset
|
||||
nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput
|
||||
evalSucceeds runBuild sendNotifications updateRepository
|
||||
captureStdoutStderr);
|
||||
our @EXPORT = qw(
|
||||
captureStdoutStderr
|
||||
createBaseJobset
|
||||
createJobsetWithOneInput
|
||||
evalFails
|
||||
evalSucceeds
|
||||
hydra_setup
|
||||
nrBuildsForJobset
|
||||
nrQueuedBuildsForJobset
|
||||
queuedBuildsForJobset
|
||||
runBuild
|
||||
sendNotifications
|
||||
test_context
|
||||
test_init
|
||||
updateRepository
|
||||
write_file
|
||||
);
|
||||
|
||||
# Set up the environment for running tests.
|
||||
#
|
||||
|
|
91
t/scripts/hydra-eval-jobset/notifications.t
Normal file
91
t/scripts/hydra-eval-jobset/notifications.t
Normal file
|
@ -0,0 +1,91 @@
|
|||
use feature 'unicode_strings';
|
||||
use strict;
|
||||
use warnings;
|
||||
use Setup;
|
||||
use Test2::V0;
|
||||
use File::Copy;
|
||||
use Hydra::PostgresListener;
|
||||
|
||||
my $ctx = test_context(
|
||||
hydra_config => q|
|
||||
# No caching for PathInput plugin, otherwise we get wrong values
|
||||
# (as it has a 30s window where no changes to the file are considered).
|
||||
path_input_cache_validity_seconds = 0
|
||||
|
|
||||
);
|
||||
|
||||
my $dbh = $ctx->db()->storage->dbh;
|
||||
my $listener = Hydra::PostgresListener->new($dbh);
|
||||
|
||||
$listener->subscribe("cached_build_finished");
|
||||
$listener->subscribe("cached_build_queued");
|
||||
$listener->subscribe("build_queued");
|
||||
$listener->subscribe("eval_failed");
|
||||
$listener->subscribe("eval_cached");
|
||||
$listener->subscribe("eval_added");
|
||||
$listener->subscribe("eval_started");
|
||||
$listener->subscribe("builds_added");
|
||||
|
||||
|
||||
my $jobsetdir = $ctx->tmpdir . '/jobset';
|
||||
mkdir($jobsetdir);
|
||||
copy($ctx->jobsdir . '/hydra-eval-notifications.nix', "$jobsetdir/default.nix");
|
||||
|
||||
my $builds = $ctx->makeAndEvaluateJobset(
|
||||
expression => "default.nix",
|
||||
jobsdir => $jobsetdir,
|
||||
build => 0
|
||||
);
|
||||
|
||||
subtest "on the initial evaluation" => sub {
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "every eval starts with a notification");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 1/4 builds being queued");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 2/4 builds being queued");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 3/4 builds being queued");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect 4/4 builds being queued");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "eval_added", "the evaluation has completed");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "builds_added", "new builds have been scheduled");
|
||||
is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
|
||||
};
|
||||
|
||||
subtest "on a subsequent, totally cached / unchanged evaluation" => sub {
|
||||
ok(evalSucceeds($builds->{"variable-job"}->jobset), "evaluating for the second time");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "an evaluation has started");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "eval_cached", "the evaluation finished and nothing changed");
|
||||
is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
|
||||
};
|
||||
|
||||
subtest "on a fresh evaluation with changed sources" => sub {
|
||||
open(my $fh, ">>", "${jobsetdir}/default.nix") or die "didn't open?";
|
||||
say $fh "\n";
|
||||
close $fh;
|
||||
|
||||
ok(runBuild($builds->{"stable-job-passing"}), "building the stable passing job");
|
||||
$builds->{"stable-job-passing"}->discard_changes();
|
||||
|
||||
ok(runBuild($builds->{"stable-job-failing"}), "building the stable failing job");
|
||||
$builds->{"stable-job-failing"}->discard_changes();
|
||||
|
||||
ok(evalSucceeds($builds->{"variable-job"}->jobset), "evaluating for the third time");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "the evaluation started");
|
||||
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "build_queued", "expect only one new build being queued");
|
||||
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "eval_added", "a new evaluation was added");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "builds_added", "a new build was added");
|
||||
is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
|
||||
};
|
||||
|
||||
subtest "on a fresh evaluation with corrupted sources" => sub {
|
||||
open(my $fh, ">>", "${jobsetdir}/default.nix") or die "didn't open?";
|
||||
say $fh "this is not valid nix code!\n";
|
||||
close $fh;
|
||||
|
||||
ok(evalFails($builds->{"variable-job"}->jobset), "evaluating the corrupted job");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "eval_started", "the evaluation started");
|
||||
is($listener->block_for_messages(0)->()->{"channel"}, "eval_failed", "the evaluation failed");
|
||||
is($listener->block_for_messages(0)->(), undef, "there are no more messages from the evaluator");
|
||||
|
||||
};
|
||||
|
||||
done_testing;
|
Loading…
Reference in a new issue