Merge pull request #878 from grahamc/test-runcommand

Test RunCommand's behavior

src/script/hydra-notify

@@ -6,11 +6,18 @@ use Hydra::Plugin;
 use Hydra::Helper::Nix;
 use Hydra::Helper::AddBuilds;
 use IO::Select;
+use Getopt::Long;
 
 STDERR->autoflush(1);
 STDOUT->autoflush(1);
 binmode STDERR, ":encoding(utf8)";
 
+my $queued_only;
+
+GetOptions(
+    "queued-only" => \$queued_only
+) or exit 1;
+
 my $config = getHydraConfig();
 
 my $db = Hydra::Model::DB->new();
@@ -103,11 +110,12 @@ for my $build ($db->resultset('Builds')->search(
     buildFinished($build);
 }
 
+
 # Process incoming notifications.
 my $fd = $dbh->func("getfd");
 my $sel = IO::Select->new($fd);
 
-while (1) {
+while (!$queued_only) {
     $sel->can_read;
 
     while (my $notify = $dbh->func("pg_notifies")) {

tests/jobs/runcommand.nix

@@ -0,0 +1,25 @@
+with import ./config.nix;
+{
+  metrics = mkDerivation {
+    name = "my-build-product";
+    builder = "/bin/sh";
+    outputs = [ "out" "bin" ];
+    args = [
+      (
+        builtins.toFile "builder.sh" ''
+          #! /bin/sh
+
+          echo "$PATH"
+
+          mkdir $bin
+          echo "foo" > $bin/bar
+
+          metrics=$out/nix-support/hydra-metrics
+          mkdir -p "$(dirname "$metrics")"
+          echo "lineCoverage 18 %" >> "$metrics"
+          echo "maxResident 27 KiB" >> "$metrics"
+        ''
+      )
+    ];
+  };
+}

tests/lib/Setup.pm

@@ -8,9 +8,32 @@ use File::Path qw(make_path);
 use Cwd;
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(test_init hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild updateRepository);
+our @EXPORT = qw(test_init hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild sendNotifications updateRepository);
+
+# Set up the environment for running tests.
+#
+# Hash Parameters:
+#
+#  * hydra_config: configuration for the Hydra processes for your test.
+#
+# This clears several environment variables and sets them to ephemeral
+# values: a temporary database, temporary Nix store, temporary Hydra
+# data directory, etc.
+#
+# Note: This function must run _very_ early, before nearly any Hydra
+# libraries are loaded. To use this, you very likely need to `use Setup`
+# and then run `test_init`, and then `require` the Hydra libraries you
+# need.
+#
+# It returns a tuple: a handle to a temporary directory and a handle to
+# the postgres service. If either of these variables go out of scope,
+# those resources are released and the test environment becomes invalid.
+#
+# Look at the top of an existing `.t` file to see how this should be used
+# in practice.
+sub test_init {
+    my %opts = @_;
 
-sub test_init() {
     my $dir = File::Temp->newdir();
 
     $ENV{'HYDRA_DATA'} = "$dir/hydra-data";
@@ -22,6 +45,12 @@ sub test_init() {
     print $fh "sandbox = false\n";
     close $fh;
 
+    $ENV{'HYDRA_CONFIG'} = "$dir/hydra.conf";
+
+    open(my $fh, '>', $ENV{'HYDRA_CONFIG'}) or die "Could not open file '" . $ENV{'HYDRA_CONFIG'}. " $!";
+    print $fh $opts{'hydra_config'} || "";
+    close $fh;
+
     $ENV{'NIX_STATE_DIR'} = "$dir/nix/var/nix";
 
     $ENV{'NIX_MANIFESTS_DIR'} = "$dir/nix/var/nix/manifests";
@@ -114,6 +143,15 @@ sub runBuild {
     return !$res;
 }
 
+sub sendNotifications() {
+    my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-notify", "--queued-only"));
+    if ($res) {
+        print STDERR "hydra notify stdout: $stdout\n" if $stdout ne "";
+        print STDERR "hydra notify stderr: $stderr\n" if $stderr ne "";
+    }
+    return !$res;
+}
+
 sub updateRepository {
     my ($scm, $update, $scratchdir) = @_;
     my $curdir = getcwd;

tests/plugins/runcommand.t

@@ -0,0 +1,136 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use Cwd;
+use JSON;
+use Setup;
+
+(my $datadir, my $pgsql) = test_init(
+    hydra_config => q|
+    <runcommand>
+      command = cp "$HYDRA_JSON" "$HYDRA_DATA/joboutput.json"
+    </runcommand>
+|);
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use Test2::V0;
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+# Most basic test case, no parameters
+my $jobset = createBaseJobset("basic", "runcommand.nix");
+
+ok(evalSucceeds($jobset), "Evaluating jobs/runcommand.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand.nix should result in 1 build1");
+
+(my $build) = queuedBuildsForJobset($jobset);
+
+is($build->job, "metrics", "The only job should be metrics");
+ok(runBuild($build), "Build should exit with code 0");
+my $newbuild = $db->resultset('Builds')->find($build->id);
+is($newbuild->finished, 1, "Build should be finished.");
+is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
+
+ok(sendNotifications(), "Notifications execute successfully.");
+
+my $dat = do {
+    my $filename = $ENV{'HYDRA_DATA'} . "/joboutput.json";
+    open(my $json_fh, "<", $filename)
+        or die("Can't open \$filename\": $!\n");
+    local $/;
+    my $json = JSON->new;
+    $json->decode(<$json_fh>)
+};
+
+use Data::Dumper;
+print Dumper($dat);
+
+subtest "Validate the top level fields match" => sub {
+    is($dat->{build}, $newbuild->id, "The build event matches our expected ID.");
+    is($dat->{buildStatus}, 0, "The build status matches.");
+    is($dat->{event}, "buildFinished", "The build event matches.");
+    is($dat->{finished}, 1, "The build finished.");
+    is($dat->{project}, "tests", "The project matches.");
+    is($dat->{jobset}, "basic", "The jobset matches.");
+    is($dat->{job}, "metrics", "The job matches.");
+    is($dat->{drvPath}, $newbuild->drvpath, "The derivation path matches.");
+    is($dat->{timestamp}, $newbuild->timestamp, "The result has a timestamp field.");
+    is($dat->{startTime}, $newbuild->starttime, "The result has a startTime field.");
+    is($dat->{stopTime}, $newbuild->stoptime, "The result has a stopTime field.");
+};
+
+subtest "Validate the outputs match" => sub {
+    is(scalar(@{$dat->{outputs}}), 2, "There are exactly two outputs");
+
+    subtest "output: out" => sub {
+        my ($output) = grep { $_->{name} eq "out" } @{$dat->{outputs}};
+        my $expectedoutput = $newbuild->buildoutputs->find({name => "out"});
+
+        is($output->{name}, "out", "Output is named corrrectly");
+        is($output->{path}, $expectedoutput->path, "The output path matches the database's path.");
+    };
+
+    subtest "output: bin" => sub {
+        my ($output) = grep { $_->{name} eq "bin" } @{$dat->{outputs}};
+        my $expectedoutput = $newbuild->buildoutputs->find({name => "bin"});
+
+        is($output->{name}, "bin", "Output is named corrrectly");
+        is($output->{path}, $expectedoutput->path, "The output path matches the database's path.");
+    };
+};
+
+subtest "Validate the metrics match" => sub {
+    is(scalar(@{$dat->{metrics}}), 2, "There are exactly two metrics");
+
+    my ($lineCoverage)  = grep { $_->{name} eq "lineCoverage" } @{$dat->{metrics}};
+    my ($maxResident) = grep { $_->{name} eq "maxResident" } @{$dat->{metrics}};
+
+    subtest "verifying the lineCoverage metric" => sub {
+        is($lineCoverage->{name}, "lineCoverage", "The name matches.");
+        is($lineCoverage->{value}, 18, "The value matches.");
+        is($lineCoverage->{unit}, "%", "The unit matches.");
+    };
+
+    subtest "verifying the maxResident metric" => sub {
+        is($maxResident->{name}, "maxResident", "The name matches.");
+        is($maxResident->{value}, 27, "The value matches.");
+        is($maxResident->{unit}, "KiB", "The unit matches.");
+    };
+};
+
+subtest "Validate the products match" => sub {
+    is(scalar(@{$dat->{outputs}}), 2, "There are exactly two outputs");
+
+    subtest "product: out" => sub {
+        my ($product) = grep { $_->{name} eq "my-build-product" } @{$dat->{products}};
+        my $expectedproduct = $newbuild->buildproducts->find({name => "my-build-product"});
+
+        is($product->{name}, "my-build-product", "The build product is named correctly.");
+        is($product->{subtype}, "", "The subtype is empty.");
+        is($product->{productNr}, $expectedproduct->productnr, "The product number matches.");
+        is($product->{defaultPath}, "", "The default path matches.");
+        is($product->{path}, $expectedproduct->path, "The path matches the output.");
+        is($product->{fileSize}, undef, "The fileSize is undefined for the nix-build output type.");
+        is($product->{sha256hash}, undef, "The sha256hash is undefined for the nix-build output type.");
+    };
+
+    subtest "output: bin" => sub {
+        my ($product) = grep { $_->{name} eq "my-build-product-bin" } @{$dat->{products}};
+        my $expectedproduct = $newbuild->buildproducts->find({name => "my-build-product-bin"});
+
+        is($product->{name}, "my-build-product-bin", "The build product is named correctly.");
+        is($product->{subtype}, "bin", "The subtype matches the output name");
+        is($product->{productNr}, $expectedproduct->productnr, "The product number matches.");
+        is($product->{defaultPath}, "", "The default path matches.");
+        is($product->{path}, $expectedproduct->path, "The path matches the output.");
+        is($product->{fileSize}, undef, "The fileSize is undefined for the nix-build output type.");
+        is($product->{sha256hash}, undef, "The sha256hash is undefined for the nix-build output type.");
+    };
+};
+
+done_testing;