Merge pull request #1073 from DeterminateSystems/runcommand-tests

Runcommand: Expand the test coverage

src/lib/Hydra/Plugin/RunCommand.pm

@@ -38,86 +38,118 @@ sub eventMatches {
     return 0;
 }
 
+sub fanoutToCommands {
+    my ($config, $event, $project, $jobset, $job) = @_;
+
+    my @commands;
+
+    my $cfg = $config->{runcommand};
+    my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : ();
+
+    foreach my $conf (@config) {
+        my $matcher = $conf->{job} // "*:*:*";
+        next unless eventMatches($conf, $event);
+        next unless configSectionMatches(
+            $matcher,
+            $project,
+            $jobset,
+            $job);
+
+        if (!defined($conf->{command})) {
+            warn "<runcommand> section for '$matcher' lacks a 'command' option";
+            next;
+        }
+
+        push(@commands, {
+            matcher => $matcher,
+            command => $conf->{command},
+        })
+    }
+
+    return \@commands;
+}
+
+sub makeJsonPayload {
+    my ($event, $build) = @_;
+    my $json = {
+        event => $event,
+        build => $build->id,
+        finished => $build->get_column('finished') ? JSON::true : JSON::false,
+        timestamp => $build->get_column('timestamp'),
+        project => $build->get_column('project'),
+        jobset => $build->get_column('jobset'),
+        job => $build->get_column('job'),
+        drvPath => $build->get_column('drvpath'),
+        startTime => $build->get_column('starttime'),
+        stopTime => $build->get_column('stoptime'),
+        buildStatus => $build->get_column('buildstatus'),
+        nixName => $build->get_column('nixname'),
+        system => $build->get_column('system'),
+        homepage => $build->get_column('homepage'),
+        description => $build->get_column('description'),
+        license => $build->get_column('license'),
+        outputs => [],
+        products => [],
+        metrics => [],
+    };
+
+    for my $output ($build->buildoutputs) {
+        my $j = {
+            name => $output->name,
+            path => $output->path,
+        };
+        push @{$json->{outputs}}, $j;
+    }
+
+    for my $product ($build->buildproducts) {
+        my $j = {
+            productNr => $product->productnr,
+            type => $product->type,
+            subtype => $product->subtype,
+            fileSize => $product->filesize,
+            sha256hash => $product->sha256hash,
+            path => $product->path,
+            name => $product->name,
+            defaultPath => $product->defaultpath,
+        };
+        push @{$json->{products}}, $j;
+    }
+
+    for my $metric ($build->buildmetrics) {
+        my $j = {
+            name => $metric->name,
+            unit => $metric->unit,
+            value => 0 + $metric->value,
+        };
+        push @{$json->{metrics}}, $j;
+    }
+
+    return $json;
+}
+
 sub buildFinished {
     my ($self, $build, $dependents) = @_;
     my $event = "buildFinished";
 
-    my $cfg = $self->{config}->{runcommand};
+    my $commandsToRun = fanoutToCommands(
-    my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : ();
+        $self->{config},
+        $event,
+        $build->get_column('project'),
+        $build->get_column('jobset'),
+        $build->get_column('job')
+    );
 
-    my $tmp;
+    if (@$commandsToRun == 0) {
+        # No matching jobs, don't bother generating the JSON
+        return;
+    }
 
-    foreach my $conf (@config) {
+    my $tmp = File::Temp->new(SUFFIX => '.json');
-        next unless eventMatches($conf, $event);
+    print $tmp encode_json(makeJsonPayload($event, $build)) or die;
-        next unless configSectionMatches(
+    $ENV{"HYDRA_JSON"} = $tmp->filename;
-            $conf->{job} // "*:*:*",
-            $build->get_column('project'),
-            $build->get_column('jobset'),
-            $build->get_column('job'));
-
-        my $command = $conf->{command} // die "<runcommand> section lacks a 'command' option";
-
-        unless (defined $tmp) {
-            $tmp = File::Temp->new(SUFFIX => '.json');
-
-            my $json = {
-                event => $event,
-                build => $build->id,
-                finished => $build->get_column('finished') ? JSON::true : JSON::false,
-                timestamp => $build->get_column('timestamp'),
-                project => $build->get_column('project'),
-                jobset => $build->get_column('jobset'),
-                job => $build->get_column('job'),
-                drvPath => $build->get_column('drvpath'),
-                startTime => $build->get_column('starttime'),
-                stopTime => $build->get_column('stoptime'),
-                buildStatus => $build->get_column('buildstatus'),
-                nixName => $build->get_column('nixname'),
-                system => $build->get_column('system'),
-                homepage => $build->get_column('homepage'),
-                description => $build->get_column('description'),
-                license => $build->get_column('license'),
-                outputs => [],
-                products => [],
-                metrics => [],
-            };
-
-            for my $output ($build->buildoutputs) {
-                my $j = {
-                    name => $output->name,
-                    path => $output->path,
-                };
-                push @{$json->{outputs}}, $j;
-            }
-
-            for my $product ($build->buildproducts) {
-                my $j = {
-                    productNr => $product->productnr,
-                    type => $product->type,
-                    subtype => $product->subtype,
-                    fileSize => $product->filesize,
-                    sha256hash => $product->sha256hash,
-                    path => $product->path,
-                    name => $product->name,
-                    defaultPath => $product->defaultpath,
-                };
-                push @{$json->{products}}, $j;
-            }
-
-            for my $metric ($build->buildmetrics) {
-                my $j = {
-                    name => $metric->name,
-                    unit => $metric->unit,
-                    value => 0 + $metric->value,
-                };
-                push @{$json->{metrics}}, $j;
-            }
-
-            print $tmp encode_json($json) or die;
-        }
-
-        $ENV{"HYDRA_JSON"} = $tmp->filename;
 
+    foreach my $commandToRun (@{$commandsToRun}) {
+        my $command = $commandToRun->{command};
         system("$command") == 0
             or warn "notification command '$command' failed with exit status $?\n";
     }

t/plugins/RunCommand/basic.t

@@ -0,0 +1,53 @@
+use feature 'unicode_strings';
+use strict;
+use warnings;
+use JSON;
+use Setup;
+
+my %ctx = test_init(
+    hydra_config => q|
+    <runcommand>
+      command = cp "$HYDRA_JSON" "$HYDRA_DATA/joboutput.json"
+    </runcommand>
+|);
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+
+use Test2::V0;
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"});
+
+# Most basic test case, no parameters
+my $jobset = createBaseJobset("basic", "runcommand.nix", $ctx{jobsdir});
+
+ok(evalSucceeds($jobset), "Evaluating jobs/runcommand.nix should exit with return code 0");
+is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand.nix should result in 1 build1");
+
+(my $build) = queuedBuildsForJobset($jobset);
+
+is($build->job, "metrics", "The only job should be metrics");
+ok(runBuild($build), "Build should exit with return code 0");
+my $newbuild = $db->resultset('Builds')->find($build->id);
+is($newbuild->finished, 1, "Build should be finished.");
+is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
+
+ok(sendNotifications(), "Notifications execute successfully.");
+
+my $dat = do {
+    my $filename = $ENV{'HYDRA_DATA'} . "/joboutput.json";
+    open(my $json_fh, "<", $filename)
+        or die("Can't open \"$filename\": $!\n");
+    local $/;
+    my $json = JSON->new;
+    $json->decode(<$json_fh>)
+};
+
+subtest "Validate the file parsed and at least one field matches" => sub {
+    is($dat->{build}, $newbuild->id, "The build event matches our expected ID.");
+};
+
+done_testing;

t/plugins/RunCommand/json.t

@@ -1,7 +1,5 @@
-use feature 'unicode_strings';
 use strict;
 use warnings;
-use JSON;
 use Setup;
 
 my %ctx = test_init(
@@ -11,6 +9,9 @@ my %ctx = test_init(
     </runcommand>
 |);
 
+use Test2::V0;
+use Hydra::Plugin::RunCommand;
+
 require Hydra::Schema;
 require Hydra::Model::DB;
 
@@ -35,19 +36,12 @@ my $newbuild = $db->resultset('Builds')->find($build->id);
 is($newbuild->finished, 1, "Build should be finished.");
 is($newbuild->buildstatus, 0, "Build should have buildstatus 0.");
 
-ok(sendNotifications(), "Notifications execute successfully.");
+$build = $newbuild;
 
-my $dat = do {
+my $dat = Hydra::Plugin::RunCommand::makeJsonPayload("buildFinished", $build);
-    my $filename = $ENV{'HYDRA_DATA'} . "/joboutput.json";
-    open(my $json_fh, "<", $filename)
-        or die("Can't open \"$filename\": $!\n");
-    local $/;
-    my $json = JSON->new;
-    $json->decode(<$json_fh>)
-};
 
 subtest "Validate the top level fields match" => sub {
-    is($dat->{build}, $newbuild->id, "The build event matches our expected ID.");
+    is($dat->{build}, $build->id, "The build event matches our expected ID.");
     is($dat->{buildStatus}, 0, "The build status matches.");
     is($dat->{event}, "buildFinished", "The build event matches.");
     is($dat->{finished}, JSON::true, "The build finished.");
@@ -55,22 +49,23 @@ subtest "Validate the top level fields match" => sub {
     is($dat->{jobset}, "basic", "The jobset matches.");
     is($dat->{job}, "metrics", "The job matches.");
     is($dat->{nixName}, "my-build-product", "The nixName matches.");
-    is($dat->{system}, $newbuild->system, "The system matches.");
+    is($dat->{system}, $build->system, "The system matches.");
-    is($dat->{drvPath}, $newbuild->drvpath, "The derivation path matches.");
+    is($dat->{drvPath}, $build->drvpath, "The derivation path matches.");
-    is($dat->{timestamp}, $newbuild->timestamp, "The result has a timestamp field.");
+    is($dat->{timestamp}, $build->timestamp, "The result has a timestamp field.");
-    is($dat->{startTime}, $newbuild->starttime, "The result has a startTime field.");
+    is($dat->{startTime}, $build->starttime, "The result has a startTime field.");
-    is($dat->{stopTime}, $newbuild->stoptime, "The result has a stopTime field.");
+    is($dat->{stopTime}, $build->stoptime, "The result has a stopTime field.");
     is($dat->{homepage}, "https://github.com/NixOS/hydra", "The homepage is passed.");
     is($dat->{description}, "An example meta property.", "The description is passed.");
     is($dat->{license}, "GPL", "The license is passed.");
 };
 
+
 subtest "Validate the outputs match" => sub {
     is(scalar(@{$dat->{outputs}}), 2, "There are exactly two outputs");
 
     subtest "output: out" => sub {
         my ($output) = grep { $_->{name} eq "out" } @{$dat->{outputs}};
-        my $expectedoutput = $newbuild->buildoutputs->find({name => "out"});
+        my $expectedoutput = $build->buildoutputs->find({name => "out"});
 
         is($output->{name}, "out", "Output is named corrrectly");
         is($output->{path}, $expectedoutput->path, "The output path matches the database's path.");
@@ -78,7 +73,7 @@ subtest "Validate the outputs match" => sub {
 
     subtest "output: bin" => sub {
         my ($output) = grep { $_->{name} eq "bin" } @{$dat->{outputs}};
-        my $expectedoutput = $newbuild->buildoutputs->find({name => "bin"});
+        my $expectedoutput = $build->buildoutputs->find({name => "bin"});
 
         is($output->{name}, "bin", "Output is named corrrectly");
         is($output->{path}, $expectedoutput->path, "The output path matches the database's path.");
@@ -109,7 +104,7 @@ subtest "Validate the products match" => sub {
 
     subtest "product: out" => sub {
         my ($product) = grep { $_->{name} eq "my-build-product" } @{$dat->{products}};
-        my $expectedproduct = $newbuild->buildproducts->find({name => "my-build-product"});
+        my $expectedproduct = $build->buildproducts->find({name => "my-build-product"});
 
         is($product->{name}, "my-build-product", "The build product is named correctly.");
         is($product->{subtype}, "", "The subtype is empty.");
@@ -122,7 +117,7 @@ subtest "Validate the products match" => sub {
 
     subtest "output: bin" => sub {
         my ($product) = grep { $_->{name} eq "my-build-product-bin" } @{$dat->{products}};
-        my $expectedproduct = $newbuild->buildproducts->find({name => "my-build-product-bin"});
+        my $expectedproduct = $build->buildproducts->find({name => "my-build-product-bin"});
 
         is($product->{name}, "my-build-product-bin", "The build product is named correctly.");
         is($product->{subtype}, "bin", "The subtype matches the output name");

t/plugins/RunCommand/matcher.t

@@ -0,0 +1,177 @@
+use strict;
+use warnings;
+use Setup;
+use Test2::V0;
+use Hydra::Plugin::RunCommand;
+
+subtest "isEnabled" => sub {
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({}),
+        "",
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => {}}),
+        "",
+        "Disabled by default."
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::isEnabled({ config => { runcommand => {}}}),
+        1,
+        "Enabled if any runcommand blocks exist."
+    );
+};
+
+subtest "configSectionMatches" => sub {
+    subtest "Expected matches" => sub {
+        my @examples = (
+            # Exact match
+            ["project:jobset:job", "project", "jobset", "job"],
+
+            # One wildcard
+            ["project:jobset:*", "project", "jobset", "job"],
+            ["project:*:job", "project", "jobset", "job"],
+            ["*:jobset:job", "project", "jobset", "job"],
+
+            # Two wildcards
+            ["project:*:*", "project", "jobset", "job"],
+            ["*:*:job", "project", "jobset", "job"],
+
+            # Three wildcards
+            ["*:*:*", "project", "jobset", "job"],
+
+            # Implicit wildcards
+            ["", "project", "jobset", "job"],
+            ["project", "project", "jobset", "job"],
+            ["project:jobset", "project", "jobset", "job"],
+        );
+
+        for my $example (@examples) {
+            my ($matcher, $project, $jobset, $job) = @$example;
+
+            is(
+                Hydra::Plugin::RunCommand::configSectionMatches(
+                    $matcher, $project, $jobset, $job
+                ),
+                1,
+                "Expecting $matcher to match $project:$jobset:$job"
+            );
+        }
+    };
+
+    subtest "Fails to match" => sub {
+        my @examples = (
+            # Literal string non-matches
+            ["project:jobset:job", "project", "jobset", "nonmatch"],
+            ["project:jobset:job", "project", "nonmatch", "job"],
+            ["project:jobset:job", "nonmatch", "jobset", "job"],
+
+            # Wildcard based non-matches
+            ["*:*:job", "project", "jobset", "nonmatch"],
+            ["*:jobset:*", "project", "nonmatch", "job"],
+            ["project:*:*", "nonmatch", "jobset", "job"],
+
+            # These wildcards are NOT regular expressions
+            ["*:*:j.*", "project", "jobset", "job"],
+            [".*:.*:.*", "project", "nonmatch", "job"],
+        );
+
+        for my $example (@examples) {
+            my ($matcher, $project, $jobset, $job) = @$example;
+
+            is(
+                Hydra::Plugin::RunCommand::configSectionMatches(
+                    $matcher, $project, $jobset, $job
+                ),
+                0,
+                "Expecting $matcher to not match $project:$jobset:$job"
+            );
+        }
+
+        like(
+            dies {
+                Hydra::Plugin::RunCommand::configSectionMatches(
+                    "foo:bar:baz:tux", "foo", "bar", "baz"
+                ),
+            },
+            qr/invalid section name/,
+            "A matcher must have no more than 3 sections"
+        );
+    };
+};
+
+subtest "eventMatches" => sub {
+    # This is probably a misfeature that isn't very useful but let's test
+    # it anyway. At best this lets you make a RunCommand event not work
+    # by specifying the "events" key. Note: By testing it I'm not promising
+    # it'll keep working. In fact, I wouldn't be surprised if we chose to
+    # delete this support since RunCommand never runs on any event other
+    # than buildFinished.
+    is(
+        Hydra::Plugin::RunCommand::eventMatches({}, "buildFinished"),
+        1,
+        "An unspecified events key matches"
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::eventMatches({ events => ""}, "buildFinished"),
+        0,
+        "An empty events key does not match"
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::eventMatches({ events => "foo bar buildFinished baz"}, "buildFinished"),
+        1,
+        "An events key with multiple events does match when buildFinished is present"
+    );
+
+    is(
+        Hydra::Plugin::RunCommand::eventMatches({ events => "foo bar baz"}, "buildFinished"),
+        0,
+        "An events key with multiple events does not match when buildFinished is missing"
+    );
+};
+
+subtest "fanoutToCommands" => sub {
+    my $config = {
+        runcommand => [
+            {
+                job => "",
+                command => "foo"
+            },
+            {
+                job => "project:*:*",
+                command => "bar"
+            },
+            {
+                job => "project:jobset:nomatch",
+                command => "baz"
+            }
+        ]
+    };
+
+    is(
+        Hydra::Plugin::RunCommand::fanoutToCommands(
+            $config,
+            "buildFinished",
+            "project",
+            "jobset",
+            "job"
+        ),
+        [
+            {
+                matcher => "",
+                command => "foo"
+            },
+            {
+                matcher => "project:*:*",
+                command => "bar"
+            }
+        ],
+        "fanoutToCommands returns a command per matching job"
+    );
+};
+
+done_testing;