From 9dfa34682dad7c1b9e1c7aae0955b5a79cce35ce Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 8 Dec 2021 09:30:26 -0500 Subject: [PATCH 1/5] RunCommand tests: move in to a subdirectory --- t/plugins/{runcommand.t => RunCommand/basic.t} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename t/plugins/{runcommand.t => RunCommand/basic.t} (100%) diff --git a/t/plugins/runcommand.t b/t/plugins/RunCommand/basic.t similarity index 100% rename from t/plugins/runcommand.t rename to t/plugins/RunCommand/basic.t From bb91f96381e984abe15c091319fe74a6c4a626e0 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 8 Dec 2021 10:24:42 -0500 Subject: [PATCH 2/5] RunCommand: test isEnabled, configSectionMatches, and eventMatches --- t/plugins/RunCommand/matcher.t | 137 +++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 t/plugins/RunCommand/matcher.t diff --git a/t/plugins/RunCommand/matcher.t b/t/plugins/RunCommand/matcher.t new file mode 100644 index 00000000..eaa9b83d --- /dev/null +++ b/t/plugins/RunCommand/matcher.t @@ -0,0 +1,137 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; +use Hydra::Plugin::RunCommand; + +subtest "isEnabled" => sub { + is( + Hydra::Plugin::RunCommand::isEnabled({}), + "", + "Disabled by default." + ); + + is( + Hydra::Plugin::RunCommand::isEnabled({ config => {}}), + "", + "Disabled by default." + ); + + is( + Hydra::Plugin::RunCommand::isEnabled({ config => { runcommand => {}}}), + 1, + "Enabled if any runcommand blocks exist." + ); +}; + +subtest "configSectionMatches" => sub { + subtest "Expected matches" => sub { + my @examples = ( + # Exact match + ["project:jobset:job", "project", "jobset", "job"], + + # One wildcard + ["project:jobset:*", "project", "jobset", "job"], + ["project:*:job", "project", "jobset", "job"], + ["*:jobset:job", "project", "jobset", "job"], + + # Two wildcards + ["project:*:*", "project", "jobset", "job"], + ["*:*:job", "project", "jobset", "job"], + + # Three wildcards + ["*:*:*", "project", "jobset", "job"], + + # Implicit wildcards + ["", "project", "jobset", "job"], + ["project", "project", "jobset", "job"], + ["project:jobset", "project", "jobset", "job"], + ); + + for my $example (@examples) { + my ($matcher, $project, $jobset, $job) = @$example; + + is( + Hydra::Plugin::RunCommand::configSectionMatches( + $matcher, $project, $jobset, $job + ), + 1, + "Expecting $matcher to match $project:$jobset:$job" + ); + } + }; + + subtest "Fails to match" => sub { + my @examples = ( + # Literal string non-matches + ["project:jobset:job", "project", "jobset", "nonmatch"], + ["project:jobset:job", "project", "nonmatch", "job"], + ["project:jobset:job", "nonmatch", "jobset", "job"], + + # Wildcard based non-matches + ["*:*:job", "project", "jobset", "nonmatch"], + ["*:jobset:*", "project", "nonmatch", "job"], + ["project:*:*", "nonmatch", "jobset", "job"], + + # These wildcards are NOT regular expressions + ["*:*:j.*", "project", "jobset", "job"], + [".*:.*:.*", "project", "nonmatch", "job"], + ); + + for my $example (@examples) { + my ($matcher, $project, $jobset, $job) = @$example; + + is( + Hydra::Plugin::RunCommand::configSectionMatches( + $matcher, $project, $jobset, $job + ), + 0, + "Expecting $matcher to not match $project:$jobset:$job" + ); + } + + like( + dies { + Hydra::Plugin::RunCommand::configSectionMatches( + "foo:bar:baz:tux", "foo", "bar", "baz" + ), + }, + qr/invalid section name/, + "A matcher must have no more than 3 sections" + ); + }; +}; + +subtest "eventMatches" => sub { + # This is probably a misfeature that isn't very useful but let's test + # it anyway. At best this lets you make a RunCommand event not work + # by specifying the "events" key. Note: By testing it I'm not promising + # it'll keep working. In fact, I wouldn't be surprised if we chose to + # delete this support since RunCommand never runs on any event other + # than buildFinished. + is( + Hydra::Plugin::RunCommand::eventMatches({}, "buildFinished"), + 1, + "An unspecified events key matches" + ); + + is( + Hydra::Plugin::RunCommand::eventMatches({ events => ""}, "buildFinished"), + 0, + "An empty events key does not match" + ); + + is( + Hydra::Plugin::RunCommand::eventMatches({ events => "foo bar buildFinished baz"}, "buildFinished"), + 1, + "An events key with multiple events does match when buildFinished is present" + ); + + is( + Hydra::Plugin::RunCommand::eventMatches({ events => "foo bar baz"}, "buildFinished"), + 0, + "An events key with multiple events does not match when buildFinished is missing" + ); +}; + +done_testing; From 2ce0ab9f5198be7cf7c2af3b2a6a2d480639d4a5 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 8 Dec 2021 11:29:36 -0500 Subject: [PATCH 3/5] RunCommand: move JSON generation to its own function --- src/lib/Hydra/Plugin/RunCommand.pm | 114 +++++++++++++++-------------- 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm index 33295e50..91a3087a 100644 --- a/src/lib/Hydra/Plugin/RunCommand.pm +++ b/src/lib/Hydra/Plugin/RunCommand.pm @@ -38,6 +38,64 @@ sub eventMatches { return 0; } +sub makeJsonPayload { + my ($event, $build) = @_; + my $json = { + event => $event, + build => $build->id, + finished => $build->get_column('finished') ? JSON::true : JSON::false, + timestamp => $build->get_column('timestamp'), + project => $build->get_column('project'), + jobset => $build->get_column('jobset'), + job => $build->get_column('job'), + drvPath => $build->get_column('drvpath'), + startTime => $build->get_column('starttime'), + stopTime => $build->get_column('stoptime'), + buildStatus => $build->get_column('buildstatus'), + nixName => $build->get_column('nixname'), + system => $build->get_column('system'), + homepage => $build->get_column('homepage'), + description => $build->get_column('description'), + license => $build->get_column('license'), + outputs => [], + products => [], + metrics => [], + }; + + for my $output ($build->buildoutputs) { + my $j = { + name => $output->name, + path => $output->path, + }; + push @{$json->{outputs}}, $j; + } + + for my $product ($build->buildproducts) { + my $j = { + productNr => $product->productnr, + type => $product->type, + subtype => $product->subtype, + fileSize => $product->filesize, + sha256hash => $product->sha256hash, + path => $product->path, + name => $product->name, + defaultPath => $product->defaultpath, + }; + push @{$json->{products}}, $j; + } + + for my $metric ($build->buildmetrics) { + my $j = { + name => $metric->name, + unit => $metric->unit, + value => 0 + $metric->value, + }; + push @{$json->{metrics}}, $j; + } + + return $json; +} + sub buildFinished { my ($self, $build, $dependents) = @_; my $event = "buildFinished"; @@ -59,61 +117,7 @@ sub buildFinished { unless (defined $tmp) { $tmp = File::Temp->new(SUFFIX => '.json'); - - my $json = { - event => $event, - build => $build->id, - finished => $build->get_column('finished') ? JSON::true : JSON::false, - timestamp => $build->get_column('timestamp'), - project => $build->get_column('project'), - jobset => $build->get_column('jobset'), - job => $build->get_column('job'), - drvPath => $build->get_column('drvpath'), - startTime => $build->get_column('starttime'), - stopTime => $build->get_column('stoptime'), - buildStatus => $build->get_column('buildstatus'), - nixName => $build->get_column('nixname'), - system => $build->get_column('system'), - homepage => $build->get_column('homepage'), - description => $build->get_column('description'), - license => $build->get_column('license'), - outputs => [], - products => [], - metrics => [], - }; - - for my $output ($build->buildoutputs) { - my $j = { - name => $output->name, - path => $output->path, - }; - push @{$json->{outputs}}, $j; - } - - for my $product ($build->buildproducts) { - my $j = { - productNr => $product->productnr, - type => $product->type, - subtype => $product->subtype, - fileSize => $product->filesize, - sha256hash => $product->sha256hash, - path => $product->path, - name => $product->name, - defaultPath => $product->defaultpath, - }; - push @{$json->{products}}, $j; - } - - for my $metric ($build->buildmetrics) { - my $j = { - name => $metric->name, - unit => $metric->unit, - value => 0 + $metric->value, - }; - push @{$json->{metrics}}, $j; - } - - print $tmp encode_json($json) or die; + print $tmp encode_json(makeJsonPayload($event, $build)) or die; } $ENV{"HYDRA_JSON"} = $tmp->filename; From b7962c3882cc8bf05be44dcfaf2b27e2ee7b61c2 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 8 Dec 2021 11:31:16 -0500 Subject: [PATCH 4/5] RunCommand: Move the json validation testing to its own test --- t/plugins/RunCommand/basic.t | 86 +---------------------- t/plugins/RunCommand/json.t | 132 +++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 85 deletions(-) create mode 100644 t/plugins/RunCommand/json.t diff --git a/t/plugins/RunCommand/basic.t b/t/plugins/RunCommand/basic.t index deb59ffe..4771e6af 100644 --- a/t/plugins/RunCommand/basic.t +++ b/t/plugins/RunCommand/basic.t @@ -46,92 +46,8 @@ my $dat = do { $json->decode(<$json_fh>) }; -subtest "Validate the top level fields match" => sub { +subtest "Validate the file parsed and at least one field matches" => sub { is($dat->{build}, $newbuild->id, "The build event matches our expected ID."); - is($dat->{buildStatus}, 0, "The build status matches."); - is($dat->{event}, "buildFinished", "The build event matches."); - is($dat->{finished}, JSON::true, "The build finished."); - is($dat->{project}, "tests", "The project matches."); - is($dat->{jobset}, "basic", "The jobset matches."); - is($dat->{job}, "metrics", "The job matches."); - is($dat->{nixName}, "my-build-product", "The nixName matches."); - is($dat->{system}, $newbuild->system, "The system matches."); - is($dat->{drvPath}, $newbuild->drvpath, "The derivation path matches."); - is($dat->{timestamp}, $newbuild->timestamp, "The result has a timestamp field."); - is($dat->{startTime}, $newbuild->starttime, "The result has a startTime field."); - is($dat->{stopTime}, $newbuild->stoptime, "The result has a stopTime field."); - is($dat->{homepage}, "https://github.com/NixOS/hydra", "The homepage is passed."); - is($dat->{description}, "An example meta property.", "The description is passed."); - is($dat->{license}, "GPL", "The license is passed."); -}; - -subtest "Validate the outputs match" => sub { - is(scalar(@{$dat->{outputs}}), 2, "There are exactly two outputs"); - - subtest "output: out" => sub { - my ($output) = grep { $_->{name} eq "out" } @{$dat->{outputs}}; - my $expectedoutput = $newbuild->buildoutputs->find({name => "out"}); - - is($output->{name}, "out", "Output is named corrrectly"); - is($output->{path}, $expectedoutput->path, "The output path matches the database's path."); - }; - - subtest "output: bin" => sub { - my ($output) = grep { $_->{name} eq "bin" } @{$dat->{outputs}}; - my $expectedoutput = $newbuild->buildoutputs->find({name => "bin"}); - - is($output->{name}, "bin", "Output is named corrrectly"); - is($output->{path}, $expectedoutput->path, "The output path matches the database's path."); - }; -}; - -subtest "Validate the metrics match" => sub { - is(scalar(@{$dat->{metrics}}), 2, "There are exactly two metrics"); - - my ($lineCoverage) = grep { $_->{name} eq "lineCoverage" } @{$dat->{metrics}}; - my ($maxResident) = grep { $_->{name} eq "maxResident" } @{$dat->{metrics}}; - - subtest "verifying the lineCoverage metric" => sub { - is($lineCoverage->{name}, "lineCoverage", "The name matches."); - is($lineCoverage->{value}, 18, "The value matches."); - is($lineCoverage->{unit}, "%", "The unit matches."); - }; - - subtest "verifying the maxResident metric" => sub { - is($maxResident->{name}, "maxResident", "The name matches."); - is($maxResident->{value}, 27, "The value matches."); - is($maxResident->{unit}, "KiB", "The unit matches."); - }; -}; - -subtest "Validate the products match" => sub { - is(scalar(@{$dat->{outputs}}), 2, "There are exactly two outputs"); - - subtest "product: out" => sub { - my ($product) = grep { $_->{name} eq "my-build-product" } @{$dat->{products}}; - my $expectedproduct = $newbuild->buildproducts->find({name => "my-build-product"}); - - is($product->{name}, "my-build-product", "The build product is named correctly."); - is($product->{subtype}, "", "The subtype is empty."); - is($product->{productNr}, $expectedproduct->productnr, "The product number matches."); - is($product->{defaultPath}, "", "The default path matches."); - is($product->{path}, $expectedproduct->path, "The path matches the output."); - is($product->{fileSize}, undef, "The fileSize is undefined for the nix-build output type."); - is($product->{sha256hash}, undef, "The sha256hash is undefined for the nix-build output type."); - }; - - subtest "output: bin" => sub { - my ($product) = grep { $_->{name} eq "my-build-product-bin" } @{$dat->{products}}; - my $expectedproduct = $newbuild->buildproducts->find({name => "my-build-product-bin"}); - - is($product->{name}, "my-build-product-bin", "The build product is named correctly."); - is($product->{subtype}, "bin", "The subtype matches the output name"); - is($product->{productNr}, $expectedproduct->productnr, "The product number matches."); - is($product->{defaultPath}, "", "The default path matches."); - is($product->{path}, $expectedproduct->path, "The path matches the output."); - is($product->{fileSize}, undef, "The fileSize is undefined for the nix-build output type."); - is($product->{sha256hash}, undef, "The sha256hash is undefined for the nix-build output type."); - }; }; done_testing; diff --git a/t/plugins/RunCommand/json.t b/t/plugins/RunCommand/json.t new file mode 100644 index 00000000..f45ab7a8 --- /dev/null +++ b/t/plugins/RunCommand/json.t @@ -0,0 +1,132 @@ +use strict; +use warnings; +use Setup; + +my %ctx = test_init( + hydra_config => q| + + command = cp "$HYDRA_JSON" "$HYDRA_DATA/joboutput.json" + +|); + +use Test2::V0; +use Hydra::Plugin::RunCommand; + +require Hydra::Schema; +require Hydra::Model::DB; + +use Test2::V0; + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"}); + +# Most basic test case, no parameters +my $jobset = createBaseJobset("basic", "runcommand.nix", $ctx{jobsdir}); + +ok(evalSucceeds($jobset), "Evaluating jobs/runcommand.nix should exit with return code 0"); +is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand.nix should result in 1 build1"); + +(my $build) = queuedBuildsForJobset($jobset); + +is($build->job, "metrics", "The only job should be metrics"); +ok(runBuild($build), "Build should exit with return code 0"); +my $newbuild = $db->resultset('Builds')->find($build->id); +is($newbuild->finished, 1, "Build should be finished."); +is($newbuild->buildstatus, 0, "Build should have buildstatus 0."); + +$build = $newbuild; + +my $dat = Hydra::Plugin::RunCommand::makeJsonPayload("buildFinished", $build); + +subtest "Validate the top level fields match" => sub { + is($dat->{build}, $build->id, "The build event matches our expected ID."); + is($dat->{buildStatus}, 0, "The build status matches."); + is($dat->{event}, "buildFinished", "The build event matches."); + is($dat->{finished}, JSON::true, "The build finished."); + is($dat->{project}, "tests", "The project matches."); + is($dat->{jobset}, "basic", "The jobset matches."); + is($dat->{job}, "metrics", "The job matches."); + is($dat->{nixName}, "my-build-product", "The nixName matches."); + is($dat->{system}, $build->system, "The system matches."); + is($dat->{drvPath}, $build->drvpath, "The derivation path matches."); + is($dat->{timestamp}, $build->timestamp, "The result has a timestamp field."); + is($dat->{startTime}, $build->starttime, "The result has a startTime field."); + is($dat->{stopTime}, $build->stoptime, "The result has a stopTime field."); + is($dat->{homepage}, "https://github.com/NixOS/hydra", "The homepage is passed."); + is($dat->{description}, "An example meta property.", "The description is passed."); + is($dat->{license}, "GPL", "The license is passed."); +}; + + +subtest "Validate the outputs match" => sub { + is(scalar(@{$dat->{outputs}}), 2, "There are exactly two outputs"); + + subtest "output: out" => sub { + my ($output) = grep { $_->{name} eq "out" } @{$dat->{outputs}}; + my $expectedoutput = $build->buildoutputs->find({name => "out"}); + + is($output->{name}, "out", "Output is named corrrectly"); + is($output->{path}, $expectedoutput->path, "The output path matches the database's path."); + }; + + subtest "output: bin" => sub { + my ($output) = grep { $_->{name} eq "bin" } @{$dat->{outputs}}; + my $expectedoutput = $build->buildoutputs->find({name => "bin"}); + + is($output->{name}, "bin", "Output is named corrrectly"); + is($output->{path}, $expectedoutput->path, "The output path matches the database's path."); + }; +}; + +subtest "Validate the metrics match" => sub { + is(scalar(@{$dat->{metrics}}), 2, "There are exactly two metrics"); + + my ($lineCoverage) = grep { $_->{name} eq "lineCoverage" } @{$dat->{metrics}}; + my ($maxResident) = grep { $_->{name} eq "maxResident" } @{$dat->{metrics}}; + + subtest "verifying the lineCoverage metric" => sub { + is($lineCoverage->{name}, "lineCoverage", "The name matches."); + is($lineCoverage->{value}, 18, "The value matches."); + is($lineCoverage->{unit}, "%", "The unit matches."); + }; + + subtest "verifying the maxResident metric" => sub { + is($maxResident->{name}, "maxResident", "The name matches."); + is($maxResident->{value}, 27, "The value matches."); + is($maxResident->{unit}, "KiB", "The unit matches."); + }; +}; + +subtest "Validate the products match" => sub { + is(scalar(@{$dat->{outputs}}), 2, "There are exactly two outputs"); + + subtest "product: out" => sub { + my ($product) = grep { $_->{name} eq "my-build-product" } @{$dat->{products}}; + my $expectedproduct = $build->buildproducts->find({name => "my-build-product"}); + + is($product->{name}, "my-build-product", "The build product is named correctly."); + is($product->{subtype}, "", "The subtype is empty."); + is($product->{productNr}, $expectedproduct->productnr, "The product number matches."); + is($product->{defaultPath}, "", "The default path matches."); + is($product->{path}, $expectedproduct->path, "The path matches the output."); + is($product->{fileSize}, undef, "The fileSize is undefined for the nix-build output type."); + is($product->{sha256hash}, undef, "The sha256hash is undefined for the nix-build output type."); + }; + + subtest "output: bin" => sub { + my ($product) = grep { $_->{name} eq "my-build-product-bin" } @{$dat->{products}}; + my $expectedproduct = $build->buildproducts->find({name => "my-build-product-bin"}); + + is($product->{name}, "my-build-product-bin", "The build product is named correctly."); + is($product->{subtype}, "bin", "The subtype matches the output name"); + is($product->{productNr}, $expectedproduct->productnr, "The product number matches."); + is($product->{defaultPath}, "", "The default path matches."); + is($product->{path}, $expectedproduct->path, "The path matches the output."); + is($product->{fileSize}, undef, "The fileSize is undefined for the nix-build output type."); + is($product->{sha256hash}, undef, "The sha256hash is undefined for the nix-build output type."); + }; +}; + +done_testing; From 26b197ea62ce14805a01af8e8494395f620651a7 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 8 Dec 2021 14:36:27 -0500 Subject: [PATCH 5/5] RunCommand: calculate all the commands to run against before starting --- src/lib/Hydra/Plugin/RunCommand.pm | 66 +++++++++++++++++++++--------- t/plugins/RunCommand/matcher.t | 40 ++++++++++++++++++ 2 files changed, 87 insertions(+), 19 deletions(-) diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm index 91a3087a..b4abc4dd 100644 --- a/src/lib/Hydra/Plugin/RunCommand.pm +++ b/src/lib/Hydra/Plugin/RunCommand.pm @@ -38,6 +38,37 @@ sub eventMatches { return 0; } +sub fanoutToCommands { + my ($config, $event, $project, $jobset, $job) = @_; + + my @commands; + + my $cfg = $config->{runcommand}; + my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : (); + + foreach my $conf (@config) { + my $matcher = $conf->{job} // "*:*:*"; + next unless eventMatches($conf, $event); + next unless configSectionMatches( + $matcher, + $project, + $jobset, + $job); + + if (!defined($conf->{command})) { + warn " section for '$matcher' lacks a 'command' option"; + next; + } + + push(@commands, { + matcher => $matcher, + command => $conf->{command}, + }) + } + + return \@commands; +} + sub makeJsonPayload { my ($event, $build) = @_; my $json = { @@ -100,28 +131,25 @@ sub buildFinished { my ($self, $build, $dependents) = @_; my $event = "buildFinished"; - my $cfg = $self->{config}->{runcommand}; - my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : (); + my $commandsToRun = fanoutToCommands( + $self->{config}, + $event, + $build->get_column('project'), + $build->get_column('jobset'), + $build->get_column('job') + ); - my $tmp; + if (@$commandsToRun == 0) { + # No matching jobs, don't bother generating the JSON + return; + } - foreach my $conf (@config) { - next unless eventMatches($conf, $event); - next unless configSectionMatches( - $conf->{job} // "*:*:*", - $build->get_column('project'), - $build->get_column('jobset'), - $build->get_column('job')); - - my $command = $conf->{command} // die " section lacks a 'command' option"; - - unless (defined $tmp) { - $tmp = File::Temp->new(SUFFIX => '.json'); - print $tmp encode_json(makeJsonPayload($event, $build)) or die; - } - - $ENV{"HYDRA_JSON"} = $tmp->filename; + my $tmp = File::Temp->new(SUFFIX => '.json'); + print $tmp encode_json(makeJsonPayload($event, $build)) or die; + $ENV{"HYDRA_JSON"} = $tmp->filename; + foreach my $commandToRun (@{$commandsToRun}) { + my $command = $commandToRun->{command}; system("$command") == 0 or warn "notification command '$command' failed with exit status $?\n"; } diff --git a/t/plugins/RunCommand/matcher.t b/t/plugins/RunCommand/matcher.t index eaa9b83d..bc40ba77 100644 --- a/t/plugins/RunCommand/matcher.t +++ b/t/plugins/RunCommand/matcher.t @@ -134,4 +134,44 @@ subtest "eventMatches" => sub { ); }; +subtest "fanoutToCommands" => sub { + my $config = { + runcommand => [ + { + job => "", + command => "foo" + }, + { + job => "project:*:*", + command => "bar" + }, + { + job => "project:jobset:nomatch", + command => "baz" + } + ] + }; + + is( + Hydra::Plugin::RunCommand::fanoutToCommands( + $config, + "buildFinished", + "project", + "jobset", + "job" + ), + [ + { + matcher => "", + command => "foo" + }, + { + matcher => "project:*:*", + command => "bar" + } + ], + "fanoutToCommands returns a command per matching job" + ); +}; + done_testing;