From e56c49333f7d31654661f0f2f3b3d86a6ccd31cb Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 8 Dec 2021 16:03:43 -0500 Subject: [PATCH] RunCommand: Add a WIP execution of dynamic commands This in-progress feature will run a dynamically generated set of buildFinished hooks, which must be nested under the `runCommandHook.*` attribute set. This implementation is not very good, with some to-dos: 1. Only run if the build succeeded 2. Verify the output is named $out and that it is an executable file (or a symlink to a file) 3. Require the jobset itself have a flag enabling the feature, since this feature can be a bit dangerous if various people of different trust levels can create the jobs. --- src/lib/Hydra/Plugin/RunCommand.pm | 33 +++++++-- t/Hydra/Plugin/RunCommand/fanout.t | 108 ++++++++++++++++++++++++++++ t/Hydra/Plugin/RunCommand/matcher.t | 40 ----------- t/jobs/runcommand-dynamic.nix | 27 +++++++ 4 files changed, 161 insertions(+), 47 deletions(-) create mode 100644 t/Hydra/Plugin/RunCommand/fanout.t create mode 100644 t/jobs/runcommand-dynamic.nix diff --git a/src/lib/Hydra/Plugin/RunCommand.pm b/src/lib/Hydra/Plugin/RunCommand.pm index 8404f280..92acc326 100644 --- a/src/lib/Hydra/Plugin/RunCommand.pm +++ b/src/lib/Hydra/Plugin/RunCommand.pm @@ -65,10 +65,11 @@ sub eventMatches { } sub fanoutToCommands { - my ($config, $event, $project, $jobset, $job) = @_; + my ($config, $event, $build) = @_; my @commands; + # Calculate all the statically defined commands to execute my $cfg = $config->{runcommand}; my @config = defined $cfg ? ref $cfg eq "ARRAY" ? @$cfg : ($cfg) : (); @@ -77,9 +78,10 @@ sub fanoutToCommands { next unless eventMatches($conf, $event); next unless configSectionMatches( $matcher, - $project, - $jobset, - $job); + $build->get_column('project'), + $build->get_column('jobset'), + $build->get_column('job') + ); if (!defined($conf->{command})) { warn " section for '$matcher' lacks a 'command' option"; @@ -92,6 +94,25 @@ sub fanoutToCommands { }) } + # Calculate all dynamically defined commands to execute + if (areDynamicCommandsEnabled($config)) { + # missing test cases: + # + # 1. is it enabled on the jobset? + # 2. what if the result is a directory? + # 3. what if the job doens't have an out? + # 4. what if the build failed? + my $job = $build->get_column('job'); + + if ($job =~ "^runCommandHook\.") { + my $out = $build->buildoutputs->find({name => "out"}); + push(@commands, { + matcher => "DynamicRunCommand($job)", + command => $out->path + }) + } + } + return \@commands; } @@ -160,9 +181,7 @@ sub buildFinished { my $commandsToRun = fanoutToCommands( $self->{config}, $event, - $build->project->get_column('name'), - $build->jobset->get_column('name'), - $build->get_column('job') + $build ); if (@$commandsToRun == 0) { diff --git a/t/Hydra/Plugin/RunCommand/fanout.t b/t/Hydra/Plugin/RunCommand/fanout.t new file mode 100644 index 00000000..d3a7b98a --- /dev/null +++ b/t/Hydra/Plugin/RunCommand/fanout.t @@ -0,0 +1,108 @@ +use strict; +use warnings; +use Setup; + +my %ctx = test_init(); + +use Test2::V0; +use Hydra::Plugin::RunCommand; + +require Hydra::Schema; +require Hydra::Model::DB; + +use Test2::V0; + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +my $project = $db->resultset('Projects')->create({name => "tests", displayname => "", owner => "root"}); + +my $jobset = createBaseJobset("basic", "runcommand-dynamic.nix", $ctx{jobsdir}); + +ok(evalSucceeds($jobset), "Evaluating jobs/runcommand-dynamic.nix should exit with return code 0"); +is(nrQueuedBuildsForJobset($jobset), 1, "Evaluating jobs/runcommand-dynamic.nix should result in 1 build1"); + +(my $build) = queuedBuildsForJobset($jobset); + +is($build->job, "runCommandHook.example", "The only job should be runCommandHook.example"); +ok(runBuild($build), "Build should exit with return code 0"); +my $newbuild = $db->resultset('Builds')->find($build->id); +is($newbuild->finished, 1, "Build should be finished."); +is($newbuild->buildstatus, 0, "Build should have buildstatus 0."); + +subtest "fanoutToCommands" => sub { + my $config = { + runcommand => [ + { + job => "", + command => "foo" + }, + { + job => "tests:*:*", + command => "bar" + }, + { + job => "tests:basic:nomatch", + command => "baz" + } + ] + }; + + is( + Hydra::Plugin::RunCommand::fanoutToCommands( + $config, + "buildFinished", + $newbuild + ), + [ + { + matcher => "", + command => "foo" + }, + { + matcher => "tests:*:*", + command => "bar" + } + ], + "fanoutToCommands returns a command per matching job" + ); +}; + +subtest "fanoutToCommandsWithDynamicRunCommandSupport" => sub { + like( + $build->buildoutputs->find({name => "out"})->path, + qr/my-build-product$/, + "The way we find the out path is reasonable" + ); + + my $config = { + dynamicruncommand => { enable => 1 }, + runcommand => [ + { + job => "tests:basic:*", + command => "baz" + } + ] + }; + + is( + Hydra::Plugin::RunCommand::fanoutToCommands( + $config, + "buildFinished", + $build + ), + [ + { + matcher => "tests:basic:*", + command => "baz" + }, + { + matcher => "DynamicRunCommand(runCommandHook.example)", + command => $build->buildoutputs->find({name => "out"})->path + } + ], + "fanoutToCommands returns a command per matching job" + ); +}; + +done_testing; diff --git a/t/Hydra/Plugin/RunCommand/matcher.t b/t/Hydra/Plugin/RunCommand/matcher.t index 9797f7e1..ca74a84c 100644 --- a/t/Hydra/Plugin/RunCommand/matcher.t +++ b/t/Hydra/Plugin/RunCommand/matcher.t @@ -249,44 +249,4 @@ subtest "eventMatches" => sub { ); }; -subtest "fanoutToCommands" => sub { - my $config = { - runcommand => [ - { - job => "", - command => "foo" - }, - { - job => "project:*:*", - command => "bar" - }, - { - job => "project:jobset:nomatch", - command => "baz" - } - ] - }; - - is( - Hydra::Plugin::RunCommand::fanoutToCommands( - $config, - "buildFinished", - "project", - "jobset", - "job" - ), - [ - { - matcher => "", - command => "foo" - }, - { - matcher => "project:*:*", - command => "bar" - } - ], - "fanoutToCommands returns a command per matching job" - ); -}; - done_testing; diff --git a/t/jobs/runcommand-dynamic.nix b/t/jobs/runcommand-dynamic.nix new file mode 100644 index 00000000..cf231f8f --- /dev/null +++ b/t/jobs/runcommand-dynamic.nix @@ -0,0 +1,27 @@ +with import ./config.nix; +{ + runCommandHook.example = mkDerivation + { + name = "my-build-product"; + builder = "/bin/sh"; + outputs = [ "out" "bin" ]; + args = [ + ( + builtins.toFile "builder.sh" '' + #! /bin/sh + + echo "$PATH" + + mkdir $bin + echo "foo" > $bin/bar + + metrics=$out/nix-support/hydra-metrics + mkdir -p "$(dirname "$metrics")" + echo "lineCoverage 18 %" >> "$metrics" + echo "maxResident 27 KiB" >> "$metrics" + '' + ) + ]; + }; + +}