diff --git a/doc/manual/src/plugins/RunCommand.md b/doc/manual/src/plugins/RunCommand.md index b186be80..652a171e 100644 --- a/doc/manual/src/plugins/RunCommand.md +++ b/doc/manual/src/plugins/RunCommand.md @@ -33,8 +33,9 @@ Command to run. Can use the `$HYDRA_JSON` environment variable to access informa ### Dynamic Commands -Hydra can optionally run RunCommand hooks defined dynamically by the jobset. -This must be turned on explicitly in the `hydra.conf` and per jobset. +Hydra can optionally run RunCommand hooks defined dynamically by the jobset. In +order to enable dynamic commands, you must enable this feature in your +`hydra.conf`, *as well as* in the parent project and jobset configuration. #### Behavior diff --git a/hydra-api.yaml b/hydra-api.yaml index 0fe0a130..ce7e0f9a 100644 --- a/hydra-api.yaml +++ b/hydra-api.yaml @@ -178,6 +178,9 @@ paths: enabled: description: when set to true the project gets scheduled for evaluation type: boolean + enable_dynamic_run_command: + description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + type: boolean visible: description: when set to true the project is displayed in the web interface type: boolean diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index a2d48597..eeb4232a 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -261,6 +261,14 @@ sub updateJobset { my $checkinterval = int(trim($c->stash->{params}->{checkinterval})); + my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0; + if ($enable_dynamic_run_command + && !($c->config->{dynamicruncommand}->{enable} + && $jobset->project->enable_dynamic_run_command)) + { + badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project."); + } + $jobset->update( { name => $jobsetName , description => trim($c->stash->{params}->{"description"}) @@ -268,7 +276,7 @@ sub updateJobset { , nixexprinput => $nixExprInput , enabled => $enabled , enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0 - , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0 + , enable_dynamic_run_command => $enable_dynamic_run_command , emailoverride => trim($c->stash->{params}->{emailoverride}) || "" , hidden => defined $c->stash->{params}->{visible} ? 0 : 1 , keepnr => int(trim($c->stash->{params}->{keepnr} // "0")) diff --git a/src/lib/Hydra/Controller/Project.pm b/src/lib/Hydra/Controller/Project.pm index 98a8a6eb..1141de4a 100644 --- a/src/lib/Hydra/Controller/Project.pm +++ b/src/lib/Hydra/Controller/Project.pm @@ -149,6 +149,11 @@ sub updateProject { my $displayName = trim $c->stash->{params}->{displayname}; error($c, "You must specify a display name.") if $displayName eq ""; + my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0; + if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) { + badRequest($c, "Dynamic RunCommand is not enabled by the server."); + } + $project->update( { name => $projectName , displayname => $displayName @@ -157,7 +162,7 @@ sub updateProject { , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0 , hidden => defined $c->stash->{params}->{visible} ? 0 : 1 , owner => $owner - , enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0 + , enable_dynamic_run_command => $enable_dynamic_run_command , declfile => trim($c->stash->{params}->{declarative}->{file}) , decltype => trim($c->stash->{params}->{declarative}->{type}) , declvalue => trim($c->stash->{params}->{declarative}->{value}) diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index f38737d3..9e3ddfd2 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils; our @ISA = qw(Exporter); our @EXPORT = qw( + validateDeclarativeJobset + createJobsetInputsRowAndData updateDeclarativeJobset handleDeclarativeJobsetBuild handleDeclarativeJobsetJson ); -sub updateDeclarativeJobset { - my ($db, $project, $jobsetName, $declSpec) = @_; +sub validateDeclarativeJobset { + my ($config, $project, $jobsetName, $declSpec) = @_; my @allowed_keys = qw( enabled @@ -62,16 +64,39 @@ sub updateDeclarativeJobset { } } + my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0; + if ($enable_dynamic_run_command + && !($config->{dynamicruncommand}->{enable} + && $project->{enable_dynamic_run_command})) + { + die "Dynamic RunCommand is not enabled by the server or the parent project."; + } + + return %update; +} + +sub createJobsetInputsRowAndData { + my ($name, $declSpec) = @_; + my $data = $declSpec->{"inputs"}->{$name}; + my $row = { + name => $name, + type => $data->{type} + }; + $row->{emailresponsible} = $data->{emailresponsible} // 0; + + return ($row, $data); +} + +sub updateDeclarativeJobset { + my ($config, $db, $project, $jobsetName, $declSpec) = @_; + + my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec); + $db->txn_do(sub { my $jobset = $project->jobsets->update_or_create(\%update); $jobset->jobsetinputs->delete; foreach my $name (keys %{$declSpec->{"inputs"}}) { - my $data = $declSpec->{"inputs"}->{$name}; - my $row = { - name => $name, - type => $data->{type} - }; - $row->{emailresponsible} = $data->{emailresponsible} // 0; + my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec); my $input = $jobset->jobsetinputs->create($row); $input->jobsetinputalts->create({altnr => 0, value => $data->{value}}); } @@ -82,6 +107,7 @@ sub updateDeclarativeJobset { sub handleDeclarativeJobsetJson { my ($db, $project, $declSpec) = @_; + my $config = getHydraConfig(); $db->txn_do(sub { my @kept = keys %$declSpec; push @kept, ".jobsets"; @@ -89,7 +115,7 @@ sub handleDeclarativeJobsetJson { foreach my $jobsetName (keys %$declSpec) { my $spec = $declSpec->{$jobsetName}; eval { - updateDeclarativeJobset($db, $project, $jobsetName, $spec); + updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec); 1; } or do { print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n"; diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt index 40da8f61..61e3636f 100644 --- a/src/root/edit-jobset.tt +++ b/src/root/edit-jobset.tt @@ -160,7 +160,15 @@
- +
diff --git a/src/root/edit-project.tt b/src/root/edit-project.tt index 4b99f4ab..bb850e5c 100644 --- a/src/root/edit-project.tt +++ b/src/root/edit-project.tt @@ -56,7 +56,13 @@
- +
diff --git a/src/root/jobset.tt b/src/root/jobset.tt index 3d6ca6ae..56abdb50 100644 --- a/src/root/jobset.tt +++ b/src/root/jobset.tt @@ -162,7 +162,7 @@ Enable Dynamic RunCommand Hooks: - [% jobset.enable_dynamic_run_command ? "Yes" : "No" %] + [% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? jobset.enable_dynamic_run_command ? "Yes" : "No (not enabled by jobset)" : "No (not enabled by project)" : "No (not enabled by server)" %] [% IF emailNotification %] diff --git a/src/root/project.tt b/src/root/project.tt index f5a51e96..5e8ec0c8 100644 --- a/src/root/project.tt +++ b/src/root/project.tt @@ -94,7 +94,7 @@ Enable Dynamic RunCommand Hooks: - [% project.enable_dynamic_run_command ? "Yes" : "No" %] + [% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? "Yes" : "No (not enabled by project)" : "No (not enabled by server)" %] diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index de437ecd..a9bd7355 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -617,7 +617,7 @@ sub checkJobsetWrapped { } else { # Update the jobset with the spec's inputs, and the continue # evaluating the .jobsets jobset. - updateDeclarativeJobset($db, $project, ".jobsets", $declSpec); + updateDeclarativeJobset($config, $db, $project, ".jobsets", $declSpec); $jobset->discard_changes; $inputInfo->{"declInput"} = [ $declInput ]; $inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ]; diff --git a/t/Helper/AddBuilds/dynamic-disabled.t b/t/Helper/AddBuilds/dynamic-disabled.t new file mode 100644 index 00000000..0507b03e --- /dev/null +++ b/t/Helper/AddBuilds/dynamic-disabled.t @@ -0,0 +1,85 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; + +require Catalyst::Test; +use HTTP::Request::Common qw(POST PUT GET DELETE); +use JSON::MaybeXS qw(decode_json encode_json); +use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset); +use Hydra::Helper::Nix qw(getHydraConfig); + +my $ctx = test_context(); + +sub makeJobsetSpec { + my ($dynamic) = @_; + + return { + enabled => 2, + enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef, + visible => JSON::MaybeXS::true, + name => "job", + type => 1, + description => "test jobset", + flake => "github:nixos/nix", + checkinterval => 0, + schedulingshares => 100, + keepnr => 3 + }; +}; + +subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub { + my $config = getHydraConfig(); + + subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub { + like( + dies { + validateDeclarativeJobset( + $config, + { enable_dynamic_run_command => 1 }, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::true), + ), + }, + qr/Dynamic RunCommand is not enabled/, + ); + }; + + subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + { enable_dynamic_run_command => 1 }, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::false) + ), + ); + }; + + subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub { + like( + dies { + validateDeclarativeJobset( + $config, + { enable_dynamic_run_command => 0 }, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::true), + ), + }, + qr/Dynamic RunCommand is not enabled/, + ); + }; + + subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + { enable_dynamic_run_command => 0 }, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::false) + ), + ); + }; +}; + +done_testing; diff --git a/t/Helper/AddBuilds/dynamic-enabled.t b/t/Helper/AddBuilds/dynamic-enabled.t new file mode 100644 index 00000000..d2f5a386 --- /dev/null +++ b/t/Helper/AddBuilds/dynamic-enabled.t @@ -0,0 +1,88 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; + +require Catalyst::Test; +use HTTP::Request::Common qw(POST PUT GET DELETE); +use JSON::MaybeXS qw(decode_json encode_json); +use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset); +use Hydra::Helper::Nix qw(getHydraConfig); + +my $ctx = test_context( + hydra_config => q| + + enable = 1 + + | +); + +sub makeJobsetSpec { + my ($dynamic) = @_; + + return { + enabled => 2, + enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef, + visible => JSON::MaybeXS::true, + name => "job", + type => 1, + description => "test jobset", + flake => "github:nixos/nix", + checkinterval => 0, + schedulingshares => 100, + keepnr => 3 + }; +}; + +subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub { + my $config = getHydraConfig(); + + subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + { enable_dynamic_run_command => 1 }, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::true) + ), + ); + }; + + subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + { enable_dynamic_run_command => 1 }, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::false) + ), + ); + }; + + subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub { + like( + dies { + validateDeclarativeJobset( + $config, + { enable_dynamic_run_command => 0 }, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::true), + ), + }, + qr/Dynamic RunCommand is not enabled/, + ); + }; + + subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub { + ok( + validateDeclarativeJobset( + $config, + { enable_dynamic_run_command => 0 }, + "test-jobset", + makeJobsetSpec(JSON::MaybeXS::false) + ), + ); + }; +}; + +done_testing; diff --git a/t/Hydra/Plugin/RunCommand/dynamic-disabled.t b/t/Hydra/Plugin/RunCommand/dynamic-disabled.t new file mode 100644 index 00000000..ad2e9a4b --- /dev/null +++ b/t/Hydra/Plugin/RunCommand/dynamic-disabled.t @@ -0,0 +1,110 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; + +require Catalyst::Test; +use HTTP::Request::Common qw(POST PUT GET DELETE); +use JSON::MaybeXS qw(decode_json encode_json); + +my $ctx = test_context(); +Catalyst::Test->import('Hydra'); + +# Create a user to log in to +my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' }); +$user->setPassword('foobar'); +$user->userroles->update_or_create({ role => 'admin' }); + +subtest "can't enable dynamic RunCommand when disabled by server" => sub { + my $builds = $ctx->makeAndEvaluateJobset( + expression => "runcommand-dynamic.nix", + build => 1 + ); + + my $build = $builds->{"runCommandHook.example"}; + my $project = $build->project; + my $project_name = $project->name; + my $jobset = $build->jobset; + my $jobset_name = $jobset->name; + + is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default"); + is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default"); + + my $req = request(POST '/login', + Referer => 'http://localhost/', + Content => { + username => 'alice', + password => 'foobar' + } + ); + is($req->code, 302, "logged in successfully"); + my $cookie = $req->header("set-cookie"); + + subtest "can't enable dynamic RunCommand on project" => sub { + my $projectresponse = request(GET "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + + my $projectjson = decode_json($projectresponse->content); + $projectjson->{enable_dynamic_run_command} = 1; + + my $projectupdate = request(PUT "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + Content => encode_json($projectjson) + ); + + $projectresponse = request(GET "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + $projectjson = decode_json($projectresponse->content); + + is($projectupdate->code, 400); + like( + $projectupdate->content, + qr/Dynamic RunCommand is not/, + "failed to change enable_dynamic_run_command, not any other error" + ); + is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::false); + }; + + subtest "can't enable dynamic RunCommand on jobset" => sub { + my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + + my $jobsetjson = decode_json($jobsetresponse->content); + $jobsetjson->{enable_dynamic_run_command} = 1; + + my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + Content => encode_json($jobsetjson) + ); + + $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + $jobsetjson = decode_json($jobsetresponse->content); + + is($jobsetupdate->code, 400); + like( + $jobsetupdate->content, + qr/Dynamic RunCommand is not/, + "failed to change enable_dynamic_run_command, not any other error" + ); + is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::false); + }; +}; + +done_testing; diff --git a/t/Hydra/Plugin/RunCommand/dynamic-enabled.t b/t/Hydra/Plugin/RunCommand/dynamic-enabled.t new file mode 100644 index 00000000..68c6d593 --- /dev/null +++ b/t/Hydra/Plugin/RunCommand/dynamic-enabled.t @@ -0,0 +1,106 @@ +use strict; +use warnings; +use Setup; +use Test2::V0; + +require Catalyst::Test; +use HTTP::Request::Common qw(POST PUT GET DELETE); +use JSON::MaybeXS qw(decode_json encode_json); + +my $ctx = test_context( + hydra_config => q| + + enable = 1 + + | +); +Catalyst::Test->import('Hydra'); + +# Create a user to log in to +my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' }); +$user->setPassword('foobar'); +$user->userroles->update_or_create({ role => 'admin' }); + +subtest "can enable dynamic RunCommand when enabled by server" => sub { + my $builds = $ctx->makeAndEvaluateJobset( + expression => "runcommand-dynamic.nix", + build => 1 + ); + + my $build = $builds->{"runCommandHook.example"}; + my $project = $build->project; + my $project_name = $project->name; + my $jobset = $build->jobset; + my $jobset_name = $jobset->name; + + is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default"); + is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default"); + + my $req = request(POST '/login', + Referer => 'http://localhost/', + Content => { + username => 'alice', + password => 'foobar' + } + ); + is($req->code, 302, "logged in successfully"); + my $cookie = $req->header("set-cookie"); + + subtest "can enable dynamic RunCommand on project" => sub { + my $projectresponse = request(GET "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + + my $projectjson = decode_json($projectresponse->content); + $projectjson->{enable_dynamic_run_command} = 1; + + my $projectupdate = request(PUT "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + Content => encode_json($projectjson) + ); + + $projectresponse = request(GET "/project/$project_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + $projectjson = decode_json($projectresponse->content); + + is($projectupdate->code, 200); + is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::true); + }; + + subtest "can enable dynamic RunCommand on jobset" => sub { + my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + + my $jobsetjson = decode_json($jobsetresponse->content); + $jobsetjson->{enable_dynamic_run_command} = 1; + + my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + Content => encode_json($jobsetjson) + ); + + $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name", + Accept => 'application/json', + Content_Type => 'application/json', + Cookie => $cookie, + ); + $jobsetjson = decode_json($jobsetresponse->content); + + is($jobsetupdate->code, 200); + is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::true); + }; +}; + +done_testing;