Merge pull request #8 from DeterminateSystems/runcommand/dynamic-guarding

Runcommand/dynamic guarding
This commit is contained in:
Graham Christensen 2022-02-11 15:05:15 -05:00 committed by GitHub
commit 4c1daacdf1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 464 additions and 18 deletions

View file

@ -33,8 +33,9 @@ Command to run. Can use the `$HYDRA_JSON` environment variable to access informa
### Dynamic Commands ### Dynamic Commands
Hydra can optionally run RunCommand hooks defined dynamically by the jobset. Hydra can optionally run RunCommand hooks defined dynamically by the jobset. In
This must be turned on explicitly in the `hydra.conf` and per jobset. order to enable dynamic commands, you must enable this feature in your
`hydra.conf`, *as well as* in the parent project and jobset configuration.
#### Behavior #### Behavior

View file

@ -178,6 +178,9 @@ paths:
enabled: enabled:
description: when set to true the project gets scheduled for evaluation description: when set to true the project gets scheduled for evaluation
type: boolean type: boolean
enable_dynamic_run_command:
description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
type: boolean
visible: visible:
description: when set to true the project is displayed in the web interface description: when set to true the project is displayed in the web interface
type: boolean type: boolean

View file

@ -261,6 +261,14 @@ sub updateJobset {
my $checkinterval = int(trim($c->stash->{params}->{checkinterval})); my $checkinterval = int(trim($c->stash->{params}->{checkinterval}));
my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($c->config->{dynamicruncommand}->{enable}
&& $jobset->project->enable_dynamic_run_command))
{
badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project.");
}
$jobset->update( $jobset->update(
{ name => $jobsetName { name => $jobsetName
, description => trim($c->stash->{params}->{"description"}) , description => trim($c->stash->{params}->{"description"})
@ -268,7 +276,7 @@ sub updateJobset {
, nixexprinput => $nixExprInput , nixexprinput => $nixExprInput
, enabled => $enabled , enabled => $enabled
, enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0 , enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
, enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0 , enable_dynamic_run_command => $enable_dynamic_run_command
, emailoverride => trim($c->stash->{params}->{emailoverride}) || "" , emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1 , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, keepnr => int(trim($c->stash->{params}->{keepnr} // "0")) , keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))

View file

@ -149,6 +149,11 @@ sub updateProject {
my $displayName = trim $c->stash->{params}->{displayname}; my $displayName = trim $c->stash->{params}->{displayname};
error($c, "You must specify a display name.") if $displayName eq ""; error($c, "You must specify a display name.") if $displayName eq "";
my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) {
badRequest($c, "Dynamic RunCommand is not enabled by the server.");
}
$project->update( $project->update(
{ name => $projectName { name => $projectName
, displayname => $displayName , displayname => $displayName
@ -157,7 +162,7 @@ sub updateProject {
, enabled => defined $c->stash->{params}->{enabled} ? 1 : 0 , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1 , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, owner => $owner , owner => $owner
, enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0 , enable_dynamic_run_command => $enable_dynamic_run_command
, declfile => trim($c->stash->{params}->{declarative}->{file}) , declfile => trim($c->stash->{params}->{declarative}->{file})
, decltype => trim($c->stash->{params}->{declarative}->{type}) , decltype => trim($c->stash->{params}->{declarative}->{type})
, declvalue => trim($c->stash->{params}->{declarative}->{value}) , declvalue => trim($c->stash->{params}->{declarative}->{value})

View file

@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils;
our @ISA = qw(Exporter); our @ISA = qw(Exporter);
our @EXPORT = qw( our @EXPORT = qw(
validateDeclarativeJobset
createJobsetInputsRowAndData
updateDeclarativeJobset updateDeclarativeJobset
handleDeclarativeJobsetBuild handleDeclarativeJobsetBuild
handleDeclarativeJobsetJson handleDeclarativeJobsetJson
); );
sub updateDeclarativeJobset { sub validateDeclarativeJobset {
my ($db, $project, $jobsetName, $declSpec) = @_; my ($config, $project, $jobsetName, $declSpec) = @_;
my @allowed_keys = qw( my @allowed_keys = qw(
enabled enabled
@ -62,16 +64,39 @@ sub updateDeclarativeJobset {
} }
} }
my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($config->{dynamicruncommand}->{enable}
&& $project->{enable_dynamic_run_command}))
{
die "Dynamic RunCommand is not enabled by the server or the parent project.";
}
return %update;
}
sub createJobsetInputsRowAndData {
my ($name, $declSpec) = @_;
my $data = $declSpec->{"inputs"}->{$name};
my $row = {
name => $name,
type => $data->{type}
};
$row->{emailresponsible} = $data->{emailresponsible} // 0;
return ($row, $data);
}
sub updateDeclarativeJobset {
my ($config, $db, $project, $jobsetName, $declSpec) = @_;
my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec);
$db->txn_do(sub { $db->txn_do(sub {
my $jobset = $project->jobsets->update_or_create(\%update); my $jobset = $project->jobsets->update_or_create(\%update);
$jobset->jobsetinputs->delete; $jobset->jobsetinputs->delete;
foreach my $name (keys %{$declSpec->{"inputs"}}) { foreach my $name (keys %{$declSpec->{"inputs"}}) {
my $data = $declSpec->{"inputs"}->{$name}; my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec);
my $row = {
name => $name,
type => $data->{type}
};
$row->{emailresponsible} = $data->{emailresponsible} // 0;
my $input = $jobset->jobsetinputs->create($row); my $input = $jobset->jobsetinputs->create($row);
$input->jobsetinputalts->create({altnr => 0, value => $data->{value}}); $input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
} }
@ -82,6 +107,7 @@ sub updateDeclarativeJobset {
sub handleDeclarativeJobsetJson { sub handleDeclarativeJobsetJson {
my ($db, $project, $declSpec) = @_; my ($db, $project, $declSpec) = @_;
my $config = getHydraConfig();
$db->txn_do(sub { $db->txn_do(sub {
my @kept = keys %$declSpec; my @kept = keys %$declSpec;
push @kept, ".jobsets"; push @kept, ".jobsets";
@ -89,7 +115,7 @@ sub handleDeclarativeJobsetJson {
foreach my $jobsetName (keys %$declSpec) { foreach my $jobsetName (keys %$declSpec) {
my $spec = $declSpec->{$jobsetName}; my $spec = $declSpec->{$jobsetName};
eval { eval {
updateDeclarativeJobset($db, $project, $jobsetName, $spec); updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec);
1; 1;
} or do { } or do {
print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n"; print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";

View file

@ -160,7 +160,15 @@
<div class="form-group row"> <div class="form-group row">
<label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label> <label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label>
<div class="col-sm-9"> <div class="col-sm-9">
<input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/> <input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command"
[% IF !c.config.dynamicruncommand.enable %]
title="The server has not enabled dynamic RunCommands" disabled
[% ELSIF !project.enable_dynamic_run_command %]
title="The parent project has not enabled dynamic RunCommands" disabled
[% ELSIF jobset.enable_dynamic_run_command %]
checked
[% END %]
/>
</div> </div>
</div> </div>

View file

@ -56,7 +56,13 @@
<div class="form-group row"> <div class="form-group row">
<label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label> <label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label>
<div class="col-sm-9"> <div class="col-sm-9">
<input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/> <input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command"
[% IF !c.config.dynamicruncommand.enable %]
title="The server has not enabled dynamic RunCommands" disabled
[% ELSIF project.enable_dynamic_run_command %]
checked
[% END %]
/>
</div> </div>
</div> </div>

View file

@ -162,7 +162,7 @@
</tr> </tr>
<tr> <tr>
<th>Enable Dynamic RunCommand Hooks:</th> <th>Enable Dynamic RunCommand Hooks:</th>
<td>[% jobset.enable_dynamic_run_command ? "Yes" : "No" %]</td> <td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? jobset.enable_dynamic_run_command ? "Yes" : "No (not enabled by jobset)" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
</tr> </tr>
[% IF emailNotification %] [% IF emailNotification %]
<tr> <tr>

View file

@ -94,7 +94,7 @@
</tr> </tr>
<tr> <tr>
<th>Enable Dynamic RunCommand Hooks:</th> <th>Enable Dynamic RunCommand Hooks:</th>
<td>[% project.enable_dynamic_run_command ? "Yes" : "No" %]</td> <td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? "Yes" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
</tr> </tr>
</table> </table>
</div> </div>

View file

@ -617,7 +617,7 @@ sub checkJobsetWrapped {
} else { } else {
# Update the jobset with the spec's inputs, and the continue # Update the jobset with the spec's inputs, and the continue
# evaluating the .jobsets jobset. # evaluating the .jobsets jobset.
updateDeclarativeJobset($db, $project, ".jobsets", $declSpec); updateDeclarativeJobset($config, $db, $project, ".jobsets", $declSpec);
$jobset->discard_changes; $jobset->discard_changes;
$inputInfo->{"declInput"} = [ $declInput ]; $inputInfo->{"declInput"} = [ $declInput ];
$inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ]; $inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ];

View file

@ -0,0 +1,85 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
use Hydra::Helper::Nix qw(getHydraConfig);
my $ctx = test_context();
sub makeJobsetSpec {
my ($dynamic) = @_;
return {
enabled => 2,
enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
visible => JSON::MaybeXS::true,
name => "job",
type => 1,
description => "test jobset",
flake => "github:nixos/nix",
checkinterval => 0,
schedulingshares => 100,
keepnr => 3
};
};
subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub {
my $config = getHydraConfig();
subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
like(
dies {
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true),
),
},
qr/Dynamic RunCommand is not enabled/,
);
};
subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
like(
dies {
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true),
),
},
qr/Dynamic RunCommand is not enabled/,
);
};
subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
};
done_testing;

View file

@ -0,0 +1,88 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
use Hydra::Helper::Nix qw(getHydraConfig);
my $ctx = test_context(
hydra_config => q|
<dynamicruncommand>
enable = 1
</dynamicruncommand>
|
);
sub makeJobsetSpec {
my ($dynamic) = @_;
return {
enabled => 2,
enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
visible => JSON::MaybeXS::true,
name => "job",
type => 1,
description => "test jobset",
flake => "github:nixos/nix",
checkinterval => 0,
schedulingshares => 100,
keepnr => 3
};
};
subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub {
my $config = getHydraConfig();
subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true)
),
);
};
subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
like(
dies {
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true),
),
},
qr/Dynamic RunCommand is not enabled/,
);
};
subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
};
done_testing;

View file

@ -0,0 +1,110 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
my $ctx = test_context();
Catalyst::Test->import('Hydra');
# Create a user to log in to
my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
$user->setPassword('foobar');
$user->userroles->update_or_create({ role => 'admin' });
subtest "can't enable dynamic RunCommand when disabled by server" => sub {
my $builds = $ctx->makeAndEvaluateJobset(
expression => "runcommand-dynamic.nix",
build => 1
);
my $build = $builds->{"runCommandHook.example"};
my $project = $build->project;
my $project_name = $project->name;
my $jobset = $build->jobset;
my $jobset_name = $jobset->name;
is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
my $req = request(POST '/login',
Referer => 'http://localhost/',
Content => {
username => 'alice',
password => 'foobar'
}
);
is($req->code, 302, "logged in successfully");
my $cookie = $req->header("set-cookie");
subtest "can't enable dynamic RunCommand on project" => sub {
my $projectresponse = request(GET "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
my $projectjson = decode_json($projectresponse->content);
$projectjson->{enable_dynamic_run_command} = 1;
my $projectupdate = request(PUT "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
Content => encode_json($projectjson)
);
$projectresponse = request(GET "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
$projectjson = decode_json($projectresponse->content);
is($projectupdate->code, 400);
like(
$projectupdate->content,
qr/Dynamic RunCommand is not/,
"failed to change enable_dynamic_run_command, not any other error"
);
is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
};
subtest "can't enable dynamic RunCommand on jobset" => sub {
my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
my $jobsetjson = decode_json($jobsetresponse->content);
$jobsetjson->{enable_dynamic_run_command} = 1;
my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
Content => encode_json($jobsetjson)
);
$jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
$jobsetjson = decode_json($jobsetresponse->content);
is($jobsetupdate->code, 400);
like(
$jobsetupdate->content,
qr/Dynamic RunCommand is not/,
"failed to change enable_dynamic_run_command, not any other error"
);
is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
};
};
done_testing;

View file

@ -0,0 +1,106 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
my $ctx = test_context(
hydra_config => q|
<dynamicruncommand>
enable = 1
</dynamicruncommand>
|
);
Catalyst::Test->import('Hydra');
# Create a user to log in to
my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
$user->setPassword('foobar');
$user->userroles->update_or_create({ role => 'admin' });
subtest "can enable dynamic RunCommand when enabled by server" => sub {
my $builds = $ctx->makeAndEvaluateJobset(
expression => "runcommand-dynamic.nix",
build => 1
);
my $build = $builds->{"runCommandHook.example"};
my $project = $build->project;
my $project_name = $project->name;
my $jobset = $build->jobset;
my $jobset_name = $jobset->name;
is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
my $req = request(POST '/login',
Referer => 'http://localhost/',
Content => {
username => 'alice',
password => 'foobar'
}
);
is($req->code, 302, "logged in successfully");
my $cookie = $req->header("set-cookie");
subtest "can enable dynamic RunCommand on project" => sub {
my $projectresponse = request(GET "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
my $projectjson = decode_json($projectresponse->content);
$projectjson->{enable_dynamic_run_command} = 1;
my $projectupdate = request(PUT "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
Content => encode_json($projectjson)
);
$projectresponse = request(GET "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
$projectjson = decode_json($projectresponse->content);
is($projectupdate->code, 200);
is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
};
subtest "can enable dynamic RunCommand on jobset" => sub {
my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
my $jobsetjson = decode_json($jobsetresponse->content);
$jobsetjson->{enable_dynamic_run_command} = 1;
my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
Content => encode_json($jobsetjson)
);
$jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
$jobsetjson = decode_json($jobsetresponse->content);
is($jobsetupdate->code, 200);
is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
};
};
done_testing;