ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Merge pull request #8 from DeterminateSystems/runcommand/dynamic-guarding

Browse source 
Runcommand/dynamic guarding
This commit is contained in:
Graham Christensen 2022-02-11 15:05:15 -05:00 committed by GitHub
parent d8b56f022d 8c3122cacd
commit 4c1daacdf1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 464 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
doc/manual/src/plugins/RunCommand.md
View file

@ -33,8 +33,9 @@ Command to run. Can use the `$HYDRA_JSON` environment variable to access informa
### Dynamic Commands
Hydra can optionally run RunCommand hooks defined dynamically by the jobset.
This must be turned on explicitly in the `hydra.conf` and per jobset.
Hydra can optionally run RunCommand hooks defined dynamically by the jobset. In
order to enable dynamic commands, you must enable this feature in your
`hydra.conf`, *as well as* in the parent project and jobset configuration.
#### Behavior

3
hydra-api.yaml
View file

@ -178,6 +178,9 @@ paths:
enabled:
description: when set to true the project gets scheduled for evaluation
type: boolean
enable_dynamic_run_command:
description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand.
type: boolean
visible:
description: when set to true the project is displayed in the web interface
type: boolean

10
src/lib/Hydra/Controller/Jobset.pm
View file

@ -261,6 +261,14 @@ sub updateJobset {
my $checkinterval = int(trim($c->stash->{params}->{checkinterval}));
my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($c->config->{dynamicruncommand}->{enable}
&& $jobset->project->enable_dynamic_run_command))
{
badRequest($c, "Dynamic RunCommand is not enabled by the server or the parent project.");
}
$jobset->update(
{ name => $jobsetName
, description => trim($c->stash->{params}->{"description"})
@ -268,7 +276,7 @@ sub updateJobset {
, nixexprinput => $nixExprInput
, enabled => $enabled
, enableemail => defined $c->stash->{params}->{enableemail} ? 1 : 0
, enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
, enable_dynamic_run_command => $enable_dynamic_run_command
, emailoverride => trim($c->stash->{params}->{emailoverride}) || ""
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, keepnr => int(trim($c->stash->{params}->{keepnr} // "0"))

7
src/lib/Hydra/Controller/Project.pm
View file

@ -149,6 +149,11 @@ sub updateProject {
my $displayName = trim $c->stash->{params}->{displayname};
error($c, "You must specify a display name.") if $displayName eq "";
my $enable_dynamic_run_command = defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command && !$c->config->{dynamicruncommand}->{enable}) {
badRequest($c, "Dynamic RunCommand is not enabled by the server.");
}
$project->update(
{ name => $projectName
, displayname => $displayName
@ -157,7 +162,7 @@ sub updateProject {
, enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
, hidden => defined $c->stash->{params}->{visible} ? 0 : 1
, owner => $owner
, enable_dynamic_run_command => defined $c->stash->{params}->{enable_dynamic_run_command} ? 1 : 0
, enable_dynamic_run_command => $enable_dynamic_run_command
, declfile => trim($c->stash->{params}->{declarative}->{file})
, decltype => trim($c->stash->{params}->{declarative}->{type})
, declvalue => trim($c->stash->{params}->{declarative}->{value})

40
src/lib/Hydra/Helper/AddBuilds.pm
View file

@ -19,14 +19,16 @@ use Hydra::Helper::CatalystUtils;
our @ISA = qw(Exporter);
our @EXPORT = qw(
validateDeclarativeJobset
createJobsetInputsRowAndData
updateDeclarativeJobset
handleDeclarativeJobsetBuild
handleDeclarativeJobsetJson
);
sub updateDeclarativeJobset {
my ($db, $project, $jobsetName, $declSpec) = @_;
sub validateDeclarativeJobset {
my ($config, $project, $jobsetName, $declSpec) = @_;
my @allowed_keys = qw(
enabled
@ -62,16 +64,39 @@ sub updateDeclarativeJobset {
}
}
$db->txn_do(sub {
my $jobset = $project->jobsets->update_or_create(\%update);
$jobset->jobsetinputs->delete;
foreach my $name (keys %{$declSpec->{"inputs"}}) {
my $enable_dynamic_run_command = defined $update{enable_dynamic_run_command} ? 1 : 0;
if ($enable_dynamic_run_command
&& !($config->{dynamicruncommand}->{enable}
&& $project->{enable_dynamic_run_command}))
{
die "Dynamic RunCommand is not enabled by the server or the parent project.";
}
return %update;
}
sub createJobsetInputsRowAndData {
my ($name, $declSpec) = @_;
my $data = $declSpec->{"inputs"}->{$name};
my $row = {
name => $name,
type => $data->{type}
};
$row->{emailresponsible} = $data->{emailresponsible} // 0;
return ($row, $data);
}
sub updateDeclarativeJobset {
my ($config, $db, $project, $jobsetName, $declSpec) = @_;
my %update = validateDeclarativeJobset($config, $project, $jobsetName, $declSpec);
$db->txn_do(sub {
my $jobset = $project->jobsets->update_or_create(\%update);
$jobset->jobsetinputs->delete;
foreach my $name (keys %{$declSpec->{"inputs"}}) {
my ($row, $data) = createJobsetInputsRowAndData($name, $declSpec);
my $input = $jobset->jobsetinputs->create($row);
$input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
}
@ -82,6 +107,7 @@ sub updateDeclarativeJobset {
sub handleDeclarativeJobsetJson {
my ($db, $project, $declSpec) = @_;
my $config = getHydraConfig();
$db->txn_do(sub {
my @kept = keys %$declSpec;
push @kept, ".jobsets";
@ -89,7 +115,7 @@ sub handleDeclarativeJobsetJson {
foreach my $jobsetName (keys %$declSpec) {
my $spec = $declSpec->{$jobsetName};
eval {
updateDeclarativeJobset($db, $project, $jobsetName, $spec);
updateDeclarativeJobset($config, $db, $project, $jobsetName, $spec);
1;
} or do {
print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";

10
src/root/edit-jobset.tt
View file

@ -160,7 +160,15 @@
<div class="form-group row">
<label class="col-sm-3" for="editjobsetenable_dynamic_run_command">Enable Dynamic RunCommand Hooks</label>
<div class="col-sm-9">
<input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
<input type="checkbox" id="editjobsetenable_dynamic_run_command" name="enable_dynamic_run_command"
[% IF !c.config.dynamicruncommand.enable %]
title="The server has not enabled dynamic RunCommands" disabled
[% ELSIF !project.enable_dynamic_run_command %]
title="The parent project has not enabled dynamic RunCommands" disabled
[% ELSIF jobset.enable_dynamic_run_command %]
checked
[% END %]
/>
</div>
</div>

8
src/root/edit-project.tt
View file

@ -56,7 +56,13 @@
<div class="form-group row">
<label class="col-sm-3" for="editprojectenable_dynamic_run_command">Enable Dynamic RunCommand Hooks for Jobsets</label>
<div class="col-sm-9">
<input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command" [% IF jobset.enable_dynamic_run_command %]checked[% END %]/>
<input type="checkbox" id="editprojectenable_dynamic_run_command" name="enable_dynamic_run_command"
[% IF !c.config.dynamicruncommand.enable %]
title="The server has not enabled dynamic RunCommands" disabled
[% ELSIF project.enable_dynamic_run_command %]
checked
[% END %]
/>
</div>
</div>

2
src/root/jobset.tt
View file

@ -162,7 +162,7 @@
</tr>
<tr>
<th>Enable Dynamic RunCommand Hooks:</th>
<td>[% jobset.enable_dynamic_run_command ? "Yes" : "No" %]</td>
<td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? jobset.enable_dynamic_run_command ? "Yes" : "No (not enabled by jobset)" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
</tr>
[% IF emailNotification %]
<tr>

2
src/root/project.tt
View file

@ -94,7 +94,7 @@
</tr>
<tr>
<th>Enable Dynamic RunCommand Hooks:</th>
<td>[% project.enable_dynamic_run_command ? "Yes" : "No" %]</td>
<td>[% c.config.dynamicruncommand.enable ? project.enable_dynamic_run_command ? "Yes" : "No (not enabled by project)" : "No (not enabled by server)" %]</td>
</tr>
</table>
</div>

2
src/script/hydra-eval-jobset
View file

@ -617,7 +617,7 @@ sub checkJobsetWrapped {
} else {
# Update the jobset with the spec's inputs, and the continue
# evaluating the .jobsets jobset.
updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
updateDeclarativeJobset($config, $db, $project, ".jobsets", $declSpec);
$jobset->discard_changes;
$inputInfo->{"declInput"} = [ $declInput ];
$inputInfo->{"projectName"} = [ fetchInput($plugins, $db, $project, $jobset, "projectName", "string", $project->name, 0) ];

85
t/Helper/AddBuilds/dynamic-disabled.t Normal file
View file

@ -0,0 +1,85 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
use Hydra::Helper::Nix qw(getHydraConfig);
my $ctx = test_context();
sub makeJobsetSpec {
my ($dynamic) = @_;
return {
enabled => 2,
enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
visible => JSON::MaybeXS::true,
name => "job",
type => 1,
description => "test jobset",
flake => "github:nixos/nix",
checkinterval => 0,
schedulingshares => 100,
keepnr => 3
};
};
subtest "validate declarative jobset with dynamic RunCommand disabled by server" => sub {
my $config = getHydraConfig();
subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
like(
dies {
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true),
),
},
qr/Dynamic RunCommand is not enabled/,
);
};
subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
like(
dies {
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true),
),
},
qr/Dynamic RunCommand is not enabled/,
);
};
subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
};
done_testing;

88
t/Helper/AddBuilds/dynamic-enabled.t Normal file
View file

@ -0,0 +1,88 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
use Hydra::Helper::AddBuilds qw(validateDeclarativeJobset);
use Hydra::Helper::Nix qw(getHydraConfig);
my $ctx = test_context(
hydra_config => q|
<dynamicruncommand>
enable = 1
</dynamicruncommand>
|
);
sub makeJobsetSpec {
my ($dynamic) = @_;
return {
enabled => 2,
enable_dynamic_run_command => $dynamic ? JSON::MaybeXS::true : undef,
visible => JSON::MaybeXS::true,
name => "job",
type => 1,
description => "test jobset",
flake => "github:nixos/nix",
checkinterval => 0,
schedulingshares => 100,
keepnr => 3
};
};
subtest "validate declarative jobset with dynamic RunCommand enabled by server" => sub {
my $config = getHydraConfig();
subtest "project enabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true)
),
);
};
subtest "project enabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 1 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
subtest "project disabled dynamic runcommand, declarative jobset enabled dynamic runcommand" => sub {
like(
dies {
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::true),
),
},
qr/Dynamic RunCommand is not enabled/,
);
};
subtest "project disabled dynamic runcommand, declarative jobset disabled dynamic runcommand" => sub {
ok(
validateDeclarativeJobset(
$config,
{ enable_dynamic_run_command => 0 },
"test-jobset",
makeJobsetSpec(JSON::MaybeXS::false)
),
);
};
};
done_testing;

110
t/Hydra/Plugin/RunCommand/dynamic-disabled.t Normal file
View file

@ -0,0 +1,110 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
my $ctx = test_context();
Catalyst::Test->import('Hydra');
# Create a user to log in to
my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
$user->setPassword('foobar');
$user->userroles->update_or_create({ role => 'admin' });
subtest "can't enable dynamic RunCommand when disabled by server" => sub {
my $builds = $ctx->makeAndEvaluateJobset(
expression => "runcommand-dynamic.nix",
build => 1
);
my $build = $builds->{"runCommandHook.example"};
my $project = $build->project;
my $project_name = $project->name;
my $jobset = $build->jobset;
my $jobset_name = $jobset->name;
is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
my $req = request(POST '/login',
Referer => 'http://localhost/',
Content => {
username => 'alice',
password => 'foobar'
}
);
is($req->code, 302, "logged in successfully");
my $cookie = $req->header("set-cookie");
subtest "can't enable dynamic RunCommand on project" => sub {
my $projectresponse = request(GET "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
my $projectjson = decode_json($projectresponse->content);
$projectjson->{enable_dynamic_run_command} = 1;
my $projectupdate = request(PUT "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
Content => encode_json($projectjson)
);
$projectresponse = request(GET "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
$projectjson = decode_json($projectresponse->content);
is($projectupdate->code, 400);
like(
$projectupdate->content,
qr/Dynamic RunCommand is not/,
"failed to change enable_dynamic_run_command, not any other error"
);
is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
};
subtest "can't enable dynamic RunCommand on jobset" => sub {
my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
my $jobsetjson = decode_json($jobsetresponse->content);
$jobsetjson->{enable_dynamic_run_command} = 1;
my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
Content => encode_json($jobsetjson)
);
$jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
$jobsetjson = decode_json($jobsetresponse->content);
is($jobsetupdate->code, 400);
like(
$jobsetupdate->content,
qr/Dynamic RunCommand is not/,
"failed to change enable_dynamic_run_command, not any other error"
);
is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::false);
};
};
done_testing;

106
t/Hydra/Plugin/RunCommand/dynamic-enabled.t Normal file
View file

@ -0,0 +1,106 @@
use strict;
use warnings;
use Setup;
use Test2::V0;
require Catalyst::Test;
use HTTP::Request::Common qw(POST PUT GET DELETE);
use JSON::MaybeXS qw(decode_json encode_json);
my $ctx = test_context(
hydra_config => q|
<dynamicruncommand>
enable = 1
</dynamicruncommand>
|
);
Catalyst::Test->import('Hydra');
# Create a user to log in to
my $user = $ctx->db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
$user->setPassword('foobar');
$user->userroles->update_or_create({ role => 'admin' });
subtest "can enable dynamic RunCommand when enabled by server" => sub {
my $builds = $ctx->makeAndEvaluateJobset(
expression => "runcommand-dynamic.nix",
build => 1
);
my $build = $builds->{"runCommandHook.example"};
my $project = $build->project;
my $project_name = $project->name;
my $jobset = $build->jobset;
my $jobset_name = $jobset->name;
is($project->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on projects by default");
is($jobset->enable_dynamic_run_command, 0, "dynamic RunCommand is disabled on jobsets by default");
my $req = request(POST '/login',
Referer => 'http://localhost/',
Content => {
username => 'alice',
password => 'foobar'
}
);
is($req->code, 302, "logged in successfully");
my $cookie = $req->header("set-cookie");
subtest "can enable dynamic RunCommand on project" => sub {
my $projectresponse = request(GET "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
my $projectjson = decode_json($projectresponse->content);
$projectjson->{enable_dynamic_run_command} = 1;
my $projectupdate = request(PUT "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
Content => encode_json($projectjson)
);
$projectresponse = request(GET "/project/$project_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
$projectjson = decode_json($projectresponse->content);
is($projectupdate->code, 200);
is($projectjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
};
subtest "can enable dynamic RunCommand on jobset" => sub {
my $jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
my $jobsetjson = decode_json($jobsetresponse->content);
$jobsetjson->{enable_dynamic_run_command} = 1;
my $jobsetupdate = request(PUT "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
Content => encode_json($jobsetjson)
);
$jobsetresponse = request(GET "/jobset/$project_name/$jobset_name",
Accept => 'application/json',
Content_Type => 'application/json',
Cookie => $cookie,
);
$jobsetjson = decode_json($jobsetresponse->content);
is($jobsetupdate->code, 200);
is($jobsetjson->{enable_dynamic_run_command}, JSON::MaybeXS::true);
};
};
done_testing;