From 4392d3e21db42f02a70c56db9a227e6a56c9e1df Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 11 Mar 2016 18:14:58 -0500 Subject: [PATCH 1/2] Enable declarative projects. This allows fully declarative project specifications. This is best illustrated by example: * I create a new project, setting the declarative spec file to "spec.json" and the declarative input to a git repo pointing at git://github.com/shlevy/declarative-hydra-example.git * hydra creates a special ".jobsets" jobset alongside the project * Just before evaluating the ".jobsets" jobset, hydra fetches declarative-hydra-example.git, reads spec.json as a jobset spec, and updates the jobset's configuration accordingly: { "enabled": 1, "hidden": false, "description": "Jobsets", "nixexprinput": "src", "nixexprpath": "default.nix", "checkinterval": 300, "schedulingshares": 100, "enableemail": false, "emailoverride": "", "keepnr": 3, "inputs": { "src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false }, "nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false } } } * When the "jobsets" job of the ".jobsets" jobset completes, hydra reads its output as a JSON representation of a dictionary of jobset specs and creates a jobset named "master" configured accordingly (In this example, this is the same configuration as .jobsets itself, except using release.nix instead of default.nix): { "enabled": 1, "hidden": false, "description": "js", "nixexprinput": "src", "nixexprpath": "release.nix", "checkinterval": 300, "schedulingshares": 100, "enableemail": false, "emailoverride": "", "keepnr": 3, "inputs": { "src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false }, "nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false } } } --- src/lib/Hydra/Controller/Jobset.pm | 8 ++++ src/lib/Hydra/Controller/Project.pm | 12 +++++ src/lib/Hydra/Helper/AddBuilds.pm | 65 ++++++++++++++++++++++++++- src/lib/Hydra/Helper/CatalystUtils.pm | 2 +- src/lib/Hydra/Schema/Projects.pm | 25 ++++++++++- src/root/edit-project.tt | 19 ++++++++ src/root/jobset.tt | 2 + src/root/project.tt | 2 + src/script/hydra-evaluator | 24 ++++++++++ src/script/hydra-notify | 6 +++ src/sql/hydra.sql | 3 ++ src/sql/upgrade-48.sql | 4 ++ 12 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 src/sql/upgrade-48.sql diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index f2b801f1..06ccebcd 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -55,6 +55,10 @@ sub jobset_PUT { requireProjectOwner($c, $c->stash->{project}); + if (length($c->stash->{project}->declfile)) { + error($c, "can't modify jobset of declarative project", 403); + } + if (defined $c->stash->{jobset}) { txn_do($c->model('DB')->schema, sub { updateJobset($c, $c->stash->{jobset}); @@ -88,6 +92,10 @@ sub jobset_DELETE { requireProjectOwner($c, $c->stash->{project}); + if (length($c->stash->{project}->declfile)) { + error($c, "can't modify jobset of declarative project", 403); + } + txn_do($c->model('DB')->schema, sub { $c->stash->{jobset}->jobsetevals->delete; $c->stash->{jobset}->builds->delete; diff --git a/src/lib/Hydra/Controller/Project.pm b/src/lib/Hydra/Controller/Project.pm index 4cd577b1..07be4d4a 100644 --- a/src/lib/Hydra/Controller/Project.pm +++ b/src/lib/Hydra/Controller/Project.pm @@ -154,7 +154,19 @@ sub updateProject { , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0 , hidden => defined $c->stash->{params}->{visible} ? 0 : 1 , owner => $owner + , declfile => trim($c->stash->{params}->{declfile}) + , decltype => trim($c->stash->{params}->{decltype}) + , declvalue => trim($c->stash->{params}->{declvalue}) }); + if (length($project->declfile)) { + $project->jobsets->update_or_create( + { name=> ".jobsets" + , nixexprinput => "" + , nixexprpath => "" + , emailoverride => "" + , triggertime => time + }); + } } diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 737fdbd4..0fa7087f 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -22,7 +22,8 @@ use Hydra::Helper::CatalystUtils; our @ISA = qw(Exporter); our @EXPORT = qw( fetchInput evalJobs checkBuild inputsToArgs - restartBuild getPrevJobsetEval + restartBuild getPrevJobsetEval updateDeclarativeJobset + handleDeclarativeJobsetBuild ); @@ -467,4 +468,66 @@ sub checkBuild { }; +sub updateDeclarativeJobset { + my ($db, $project, $jobsetName, $declSpec) = @_; + + my @allowed_keys = qw( + enabled + hidden + description + nixexprinput + nixexprpath + checkinterval + schedulingshares + enableemail + emailoverride + keepnr + ); + my %update = ( name => $jobsetName ); + foreach my $key (@allowed_keys) { + $update{$key} = $declSpec->{$key}; + delete $declSpec->{$key}; + } + txn_do($db, sub { + my $jobset = $project->jobsets->update_or_create(\%update); + $jobset->jobsetinputs->delete; + while ((my $name, my $data) = each %{$declSpec->{"inputs"}}) { + my $input = $jobset->jobsetinputs->create( + { name => $name, + type => $data->{type}, + emailresponsible => $data->{emailresponsible} + }); + $input->jobsetinputalts->create({altnr => 0, value => $data->{value}}); + } + delete $declSpec->{"inputs"}; + die "invalid keys in declarative specification file\n" if (%{$declSpec}); + }); +}; + + +sub handleDeclarativeJobsetBuild { + my ($db, $project, $build) = @_; + + eval { + my $id = $build->id; + die "Declarative jobset build $id failed" unless $build->buildstatus == 0; + my $declPath = ($build->buildoutputs)[0]->path; + my $declText = read_file($declPath) + or die "Couldn't read declarative specification file $declPath: $!"; + my $declSpec = decode_json($declText); + txn_do($db, sub { + my @kept = keys %$declSpec; + push @kept, ".jobsets"; + $project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 }); + while ((my $jobsetName, my $spec) = each %$declSpec) { + updateDeclarativeJobset($db, $project, $jobsetName, $spec); + } + }); + }; + $project->jobsets->find({ name => ".jobsets" })->update({ errormsg => $@, errortime => time, fetcherrormsg => undef }) + if defined $@; + +}; + + 1; diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm index df966d80..019c07b7 100644 --- a/src/lib/Hydra/Helper/CatalystUtils.pm +++ b/src/lib/Hydra/Helper/CatalystUtils.pm @@ -264,7 +264,7 @@ Readonly our $inputNameRE => "(?:[A-Za-z_][A-Za-z0-9-_]*)"; sub parseJobsetName { my ($s) = @_; - $s =~ /^($projectNameRE):($jobsetNameRE)$/ or die "invalid jobset specifier ‘$s’\n"; + $s =~ /^($projectNameRE):(\.?$jobsetNameRE)$/ or die "invalid jobset specifier ‘$s’\n"; return ($1, $2); } diff --git a/src/lib/Hydra/Schema/Projects.pm b/src/lib/Hydra/Schema/Projects.pm index e04b1f8e..11405561 100644 --- a/src/lib/Hydra/Schema/Projects.pm +++ b/src/lib/Hydra/Schema/Projects.pm @@ -73,6 +73,21 @@ __PACKAGE__->table("Projects"); data_type: 'text' is_nullable: 1 +=head2 declfile + + data_type: 'text' + is_nullable: 1 + +=head2 decltype + + data_type: 'text' + is_nullable: 1 + +=head2 declvalue + + data_type: 'text' + is_nullable: 1 + =cut __PACKAGE__->add_columns( @@ -90,6 +105,12 @@ __PACKAGE__->add_columns( { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "homepage", { data_type => "text", is_nullable => 1 }, + "declfile", + { data_type => "text", is_nullable => 1 }, + "decltype", + { data_type => "text", is_nullable => 1 }, + "declvalue", + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -282,8 +303,8 @@ Composing rels: L -> username __PACKAGE__->many_to_many("usernames", "projectmembers", "username"); -# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:67kWIE0IGmEJTvOIATAKaw +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2016-03-11 10:39:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:1ats3brIVhRTWLToIYSoaQ my %hint = ( columns => [ diff --git a/src/root/edit-project.tt b/src/root/edit-project.tt index b8d9bacb..cedc36ee 100644 --- a/src/root/edit-project.tt +++ b/src/root/edit-project.tt @@ -53,6 +53,25 @@ +
+ +
+
+ project.declfile) %]/> +
+ (Leave blank for non-declarative project configuration) +
+
+ +
+ +
+ [% INCLUDE renderSelection param="decltype" options=inputTypes edit=1 curValue=project.decltype %] + value + project.declvalue, name => "declvalue") %]/> +
+
+