Enable declarative projects.

This allows fully declarative project specifications. This is best
illustrated by example:

* I create a new project, setting the declarative spec file to
  "spec.json" and the declarative input to a git repo pointing
  at git://github.com/shlevy/declarative-hydra-example.git
* hydra creates a special ".jobsets" jobset alongside the project
* Just before evaluating the ".jobsets" jobset, hydra fetches
  declarative-hydra-example.git, reads spec.json as a jobset spec,
  and updates the jobset's configuration accordingly:
{
    "enabled": 1,
    "hidden": false,
    "description": "Jobsets",
    "nixexprinput": "src",
    "nixexprpath": "default.nix",
    "checkinterval": 300,
    "schedulingshares": 100,
    "enableemail": false,
    "emailoverride": "",
    "keepnr": 3,
    "inputs": {
        "src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
        "nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
    }
}
* When the "jobsets" job of the ".jobsets" jobset completes, hydra
  reads its output as a JSON representation of a dictionary of
  jobset specs and creates a jobset named "master" configured
  accordingly (In this example, this is the same configuration as
  .jobsets itself, except using release.nix instead of default.nix):
{
    "enabled": 1,
    "hidden": false,
    "description": "js",
    "nixexprinput": "src",
    "nixexprpath": "release.nix",
    "checkinterval": 300,
    "schedulingshares": 100,
    "enableemail": false,
    "emailoverride": "",
    "keepnr": 3,
    "inputs": {
        "src": { "type": "git", "value": "git://github.com/shlevy/declarative-hydra-example.git", "emailresponsible": false },
        "nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs.git release-16.03", "emailresponsible": false }
    }
}

src/lib/Hydra/Controller/Jobset.pm

@@ -55,6 +55,10 @@ sub jobset_PUT {
 
     requireProjectOwner($c, $c->stash->{project});
 
+    if (length($c->stash->{project}->declfile)) {
+        error($c, "can't modify jobset of declarative project", 403);
+    }
+
     if (defined $c->stash->{jobset}) {
         txn_do($c->model('DB')->schema, sub {
             updateJobset($c, $c->stash->{jobset});
@@ -88,6 +92,10 @@ sub jobset_DELETE {
 
     requireProjectOwner($c, $c->stash->{project});
 
+    if (length($c->stash->{project}->declfile)) {
+        error($c, "can't modify jobset of declarative project", 403);
+    }
+
     txn_do($c->model('DB')->schema, sub {
         $c->stash->{jobset}->jobsetevals->delete;
         $c->stash->{jobset}->builds->delete;

src/lib/Hydra/Controller/Project.pm

@@ -154,7 +154,19 @@ sub updateProject {
         , enabled => defined $c->stash->{params}->{enabled} ? 1 : 0
         , hidden => defined $c->stash->{params}->{visible} ? 0 : 1
         , owner => $owner
+        , declfile => trim($c->stash->{params}->{declfile})
+        , decltype => trim($c->stash->{params}->{decltype})
+        , declvalue => trim($c->stash->{params}->{declvalue})
         });
+    if (length($project->declfile)) {
+        $project->jobsets->update_or_create(
+            { name=> ".jobsets"
+            , nixexprinput => ""
+            , nixexprpath => ""
+            , emailoverride => ""
+            , triggertime => time
+            });
+    }
 }
 
 

src/lib/Hydra/Helper/AddBuilds.pm

@@ -22,7 +22,8 @@ use Hydra::Helper::CatalystUtils;
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
     fetchInput evalJobs checkBuild inputsToArgs
-    restartBuild getPrevJobsetEval
+    restartBuild getPrevJobsetEval updateDeclarativeJobset
+    handleDeclarativeJobsetBuild
 );
 
 
@@ -467,4 +468,66 @@ sub checkBuild {
 };
 
 
+sub updateDeclarativeJobset {
+    my ($db, $project, $jobsetName, $declSpec) = @_;
+
+    my @allowed_keys = qw(
+        enabled
+        hidden
+        description
+        nixexprinput
+        nixexprpath
+        checkinterval
+        schedulingshares
+        enableemail
+        emailoverride
+        keepnr
+    );
+    my %update = ( name => $jobsetName );
+    foreach my $key (@allowed_keys) {
+        $update{$key} = $declSpec->{$key};
+        delete $declSpec->{$key};
+    }
+    txn_do($db, sub {
+        my $jobset = $project->jobsets->update_or_create(\%update);
+        $jobset->jobsetinputs->delete;
+        while ((my $name, my $data) = each %{$declSpec->{"inputs"}}) {
+            my $input = $jobset->jobsetinputs->create(
+                { name => $name,
+                  type => $data->{type},
+                  emailresponsible => $data->{emailresponsible}
+                });
+            $input->jobsetinputalts->create({altnr => 0, value => $data->{value}});
+        }
+        delete $declSpec->{"inputs"};
+        die "invalid keys in declarative specification file\n" if (%{$declSpec});
+    });
+};
+
+
+sub handleDeclarativeJobsetBuild {
+    my ($db, $project, $build) = @_;
+
+    eval {
+        my $id = $build->id;
+        die "Declarative jobset build $id failed" unless $build->buildstatus == 0;
+        my $declPath = ($build->buildoutputs)[0]->path;
+        my $declText = read_file($declPath)
+            or die "Couldn't read declarative specification file $declPath: $!";
+        my $declSpec = decode_json($declText);
+        txn_do($db, sub {
+            my @kept = keys %$declSpec;
+            push @kept, ".jobsets";
+            $project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 });
+            while ((my $jobsetName, my $spec) = each %$declSpec) {
+                updateDeclarativeJobset($db, $project, $jobsetName, $spec);
+            }
+        });
+    };
+    $project->jobsets->find({ name => ".jobsets" })->update({ errormsg => $@, errortime => time, fetcherrormsg => undef })
+        if defined $@;
+
+};
+
+
 1;

src/lib/Hydra/Helper/CatalystUtils.pm

@@ -264,7 +264,7 @@ Readonly our $inputNameRE   => "(?:[A-Za-z_][A-Za-z0-9-_]*)";
 
 sub parseJobsetName {
     my ($s) = @_;
-    $s =~ /^($projectNameRE):($jobsetNameRE)$/ or die "invalid jobset specifier ‘$s’\n";
+    $s =~ /^($projectNameRE):(\.?$jobsetNameRE)$/ or die "invalid jobset specifier ‘$s’\n";
     return ($1, $2);
 }
 

src/lib/Hydra/Schema/Projects.pm

@@ -73,6 +73,21 @@ __PACKAGE__->table("Projects");
   data_type: 'text'
   is_nullable: 1
 
+=head2 declfile
+
+  data_type: 'text'
+  is_nullable: 1
+
+=head2 decltype
+
+  data_type: 'text'
+  is_nullable: 1
+
+=head2 declvalue
+
+  data_type: 'text'
+  is_nullable: 1
+
 =cut
 
 __PACKAGE__->add_columns(
@@ -90,6 +105,12 @@ __PACKAGE__->add_columns(
   { data_type => "text", is_foreign_key => 1, is_nullable => 0 },
   "homepage",
   { data_type => "text", is_nullable => 1 },
+  "declfile",
+  { data_type => "text", is_nullable => 1 },
+  "decltype",
+  { data_type => "text", is_nullable => 1 },
+  "declvalue",
+  { data_type => "text", is_nullable => 1 },
 );
 
 =head1 PRIMARY KEY
@@ -282,8 +303,8 @@ Composing rels: L</projectmembers> -> username
 __PACKAGE__->many_to_many("usernames", "projectmembers", "username");
 
 
-# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:67kWIE0IGmEJTvOIATAKaw
+# Created by DBIx::Class::Schema::Loader v0.07043 @ 2016-03-11 10:39:17
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:1ats3brIVhRTWLToIYSoaQ
 
 my %hint = (
     columns => [

src/root/edit-project.tt

@@ -53,6 +53,25 @@
       </div>
     </div>
 
+    <div class="control-group">
+      <label class="control-label">Declarative spec file</label>
+      <div class="controls">
+        <div class="input-append">
+          <input type="text" class="span3" name="declfile" [% HTML.attributes(value => project.declfile) %]/>
+        </div>
+        <span class="help-inline">(Leave blank for non-declarative project configuration)</span>
+      </div>
+    </div>
+
+    <div class="control-group">
+      <label class="control-label">Declarative input type</label>
+      <div class="controls">
+        [% INCLUDE renderSelection param="decltype" options=inputTypes edit=1 curValue=project.decltype %]
+        value
+        <input style="width: 70%" type="text" [% HTML.attributes(value => project.declvalue, name => "declvalue") %]/>
+      </div>
+    </div>
+
     <div class="form-actions">
       <button id="submit-project" type="submit" class="btn btn-primary">
         <i class="icon-ok icon-white"></i>

src/root/jobset.tt

@@ -49,9 +49,11 @@
         <b class="caret"></b>
       </a>
       <ul class="dropdown-menu">
+        [% UNLESS project.declfile %]
         [% INCLUDE menuItem title="Edit configuration" icon="icon-edit" uri=c.uri_for(c.controller('Jobset').action_for('edit'), c.req.captures) %]
         [% INCLUDE menuItem title="Delete this jobset" icon="icon-trash" uri="javascript:deleteJobset()" %]
         [% INCLUDE menuItem title="Clone this jobset" uri=c.uri_for(c.controller('Jobset').action_for('edit'), c.req.captures, { cloneJobset => 1 }) %]
+        [% END %]
         [% INCLUDE menuItem title="Evaluate this jobset" uri="javascript:confirmEvaluateJobset()" %]
       </ul>
     </li>

src/root/project.tt

@@ -11,7 +11,9 @@
       <ul class="dropdown-menu">
         [% INCLUDE menuItem title="Edit configuration" icon="icon-edit" uri=c.uri_for(c.controller('Project').action_for('edit'), c.req.captures) %]
         [% INCLUDE menuItem title="Delete this project" icon="icon-trash" uri="javascript:deleteProject()" %]
+        [% UNLESS project.declfile %]
         [% INCLUDE menuItem title="Create jobset" icon="icon-plus" uri=c.uri_for(c.controller('Project').action_for('create_jobset'), c.req.captures) %]
+        [% END %]
         [% INCLUDE menuItem title="Create release" icon="icon-plus" uri=c.uri_for(c.controller('Project').action_for('create_release'), c.req.captures) %]
       </ul>
     </li>

src/script/hydra-evaluator

@@ -14,6 +14,8 @@ use Data::Dump qw(dump);
 use Try::Tiny;
 use Net::Statsd;
 use Time::HiRes qw(clock_gettime CLOCK_REALTIME);
+use JSON;
+use File::Slurp;
 
 STDOUT->autoflush();
 STDERR->autoflush(1);
@@ -100,6 +102,23 @@ sub permute {
 sub checkJobsetWrapped {
     my ($jobset) = @_;
     my $project = $jobset->project;
+    my $jobsetsJobset = length($project->declfile) && $jobset->name eq ".jobsets";
+    if ($jobsetsJobset) {
+        my @declInputs = fetchInput($plugins, $db, $project, $jobset, "decl", $project->decltype, $project->declvalue, 0);
+        my $declInput = @declInputs[0] or die "cannot find the input containing the declarative project specification\n";
+        die "multiple alternatives for the input containing the declarative project specificaiton are not supported\n"
+            if scalar @declInputs != 1;
+        my $declFile = $declInput->{storePath} . "/" . $project->declfile;
+        my $declText = read_file($declFile)
+            or die "Couldn't read declarative specification file $declFile: $!\n";
+        my $declSpec;
+        eval {
+            $declSpec = decode_json($declText);
+        };
+        die "Declarative specification file $declFile not valid JSON: $@\n" if $@;
+        updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
+        $jobset->discard_changes;
+    }
     my $inputInfo = {};
     my $exprType = $jobset->nixexprpath =~ /.scm$/ ? "guile" : "nix";
 
@@ -143,6 +162,11 @@ sub checkJobsetWrapped {
     my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
     my $evalStop = clock_gettime(CLOCK_REALTIME);
 
+    if ($jobsetsJobset) {
+        my @keys = keys %$jobs;
+        die "The .jobsets jobset must only have a single job named 'jobsets'"
+            unless (scalar @keys) == 1 && $keys[0] eq "jobsets";
+    }
     Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000));
 
     if ($dryRun) {

src/script/hydra-notify

@@ -5,6 +5,7 @@ use utf8;
 use Hydra::Plugin;
 use Hydra::Helper::Nix;
 use Hydra::Helper::PluginHooks;
+use Hydra::Helper::AddBuilds;
 
 STDERR->autoflush(1);
 binmode STDERR, ":encoding(utf8)";
@@ -21,6 +22,11 @@ my $buildId = shift @ARGV or die;
 my $build = $db->resultset('Builds')->find($buildId)
     or die "build $buildId does not exist\n";
 if ($cmd eq "build-finished") {
+    my $project = $build->project;
+    my $jobset = $build->jobset;
+    if (length($project->declfile) && $jobset->name eq ".jobsets" && $build->iscurrent) {
+        handleDeclarativeJobsetBuild($db, $project, $build);
+    }
     my @dependents;
     foreach my $id (@ARGV) {
         my $dep = $db->resultset('Builds')->find($id)

src/sql/hydra.sql

@@ -30,6 +30,9 @@ create table Projects (
     hidden        integer not null default 0,
     owner         text not null,
     homepage      text, -- URL for the project
+    declfile      text, -- File containing declarative jobset specification
+    decltype      text, -- Type of the input containing declarative jobset specification
+    declvalue     text, -- Value of the input containing declarative jobset specification
     foreign key   (owner) references Users(userName) on update cascade
 );
 

src/sql/upgrade-48.sql

@@ -0,0 +1,4 @@
+-- Add declarative fields to Projects
+alter table Projects add column declfile text;
+alter table Projects add column decltype text;
+alter table Projects add column declvalue text;