Merge pull request #937 from DeterminateSystems/flesh-out-api

hydra-api: flesh out Jobset, JobsetInput schemas; implement DELETE /jobset/{project-id}/{jobset-id}

hydra-api.yaml

@@ -306,35 +306,7 @@ paths:
         content:
           application/json:
             schema:
-              type: object
+              $ref: '#/components/schemas/Jobset'
-              properties:
-                'description':
-                  description: a description of the jobset
-                  type: string
-                checkinterval:
-                  description: interval (in seconds) in which to check for evaluation
-                  type: integer
-                enabled:
-                  description: when true the jobset gets scheduled for evaluation
-                  type: boolean
-                visible:
-                  description: when true the jobset is visible in the web frontend
-                  type: boolean
-                keepnr:
-                  description: number or evaluations to keep
-                  type: integer
-                nixexprinput:
-                  description: the name of the jobset input which contains the nixexprpath
-                  type: string
-                nixexprpath:
-                  nullable: true
-                  description: the path to the file to evaluate
-                  type: string
-                inputs:
-                  description: inputs for this jobset
-                  type: object
-                  additionalProperties:
-                    $ref: '#/components/schemas/JobsetInput'
       responses:
         '201':
           description: jobset creation response
@@ -398,6 +370,39 @@ paths:
               schema:
                 $ref: '#/components/schemas/Error'
 
+    delete:
+      summary: Deletes a jobset designated by project and jobset id
+      parameters:
+        - name: project-id
+          in: path
+          description: name of the project the jobset belongs to
+          required: true
+          schema:
+            type: string
+        - name: jobset-id
+          in: path
+          description: name of the jobset to retrieve
+          required: true
+          schema:
+            type: string
+      responses:
+        '200':
+          description: jobset successfully deleted
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  redirect:
+                    type: string
+                    description: root of the Hydra instance
+        '404':
+          description: jobset couldn't be found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /jobset/{project-id}/{jobset-id}/evals:
     get:
       summary: Retrieves all evaluations of a jobset
@@ -581,6 +586,15 @@ components:
     JobsetInput:
       type: object
       properties:
+        name:
+          description: name of the input
+          type: string
+        type:
+          description: type of input
+          type: string
+        emailresponsible:
+          description: whether or not to email responsible parties
+          type: boolean
         jobsetinputalts:
           type: array
           description: ???
@@ -590,26 +604,76 @@ components:
     Jobset:
       type: object
       properties:
-        fetcherrormsg:
+        name:
+          description: the name of the jobset
+          type: string
+        project:
+          description: the project this jobset belongs to
+          type: string
+        description:
           nullable: true
-          description: contains the error message when there was a problem fetching sources for a jobset
+          description: a description of the jobset
           type: string
         nixexprinput:
+          nullable: true
           description: the name of the jobset input which contains the nixexprpath
           type: string
-        errormsg:
-          description: contains the stderr output of the nix-instantiate command
-          type: string
-        emailoverride:
-          description: email address to send notices to instead of the package maintainer (can be a comma separated list)
-          type: string
         nixexprpath:
           nullable: true
           description: the path to the file to evaluate
           type: string
+        errormsg:
+          nullable: true
+          description: contains the stderr output of the nix-instantiate command
+          type: string
+        errortime:
+          nullable: true
+          description: timestamp associated with errormsg
+          type: integer
+        lastcheckedtime:
+          nullable: true
+          description: the last time the evaluator looked at this jobset
+          type: integer
+        triggertime:
+          nullable: true
+          description: set to the time we were triggered by a push event
+          type: integer
         enabled:
-          description: when set to true the jobset gets scheduled for evaluation
+          description: 0 is disabled, 1 is enabled, 2 is one-shot, and 3 is one-at-a-time
+          type: integer
+        enableemail:
+          description: when true the jobset sends emails when previously-successful builds fail
           type: boolean
+        hidden:
+          description: when false the jobset is visible in the web frontend
+          type: boolean
+        emailoverride:
+          description: email address to send notices to instead of the package maintainer (can be a comma separated list)
+          type: string
+        keepnr:
+          description: number or evaluations to keep
+          type: integer
+        checkinterval:
+          description: interval (in seconds) in which to check for evaluation
+          type: integer
+        schedulingshares:
+          description: how many shares to be allocated to the jobset
+          type: integer
+        fetcherrormsg:
+          nullable: true
+          description: contains the error message when there was a problem fetching sources for a jobset
+          type: string
+        startime:
+          nullable: true
+          description: set to the time the latest evaluation started (if one is currently running)
+          type: integer
+        type:
+          description: the type of the jobset
+          type: string
+        flake:
+          nullable: true
+          description: the flake uri to evaluate
+          type: string
         jobsetinputs:
           description: inputs configured for this jobset
           type: object
@@ -871,49 +935,72 @@ components:
   examples:
     projects-success:
       value:
-      - enabled: 1
+      - displayname: Foo Bar
-        name: example-hello
+        description: Foo Bar Baz Qux
-        hidden: 0
+        enabled: true
-        description: hello
+        owner: alice
-        owner: hydra-user
         jobsets:
-          - hello
+          - bar-jobset
-        displayname: example-hello
+        hidden: false
-      - displayname: foo
+        homepage: https://example.com/
-        jobsets:
+        name: foobar
-          - foobar
+      - jobsets:
-        owner: hydra-user
+          - test-jobset
-        name: foo
+        hidden: false
-        enabled: 1
+        name: hello
-        description: foo project
+        homepage: https://example.com/
-        hidden: 0
+        description: Hi There
+        displayname: Hello
+        enabled: true
+        owner: alice
 
     project-success:
       value:
-        name: foo
-        enabled: 1
-        hidden: 0
-        description: foo project
-        displayname: foo
-        owner: gilligan
         jobsets:
-          - foobar
+          - bar-jobset
+        homepage: https://example.com/
+        name: foobar
+        hidden: false
+        enabled: true
+        displayname: Foo Bar
+        description: Foo Bar Baz Qux
+        owner: alice
 
     jobset-success:
       value:
-        nixexprpath: examples/hello.nix
+        triggertime: null
-        enabled: 1
+        enableemail: false
         jobsetinputs:
-          hydra:
-            jobsetinputalts:
-              - 'https://github.com/gilligan/hydra extend-readme'
           nixpkgs:
+            type: git
+            name: nixpkgs
+            emailresponsible: false
             jobsetinputalts:
-              - 'https://github.com/nixos/nixpkgs-channels nixos-20.03'
+              - https://github.com/NixOS/nixpkgs.git
+          officialRelease:
+            jobsetinputalts:
+              - 'false'
+            emailresponsible: false
+            name: officialRelease
+            type: boolean
+        fetcherrormsg: ''
+        hidden: false
+        schedulingshares: 1
         emailoverride: ''
+        starttime: null
+        description: ''
         errormsg: ''
-        nixexprinput: hydra
+        lastcheckedtime: null
-        fetcherrormsg: null
+        nixexprinput: nixpkgs
+        checkinterval: 0
+        project: foobar
+        flake: ''
+        type: 0
+        enabled: 1
+        name: bar-jobset
+        keepnr: 0
+        nixexprpath: pkgs/top-level/release.nix
+        errortime: null
 
     evals-success:
       value:

src/lib/Hydra/Controller/Jobset.pm

@@ -231,7 +231,7 @@ sub updateJobset {
     if ($type == 0) {
         ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c;
     } elsif ($type == 1) {
-        $flake = trim($c->stash->{params}->{"flakeref"});
+        $flake = trim($c->stash->{params}->{"flake"});
         error($c, "Invalid flake URI ‘$flake’.") if $flake !~ /^[a-zA-Z]/;
     } else {
         error($c, "Invalid jobset type.");
@@ -270,8 +270,8 @@ sub updateJobset {
     $jobset->jobsetinputs->delete;
 
     if ($type == 0) {
-        foreach my $name (keys %{$c->stash->{params}->{inputs}}) {
+        foreach my $name (keys %{$c->stash->{params}->{jobsetinputs}}) {
-            my $inputData = $c->stash->{params}->{inputs}->{$name};
+            my $inputData = $c->stash->{params}->{jobsetinputs}->{$name};
             my $type = $inputData->{type};
             my $value = $inputData->{value};
             my $emailresponsible = defined $inputData->{emailresponsible} ? 1 : 0;

src/lib/Hydra/Controller/User.pm

@@ -27,8 +27,8 @@ sub login_POST {
     my $username = $c->stash->{params}->{username} // "";
     my $password = $c->stash->{params}->{password} // "";
 
-    error($c, "You must specify a user name.") if $username eq "";
+    badRequest($c, "You must specify a user name.") if $username eq "";
-    error($c, "You must specify a password.") if $password eq "";
+    badRequest($c, "You must specify a password.") if $password eq "";
 
     if ($c->get_auth_realm('ldap') && $c->authenticate({username => $username, password => $password}, 'ldap')) {
         doLDAPLogin($self, $c, $username);
@@ -37,7 +37,11 @@ sub login_POST {
         accessDenied($c, "Bad username or password.")
     }
 
-    currentUser_GET($self, $c);
+    $self->status_found(
+        $c,
+        location => $c->uri_for("current-user"),
+        entity => {}
+    );
 }
 
 

src/lib/Hydra/Schema/JobsetInputs.pm

@@ -135,6 +135,13 @@ __PACKAGE__->has_many(
 # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5uKwEhDXso4IR1TFmwRxiA
 
 my %hint = (
+    string_columns => [
+        "name",
+        "type"
+    ],
+    boolean_columns => [
+        "emailresponsible"
+    ],
     relations => {
         "jobsetinputalts" => "value"
     }

src/lib/Hydra/Schema/Jobsets.pm

@@ -412,12 +412,30 @@ __PACKAGE__->add_column(
 
 my %hint = (
     columns => [
+        "errortime",
+        "lastcheckedtime",
+        "triggertime",
         "enabled",
-        "errormsg",
+        "keepnr",
-        "fetcherrormsg",
+        "checkinterval",
-        "emailoverride",
+        "schedulingshares",
+        "starttime"
+    ],
+    string_columns => [
+        "name",
+        "project",
+        "description",
+        "nixexprinput",
         "nixexprpath",
-        "nixexprinput"
+        "errormsg",
+        "emailoverride",
+        "fetcherrormsg",
+        "type",
+        "flake"
+    ],
+    boolean_columns => [
+        "enableemail",
+        "hidden"
     ],
     eager_relations => {
         jobsetinputs => "name"

src/root/edit-jobset.tt

@@ -46,8 +46,8 @@
     <thead>
       <tr><th></th><th>Input name</th><th>Type</th><th style="width: 50%">Value</th><th>Notify committers</th></tr>
     </thead>
-    <tbody class="inputs">
+    <tbody class="jobsetinputs">
-      [% inputs = createFromEval ? eval.jobsetevalinputs : jobset.jobsetinputs; FOREACH input IN inputs %]
+      [% jobsetinputs = createFromEval ? eval.jobsetevalinputs : jobset.jobsetinputs; FOREACH input IN jobsetinputs %]
         [% INCLUDE renderJobsetInput input=input baseName="input-$input.name" %]
       [% END %]
       <tr>
@@ -111,9 +111,9 @@
   </div>
 
   <div class="form-group row show-on-flake">
-    <label class="col-form-label col-sm-3" for="editjobsetflakeref">Flake URI</label>
+    <label class="col-form-label col-sm-3" for="editjobsetflake">Flake URI</label>
     <div class="col-sm-9">
-      <input type="text" class="form-control" id="editjobsetflakeref" name="flakeref" [% HTML.attributes(value => jobset.flake) %]/>
+      <input type="text" class="form-control" id="editjobsetflake" name="flake" [% HTML.attributes(value => jobset.flake) %]/>
     </div>
   </div>
 
@@ -220,8 +220,8 @@
 
   $("#submit-jobset").click(function() {
     var formElements = $(this).parents("form").serializeArray();
-    var data = { 'inputs': {} };
+    var data = { 'jobsetinputs': {} };
-    var inputs = {};
+    var jobsetinputs = {};
     for (var i = 0; formElements.length > i; i++) {
       var elem = formElements[i];
       var match = elem.name.match(/^input-([\w-]+)-(\w+)$/);
@@ -233,13 +233,13 @@
 
         if (baseName === "template") continue;
 
-        if (!(baseName in inputs))
+        if (!(baseName in jobsetinputs))
-            inputs[baseName] = {};
+            jobsetinputs[baseName] = {};
 
         if (param === "name")
-            data.inputs[elem.value] = inputs[baseName];
+            data.jobsetinputs[elem.value] = jobsetinputs[baseName];
         else
-            inputs[baseName][param] = elem.value;
+            jobsetinputs[baseName][param] = elem.value;
       }
     }
     redirectJSON({

t/Controller/Jobset/http.t

@@ -0,0 +1,176 @@
+use feature 'unicode_strings';
+use strict;
+use Setup;
+use JSON qw(decode_json encode_json);
+
+my %ctx = test_init();
+
+require Hydra::Schema;
+require Hydra::Model::DB;
+require Hydra::Helper::Nix;
+
+use Test2::V0;
+require Catalyst::Test;
+Catalyst::Test->import('Hydra');
+use HTTP::Request::Common qw(POST PUT GET DELETE);
+
+# This test verifies that creating, reading, updating, and deleting a jobset via
+# the HTTP API works as expected.
+
+my $db = Hydra::Model::DB->new;
+hydra_setup($db);
+
+# Create a user to log in to
+my $user = $db->resultset('Users')->create({ username => 'alice', emailaddress => 'root@invalid.org', password => '!' });
+$user->setPassword('foobar');
+$user->userroles->update_or_create({ role => 'admin' });
+
+my $project = $db->resultset('Projects')->create({name => 'tests', displayname => 'Tests', owner => 'alice'});
+
+# Login and save cookie for future requests
+my $req = request(POST '/login',
+    Referer => 'http://localhost/',
+    Content => {
+        username => 'alice',
+        password => 'foobar'
+    }
+);
+is($req->code, 302);
+my $cookie = $req->header("set-cookie");
+
+
+subtest 'Create new jobset "job" as flake type' => sub {
+  my $jobsetcreate = request(PUT '/jobset/tests/job',
+      Accept => 'application/json',
+      Content_Type => 'application/json',
+      Cookie => $cookie,
+      Content => encode_json({
+        enabled => 2,
+        visible => 1,
+        name => "job",
+        type => 1,
+        description => "test jobset",
+        flake => "github:nixos/nix",
+        checkinterval => 0,
+        schedulingshares => 100,
+        keepnr => 3
+      })
+  );
+  ok($jobsetcreate->is_success);
+  is($jobsetcreate->header("location"), "http://localhost/jobset/tests/job");
+};
+
+
+subtest 'Read newly-created jobset "job"' => sub {
+  my $jobsetinfo = request(GET '/jobset/tests/job',
+      Accept => 'application/json',
+  );
+  ok($jobsetinfo->is_success);
+  is(decode_json($jobsetinfo->content), {
+     checkinterval => 0,
+     description => "test jobset",
+     emailoverride => "",
+     enabled => 2,
+     enableemail => JSON::false,
+     errortime => undef,
+     errormsg => "",
+     fetcherrormsg => "",
+     flake => "github:nixos/nix",
+     hidden => JSON::false,
+     jobsetinputs => {},
+     keepnr => 3,
+     lastcheckedtime => undef,
+     name => "job",
+     nixexprinput => "",
+     nixexprpath => "",
+     project => "tests",
+     schedulingshares => 100,
+     starttime => undef,
+     triggertime => undef,
+     type => 1
+  });
+};
+
+
+subtest 'Update jobset "job" to legacy type' => sub {
+  my $jobsetupdate = request(PUT '/jobset/tests/job',
+      Accept => 'application/json',
+      Content_Type => 'application/json',
+      Cookie => $cookie,
+      Content => encode_json({
+        enabled => 3,
+        visible => 1,
+        name => "job",
+        type => 0,
+        nixexprinput => "ofborg",
+        nixexprpath => "release.nix",
+        jobsetinputs => {
+          ofborg => {
+            name => "ofborg",
+            type => "git",
+            value => "https://github.com/NixOS/ofborg.git released"
+          }
+        },
+        description => "test jobset",
+        checkinterval => 0,
+        schedulingshares => 50,
+        keepnr => 1
+      })
+  );
+  ok($jobsetupdate->is_success);
+
+  # Read newly-updated jobset "job"
+  my $jobsetinfo = request(GET '/jobset/tests/job',
+      Accept => 'application/json',
+  );
+  ok($jobsetinfo->is_success);
+  is(decode_json($jobsetinfo->content), {
+    checkinterval => 0,
+    description => "test jobset",
+    emailoverride => "",
+    enabled => 3,
+    enableemail => JSON::false,
+    errortime => undef,
+    errormsg => "",
+    fetcherrormsg => "",
+    flake => "",
+    hidden => JSON::false,
+    jobsetinputs => {
+      ofborg => {
+        name => "ofborg",
+        type => "git",
+        emailresponsible => JSON::false,
+        jobsetinputalts => [
+          "https://github.com/NixOS/ofborg.git released"
+        ]
+      }
+    },
+    keepnr => 1,
+    lastcheckedtime => undef,
+    name => "job",
+    nixexprinput => "ofborg",
+    nixexprpath => "release.nix",
+    project => "tests",
+    schedulingshares => 50,
+    starttime => undef,
+    triggertime => undef,
+    type => 0
+  });
+};
+
+
+subtest 'Delete jobset "job"' => sub {
+  my $jobsetinfo = request(DELETE '/jobset/tests/job',
+      Accept => 'application/json',
+      Cookie => $cookie
+  );
+  ok($jobsetinfo->is_success);
+
+  # Jobset "job" should no longer exist.
+  $jobsetinfo = request(GET '/jobset/tests/job',
+      Accept => 'application/json',
+  );
+  ok(!$jobsetinfo->is_success);
+};
+
+done_testing;