Merge pull request #804 from grahamc/fully-static-jobsets

declarative projects: support fully static, declarative configuration

doc/manual/declarative-projects.xml

@@ -2,10 +2,110 @@
          xmlns:xlink="http://www.w3.org/1999/xlink"
          xml:id="sec-declarative-projects">
 
-  <title>Declarative projects</title>
+<title>Declarative projects</title>
+
+<para>
+  Hydra supports declaratively configuring a project's jobsets. This
+  configuration can be done statically, or generated by a build job.
+</para>
+
+<note><para>
+  Hydra will treat the project's declarative input as a static definition
+  if and only if the spec file contains a dictionary of dictionaries.
+  If the value of any key in the spec is not a dictionary, it will
+  treat the spec as a generated declarative spec.
+</para></note>
+
+<section xml:id="sec-static-declarative-projects">
+
+  <title>Static, Declarative Projects</title>
   <para>
-    Hydra also supports declarative projects, where jobsets are generated
+    Hydra supports declarative projects, where jobsets are configured
-    and configured automatically from specification files instead of being
+    from a static JSON document in a repository.
+  </para>
+
+  <para>
+    To configure a static declarative project, take the following steps:
+  </para>
+  <orderedlist numeration="arabic" spacing="compact">
+    <listitem>
+      <para>
+        Create a Hydra-fetchable source like a Git repository or local path.
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        In that source, create a file called <filename>spec.json</filename>,
+        and add the specification for all of the jobsets. Each key is jobset
+        and each value is a jobset's specification. For example:
+
+        <programlisting language="json">
+{
+  "nixpkgs": {
+    "enabled": 1,
+    "hidden": false,
+    "description": "Nixpkgs",
+    "nixexprinput": "nixpkgs",
+    "nixexprpath": "pkgs/top-level/release.nix",
+    "checkinterval": 300,
+    "schedulingshares": 100,
+    "enableemail": false,
+    "emailoverride": "",
+    "keepnr": 3,
+    "inputs": {
+      "nixpkgs": {
+          "type": "git",
+          "value": "git://github.com/NixOS/nixpkgs.git master",
+          "emailresponsible": false
+      }
+    }
+  },
+  "nixos": {
+    "enabled": 1,
+    "hidden": false,
+    "description": "NixOS: Small Evaluation",
+    "nixexprinput": "nixpkgs",
+    "nixexprpath": "nixos/release-small.nix",
+    "checkinterval": 300,
+    "schedulingshares": 100,
+    "enableemail": false,
+    "emailoverride": "",
+    "keepnr": 3,
+    "inputs": {
+      "nixpkgs": {
+        "type": "git",
+        "value": "git://github.com/NixOS/nixpkgs.git master",
+        "emailresponsible": false
+      }
+    }
+  }
+}
+</programlisting>
+      </para>
+    </listitem>
+    <listitem>
+      <para>
+        Create a new project, and set the project's declarative input type,
+        declarative input value, and declarative spec file to point to the
+        source and JSON file you created in step 2.
+      </para>
+    </listitem>
+  </orderedlist>
+  <para>
+    Hydra will create a special jobset named <literal>.jobsets</literal>.
+    When the <literal>.jobsets</literal> jobset is evaluated, this static
+    specification will be used for configuring the rest of the project's
+    jobsets.
+  </para>
+</section>
+
+<section xml:id="sec-generated-declarative-projects">
+
+  <title>Generated, Declarative Projects</title>
+  <para>
+  <para>
+    Hydra also supports generated declarative projects, where jobsets are
+    configured automatically from specification files instead of being
     managed through the UI. A jobset specification is a JSON object
     containing the configuration of the jobset, for example:
   </para>
@@ -94,3 +194,4 @@
     </listitem>
   </orderedlist>
 </section>
+</section>

src/lib/Hydra/Helper/AddBuilds.pm

@@ -21,6 +21,7 @@ our @ISA = qw(Exporter);
 our @EXPORT = qw(
     updateDeclarativeJobset
     handleDeclarativeJobsetBuild
+    handleDeclarativeJobsetJson
 );
 
 
@@ -65,6 +66,22 @@ sub updateDeclarativeJobset {
     });
 };
 
+sub handleDeclarativeJobsetJson {
+    my ($db, $project, $declSpec) = @_;
+    $db->txn_do(sub {
+            my @kept = keys %$declSpec;
+            push @kept, ".jobsets";
+            $project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 });
+            while ((my $jobsetName, my $spec) = each %$declSpec) {
+                eval {
+                    updateDeclarativeJobset($db, $project, $jobsetName, $spec);
+                    1;
+                } or do {
+                    print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";
+                }
+            }
+        });
+}
 
 sub handleDeclarativeJobsetBuild {
     my ($db, $project, $build) = @_;
@@ -82,19 +99,7 @@ sub handleDeclarativeJobsetBuild {
         };
 
         my $declSpec = decode_json($declText);
-        $db->txn_do(sub {
+        handleDeclarativeJobsetJson($db, $project, $declSpec);
-            my @kept = keys %$declSpec;
-            push @kept, ".jobsets";
-            $project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 });
-            while ((my $jobsetName, my $spec) = each %$declSpec) {
-                eval {
-                    updateDeclarativeJobset($db, $project, $jobsetName, $spec);
-                    1;
-                } or do {
-                    print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";
-                }
-            }
-        });
         1;
     } or do {
         # note the error in the database in the case eval fails for whatever reason

src/script/hydra-eval-jobset

@@ -569,10 +569,28 @@ sub checkJobsetWrapped {
         eval {
             $declSpec = decode_json($declText);
         };
+
         die "Declarative specification file $declFile not valid JSON: $@\n" if $@;
-        updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
+
-        $jobset->discard_changes;
+        if (ref $declSpec eq "HASH") {
-        $inputInfo->{"declInput"} = [ $declInput ];
+            if (grep ref $_ eq "HASH", values %$declSpec) {
+                # Since all of its keys are hashes, assume the json document
+                # itself is the entire set of jobs
+                handleDeclarativeJobsetJson($db, $project, $declSpec);
+                $db->txn_do(sub {
+                    $jobset->update({ lastcheckedtime => time, fetcherrormsg => undef });
+                });
+                return;
+            } else {
+                # Update the jobset with the spec's inputs, and the continue
+                # evaluating the .jobsets jobset.
+                updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
+                $jobset->discard_changes;
+                $inputInfo->{"declInput"} = [ $declInput ];
+            }
+        } else {
+            die "Declarative specification file $declFile is not a dictionary"
+        }
     }
 
     # Fetch all values for all inputs.