From 7f16c0d2430d570e94cff577a6b0bead194f9f90 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 2 Sep 2020 12:34:34 -0400 Subject: [PATCH] declarative projects: support fully static, declarative configuration --- doc/manual/declarative-projects.xml | 107 +++++++++++++++++++++++++++- src/lib/Hydra/Helper/AddBuilds.pm | 31 ++++---- src/script/hydra-eval-jobset | 24 ++++++- 3 files changed, 143 insertions(+), 19 deletions(-) diff --git a/doc/manual/declarative-projects.xml b/doc/manual/declarative-projects.xml index 1c9e76c1..df909a89 100644 --- a/doc/manual/declarative-projects.xml +++ b/doc/manual/declarative-projects.xml @@ -2,10 +2,110 @@ xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="sec-declarative-projects"> - Declarative projects +Declarative projects + + + Hydra supports declaratively configuring a project's jobsets. This + configuration can be done statically, or generated by a build job. + + + + Hydra will treat the project's declarative input as a static definition + if and only if the spec file contains a dictionary of dictionaries. + If the value of any key in the spec is not a dictionary, it will + treat the spec as a generated declarative spec. + + +
+ + Static, Declarative Projects - Hydra also supports declarative projects, where jobsets are generated - and configured automatically from specification files instead of being + Hydra supports declarative projects, where jobsets are configured + from a static JSON document in a repository. + + + + To configure a static declarative project, take the following steps: + + + + + Create a Hydra-fetchable source like a Git repository or local path. + + + + + In that source, create a file called spec.json, + and add the specification for all of the jobsets. Each key is jobset + and each value is a jobset's specification. For example: + + +{ + "nixpkgs": { + "enabled": 1, + "hidden": false, + "description": "Nixpkgs", + "nixexprinput": "nixpkgs", + "nixexprpath": "pkgs/top-level/release.nix", + "checkinterval": 300, + "schedulingshares": 100, + "enableemail": false, + "emailoverride": "", + "keepnr": 3, + "inputs": { + "nixpkgs": { + "type": "git", + "value": "git://github.com/NixOS/nixpkgs.git master", + "emailresponsible": false + } + } + }, + "nixos": { + "enabled": 1, + "hidden": false, + "description": "NixOS: Small Evaluation", + "nixexprinput": "nixpkgs", + "nixexprpath": "nixos/release-small.nix", + "checkinterval": 300, + "schedulingshares": 100, + "enableemail": false, + "emailoverride": "", + "keepnr": 3, + "inputs": { + "nixpkgs": { + "type": "git", + "value": "git://github.com/NixOS/nixpkgs.git master", + "emailresponsible": false + } + } + } +} + + + + + + Create a new project, and set the project's declarative input type, + declarative input value, and declarative spec file to point to the + source and JSON file you created in step 2. + + + + + Hydra will create a special jobset named .jobsets. + When the .jobsets jobset is evaluated, this static + specification will be used for configuring the rest of the project's + jobsets. + +
+ +
+ + Generated, Declarative Projects + + + Hydra also supports generated declarative projects, where jobsets are + configured automatically from specification files instead of being managed through the UI. A jobset specification is a JSON object containing the configuration of the jobset, for example: @@ -94,3 +194,4 @@
+ diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 7a6fcef9..ae75e1fa 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -21,6 +21,7 @@ our @ISA = qw(Exporter); our @EXPORT = qw( updateDeclarativeJobset handleDeclarativeJobsetBuild + handleDeclarativeJobsetJson ); @@ -65,6 +66,22 @@ sub updateDeclarativeJobset { }); }; +sub handleDeclarativeJobsetJson { + my ($db, $project, $declSpec) = @_; + $db->txn_do(sub { + my @kept = keys %$declSpec; + push @kept, ".jobsets"; + $project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 }); + while ((my $jobsetName, my $spec) = each %$declSpec) { + eval { + updateDeclarativeJobset($db, $project, $jobsetName, $spec); + 1; + } or do { + print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n"; + } + } + }); +} sub handleDeclarativeJobsetBuild { my ($db, $project, $build) = @_; @@ -82,19 +99,7 @@ sub handleDeclarativeJobsetBuild { }; my $declSpec = decode_json($declText); - $db->txn_do(sub { - my @kept = keys %$declSpec; - push @kept, ".jobsets"; - $project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 }); - while ((my $jobsetName, my $spec) = each %$declSpec) { - eval { - updateDeclarativeJobset($db, $project, $jobsetName, $spec); - 1; - } or do { - print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n"; - } - } - }); + handleDeclarativeJobsetJson($db, $project, $declSpec); 1; } or do { # note the error in the database in the case eval fails for whatever reason diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index ea336bfc..ff76f4ef 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -569,10 +569,28 @@ sub checkJobsetWrapped { eval { $declSpec = decode_json($declText); }; + die "Declarative specification file $declFile not valid JSON: $@\n" if $@; - updateDeclarativeJobset($db, $project, ".jobsets", $declSpec); - $jobset->discard_changes; - $inputInfo->{"declInput"} = [ $declInput ]; + + if (ref $declSpec eq "HASH") { + if (grep ref $_ eq "HASH", values %$declSpec) { + # Since all of its keys are hashes, assume the json document + # itself is the entire set of jobs + handleDeclarativeJobsetJson($db, $project, $declSpec); + $db->txn_do(sub { + $jobset->update({ lastcheckedtime => time, fetcherrormsg => undef }); + }); + return; + } else { + # Update the jobset with the spec's inputs, and the continue + # evaluating the .jobsets jobset. + updateDeclarativeJobset($db, $project, ".jobsets", $declSpec); + $jobset->discard_changes; + $inputInfo->{"declInput"} = [ $declInput ]; + } + } else { + die "Declarative specification file $declFile is not a dictionary" + } } # Fetch all values for all inputs.