ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Merge pull request #804 from grahamc/fully-static-jobsets

Browse source 
declarative projects: support fully static, declarative configuration
This commit is contained in:
Eelco Dolstra 2020-09-02 21:09:01 +02:00 committed by GitHub
parent dd0145a7f4 7f16c0d243
commit e707990e2d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 143 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
doc/manual/declarative-projects.xml
View file

@ -2,10 +2,110 @@
xmlns:xlink="http://www.w3.org/1999/xlink"
xml:id="sec-declarative-projects">
<title>Declarative projects</title>
<title>Declarative projects</title>
<para>
Hydra supports declaratively configuring a project's jobsets. This
configuration can be done statically, or generated by a build job.
</para>
<note><para>
Hydra will treat the project's declarative input as a static definition
if and only if the spec file contains a dictionary of dictionaries.
If the value of any key in the spec is not a dictionary, it will
treat the spec as a generated declarative spec.
</para></note>
<section xml:id="sec-static-declarative-projects">
<title>Static, Declarative Projects</title>
<para>
Hydra also supports declarative projects, where jobsets are generated
and configured automatically from specification files instead of being
Hydra supports declarative projects, where jobsets are configured
from a static JSON document in a repository.
</para>
<para>
To configure a static declarative project, take the following steps:
</para>
<orderedlist numeration="arabic" spacing="compact">
<listitem>
<para>
Create a Hydra-fetchable source like a Git repository or local path.
</para>
</listitem>
<listitem>
<para>
In that source, create a file called <filename>spec.json</filename>,
and add the specification for all of the jobsets. Each key is jobset
and each value is a jobset's specification. For example:
<programlisting language="json">
{
"nixpkgs": {
"enabled": 1,
"hidden": false,
"description": "Nixpkgs",
"nixexprinput": "nixpkgs",
"nixexprpath": "pkgs/top-level/release.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"nixpkgs": {
"type": "git",
"value": "git://github.com/NixOS/nixpkgs.git master",
"emailresponsible": false
}
}
},
"nixos": {
"enabled": 1,
"hidden": false,
"description": "NixOS: Small Evaluation",
"nixexprinput": "nixpkgs",
"nixexprpath": "nixos/release-small.nix",
"checkinterval": 300,
"schedulingshares": 100,
"enableemail": false,
"emailoverride": "",
"keepnr": 3,
"inputs": {
"nixpkgs": {
"type": "git",
"value": "git://github.com/NixOS/nixpkgs.git master",
"emailresponsible": false
}
}
}
}
</programlisting>
</para>
</listitem>
<listitem>
<para>
Create a new project, and set the project's declarative input type,
declarative input value, and declarative spec file to point to the
source and JSON file you created in step 2.
</para>
</listitem>
</orderedlist>
<para>
Hydra will create a special jobset named <literal>.jobsets</literal>.
When the <literal>.jobsets</literal> jobset is evaluated, this static
specification will be used for configuring the rest of the project's
jobsets.
</para>
</section>
<section xml:id="sec-generated-declarative-projects">
<title>Generated, Declarative Projects</title>
<para>
<para>
Hydra also supports generated declarative projects, where jobsets are
configured automatically from specification files instead of being
managed through the UI. A jobset specification is a JSON object
containing the configuration of the jobset, for example:
</para>
@ -94,3 +194,4 @@
</listitem>
</orderedlist>
</section>
</section>

31
src/lib/Hydra/Helper/AddBuilds.pm
View file

@ -21,6 +21,7 @@ our @ISA = qw(Exporter);
our @EXPORT = qw(
updateDeclarativeJobset
handleDeclarativeJobsetBuild
handleDeclarativeJobsetJson
);
@ -65,6 +66,22 @@ sub updateDeclarativeJobset {
});
};
sub handleDeclarativeJobsetJson {
my ($db, $project, $declSpec) = @_;
$db->txn_do(sub {
my @kept = keys %$declSpec;
push @kept, ".jobsets";
$project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 });
while ((my $jobsetName, my $spec) = each %$declSpec) {
eval {
updateDeclarativeJobset($db, $project, $jobsetName, $spec);
1;
} or do {
print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";
}
}
});
}
sub handleDeclarativeJobsetBuild {
my ($db, $project, $build) = @_;
@ -82,19 +99,7 @@ sub handleDeclarativeJobsetBuild {
};
my $declSpec = decode_json($declText);
$db->txn_do(sub {
my @kept = keys %$declSpec;
push @kept, ".jobsets";
$project->jobsets->search({ name => { "not in" => \@kept } })->update({ enabled => 0, hidden => 1 });
while ((my $jobsetName, my $spec) = each %$declSpec) {
eval {
updateDeclarativeJobset($db, $project, $jobsetName, $spec);
1;
} or do {
print STDERR "ERROR: failed to process declarative jobset ", $project->name, ":${jobsetName}, ", $@, "\n";
}
}
});
handleDeclarativeJobsetJson($db, $project, $declSpec);
1;
} or do {
# note the error in the database in the case eval fails for whatever reason

18
src/script/hydra-eval-jobset
View file

@ -569,11 +569,29 @@ sub checkJobsetWrapped {
eval {
$declSpec = decode_json($declText);
};
die "Declarative specification file $declFile not valid JSON: $@\n" if $@;
if (ref $declSpec eq "HASH") {
if (grep ref $_ eq "HASH", values %$declSpec) {
# Since all of its keys are hashes, assume the json document
# itself is the entire set of jobs
handleDeclarativeJobsetJson($db, $project, $declSpec);
$db->txn_do(sub {
$jobset->update({ lastcheckedtime => time, fetcherrormsg => undef });
});
return;
} else {
# Update the jobset with the spec's inputs, and the continue
# evaluating the .jobsets jobset.
updateDeclarativeJobset($db, $project, ".jobsets", $declSpec);
$jobset->discard_changes;
$inputInfo->{"declInput"} = [ $declInput ];
}
} else {
die "Declarative specification file $declFile is not a dictionary"
}
}
# Fetch all values for all inputs.
my $checkoutStart = clock_gettime(CLOCK_MONOTONIC);