hydra-eval-jobset: use nix-eval-jobs instead of hydra-eval-jobs

This commit is contained in:
Pierre Bourdon 2024-07-16 04:22:41 +02:00
parent 684cc50d86
commit 1925d16619
Signed by: delroth
GPG key ID: 6FB80DCD84DA0F1C
5 changed files with 109 additions and 37 deletions

View file

@ -10,7 +10,7 @@
inputs.nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs"; inputs.nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs";
inputs.nix-eval-jobs.inputs.lix.follows = "lix"; inputs.nix-eval-jobs.inputs.lix.follows = "lix";
outputs = { self, nixpkgs, lix }: outputs = { self, nix-eval-jobs, nixpkgs, lix }:
let let
systems = [ "x86_64-linux" "aarch64-linux" ]; systems = [ "x86_64-linux" "aarch64-linux" ];
forEachSystem = nixpkgs.lib.genAttrs systems; forEachSystem = nixpkgs.lib.genAttrs systems;
@ -29,6 +29,7 @@
overlays.default = final: prev: { overlays.default = final: prev: {
hydra = final.callPackage ./package.nix { hydra = final.callPackage ./package.nix {
inherit (final.lib) fileset; inherit (final.lib) fileset;
nix-eval-jobs = nix-eval-jobs.packages.${final.system}.default;
rawSrc = self; rawSrc = self;
}; };
}; };

View file

@ -48,6 +48,7 @@
, xz , xz
, gnutar , gnutar
, gnused , gnused
, nix-eval-jobs
, rpm , rpm
, dpkg , dpkg
@ -192,6 +193,7 @@ stdenv.mkDerivation (finalAttrs: {
openldap openldap
postgresql_13 postgresql_13
pixz pixz
nix-eval-jobs
]; ];
checkInputs = [ checkInputs = [
@ -220,6 +222,7 @@ stdenv.mkDerivation (finalAttrs: {
darcs darcs
gnused gnused
breezy breezy
nix-eval-jobs
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
); );

View file

@ -357,22 +357,26 @@ sub evalJobs {
my @cmd; my @cmd;
if (defined $flakeRef) { if (defined $flakeRef) {
@cmd = ("hydra-eval-jobs", @cmd = ("nix-eval-jobs",
"--flake", $flakeRef, "--flake", $flakeRef . '#hydraJobs',
"--gc-roots-dir", getGCRootsDir, "--gc-roots-dir", getGCRootsDir,
"--max-jobs", 1); "--max-jobs", 1,
"--meta",
"--force-recurse");
} else { } else {
my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] my $nixExprInput = $inputInfo->{$nixExprInputName}->[0]
or die "cannot find the input containing the job expression\n"; or die "cannot find the input containing the job expression\n";
@cmd = ("hydra-eval-jobs", @cmd = ("nix-eval-jobs",
"<" . $nixExprInputName . "/" . $nixExprPath . ">", "<" . $nixExprInputName . "/" . $nixExprPath . ">",
"--gc-roots-dir", getGCRootsDir, "--gc-roots-dir", getGCRootsDir,
"--max-jobs", 1, "--max-jobs", 1,
"--meta",
"--force-recurse",
inputsToArgs($inputInfo)); inputsToArgs($inputInfo));
} }
push @cmd, "--no-allow-import-from-derivation" if $config->{allow_import_from_derivation} // "true" ne "true"; #push @cmd, "--no-allow-import-from-derivation" if $config->{allow_import_from_derivation} // "true" ne "true";
if (defined $ENV{'HYDRA_DEBUG'}) { if (defined $ENV{'HYDRA_DEBUG'}) {
sub escape { sub escape {
@ -384,14 +388,23 @@ sub evalJobs {
print STDERR "evaluator: @escaped\n"; print STDERR "evaluator: @escaped\n";
} }
(my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(21600, @cmd); (my $res, my $jobsJSONLines, my $stderr) = captureStdoutStderr(21600, @cmd);
die "hydra-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) die "nix-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
. ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n") . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n")
if $res; if $res;
print STDERR "$stderr"; print STDERR "$stderr";
return decode_json($jobsJSON); # XXX: take advantage of nix-eval-jobs's streaming instead of parsing everything in one block at
# the end.
my @jobs;
foreach my $line (split(/\n/, $jobsJSONLines)) {
last if $line eq "";
push(@jobs, decode_json($line));
};
return @jobs;
} }
@ -420,7 +433,7 @@ sub checkBuild {
my $firstOutputName = $outputNames[0]; my $firstOutputName = $outputNames[0];
my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName}; my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName};
my $jobName = $buildInfo->{jobName} or die; my $jobName = $buildInfo->{attr} or die;
my $drvPath = $buildInfo->{drvPath} or die; my $drvPath = $buildInfo->{drvPath} or die;
my $build; my $build;
@ -474,9 +487,30 @@ sub checkBuild {
my $time = time(); my $time = time();
sub null { sub getMeta {
my ($s) = @_; my ($s, $def) = @_;
return $s eq "" ? undef : $s; return ($s || "") eq "" ? $def : $s;
}
sub getMetaStrings {
my ($v, $k, $acc) = @_;
my $t = ref $v;
if ($t eq 'HASH') {
push @$acc, $v->{$k} if exists $v->{$k};
} elsif ($t eq 'ARRAY') {
getMetaStrings($_, $k, $acc) foreach @$v;
} elsif (defined $v) {
push @$acc, $v;
}
}
sub getMetaConcatStrings {
my ($v, $k) = @_;
my @strings;
getMetaStrings($v, $k, \@strings);
return join(", ", @strings) || undef;
} }
# Add the build to the database. # Add the build to the database.
@ -484,19 +518,19 @@ sub checkBuild {
{ timestamp => $time { timestamp => $time
, jobset_id => $jobset->id , jobset_id => $jobset->id
, job => $jobName , job => $jobName
, description => null($buildInfo->{description}) , description => getMeta($buildInfo->{meta}->{description}, undef)
, license => null($buildInfo->{license}) , license => getMetaConcatStrings($buildInfo->{meta}->{license}, "shortName")
, homepage => null($buildInfo->{homepage}) , homepage => getMeta($buildInfo->{meta}->{homepage}, undef)
, maintainers => null($buildInfo->{maintainers}) , maintainers => getMetaConcatStrings($buildInfo->{meta}->{maintainers}, "email")
, maxsilent => $buildInfo->{maxSilent} , maxsilent => getMeta($buildInfo->{meta}->{maxSilent}, 7200)
, timeout => $buildInfo->{timeout} , timeout => getMeta($buildInfo->{meta}->{timeout}, 36000)
, nixname => $buildInfo->{nixName} , nixname => $buildInfo->{name}
, drvpath => $drvPath , drvpath => $drvPath
, system => $buildInfo->{system} , system => $buildInfo->{system}
, priority => $buildInfo->{schedulingPriority} , priority => getMeta($buildInfo->{meta}->{schedulingPriority}, 100)
, finished => 0 , finished => 0
, iscurrent => 1 , iscurrent => 1
, ischannel => $buildInfo->{isChannel} , ischannel => getMeta($buildInfo->{meta}->{isChannel}, 0)
}); });
$build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} }) $build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} })
@ -665,7 +699,7 @@ sub checkJobsetWrapped {
return; return;
} }
# Hash the arguments to hydra-eval-jobs and check the # Hash the arguments to nix-eval-jobs and check the
# JobsetInputHashes to see if the previous evaluation had the same # JobsetInputHashes to see if the previous evaluation had the same
# inputs. If so, bail out. # inputs. If so, bail out.
my @args = ($jobset->nixexprinput // "", $jobset->nixexprpath // "", inputsToArgs($inputInfo)); my @args = ($jobset->nixexprinput // "", $jobset->nixexprpath // "", inputsToArgs($inputInfo));
@ -687,19 +721,18 @@ sub checkJobsetWrapped {
# Evaluate the job expression. # Evaluate the job expression.
my $evalStart = clock_gettime(CLOCK_MONOTONIC); my $evalStart = clock_gettime(CLOCK_MONOTONIC);
my $jobs = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef); my @jobs = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef);
my $evalStop = clock_gettime(CLOCK_MONOTONIC); my $evalStop = clock_gettime(CLOCK_MONOTONIC);
if ($jobsetsJobset) { if ($jobsetsJobset) {
my @keys = keys %$jobs;
die "The .jobsets jobset must only have a single job named 'jobsets'" die "The .jobsets jobset must only have a single job named 'jobsets'"
unless (scalar @keys) == 1 && $keys[0] eq "jobsets"; unless (scalar @jobs) == 1 && $jobs[0]->{attr} eq "jobsets";
} }
Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000)); Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000));
if ($dryRun) { if ($dryRun) {
foreach my $name (keys %{$jobs}) { foreach my $job (@jobs) {
my $job = $jobs->{$name}; my $name = $job->{attr};
if (defined $job->{drvPath}) { if (defined $job->{drvPath}) {
print STDERR "good job $name: $job->{drvPath}\n"; print STDERR "good job $name: $job->{drvPath}\n";
} else { } else {
@ -709,11 +742,6 @@ sub checkJobsetWrapped {
return; return;
} }
die "Jobset contains a job with an empty name. Make sure the jobset evaluates to an attrset of jobs.\n"
if defined $jobs->{""};
$jobs->{$_}->{jobName} = $_ for keys %{$jobs};
my $jobOutPathMap = {}; my $jobOutPathMap = {};
my $jobsetChanged = 0; my $jobsetChanged = 0;
my $dbStart = clock_gettime(CLOCK_MONOTONIC); my $dbStart = clock_gettime(CLOCK_MONOTONIC);
@ -722,10 +750,10 @@ sub checkJobsetWrapped {
# Store the error messages for jobs that failed to evaluate. # Store the error messages for jobs that failed to evaluate.
my $evaluationErrorTime = time; my $evaluationErrorTime = time;
my $evaluationErrorMsg = ""; my $evaluationErrorMsg = "";
foreach my $job (values %{$jobs}) { foreach my $job (@jobs) {
next unless defined $job->{error}; next unless defined $job->{error};
$evaluationErrorMsg .= $evaluationErrorMsg .=
($job->{jobName} ne "" ? "in job $job->{jobName}" : "at top-level") . ($job->{attr} ne "" ? "in job $job->{attr}" : "at top-level") .
":\n" . $job->{error} . "\n\n"; ":\n" . $job->{error} . "\n\n";
} }
setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime); setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
@ -760,7 +788,7 @@ sub checkJobsetWrapped {
}); });
# Schedule each successfully evaluated job. # Schedule each successfully evaluated job.
foreach my $job (permute(values %{$jobs})) { foreach my $job (permute(@jobs)) {
next if defined $job->{error}; next if defined $job->{error};
#print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; #print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
@ -801,7 +829,8 @@ sub checkJobsetWrapped {
$drvPathToId{$x->{drvPath}} = $x; $drvPathToId{$x->{drvPath}} = $x;
} }
foreach my $job (values %{$jobs}) { # XXX: dead code with nix-eval-jobs. To be removed.
foreach my $job (values @jobs) {
next unless $job->{constituents}; next unless $job->{constituents};
if (defined $job->{error}) { if (defined $job->{error}) {

View file

@ -0,0 +1,22 @@
use feature 'unicode_strings';
use strict;
use warnings;
use Setup;
use Test2::V0;
my $ctx = test_context();
my $builds = $ctx->makeAndEvaluateJobset(
expression => "meta.nix",
build => 1
);
my $build = $builds->{"full-of-meta"};
is($build->finished, 1, "Build should be finished.");
is($build->description, "This is the description of the job.", "Wrong description extracted from the build.");
is($build->license, "MIT, BSD", "Wrong licenses extracted from the build.");
is($build->homepage, "https://example.com/", "Wrong homepage extracted from the build.");
is($build->maintainers, 'alice@example.com, bob@not.found', "Wrong maintainers extracted from the build.");
done_testing;

17
t/jobs/meta.nix Normal file
View file

@ -0,0 +1,17 @@
with import ./config.nix;
{
full-of-meta =
mkDerivation {
name = "full-of-meta";
builder = ./empty-dir-builder.sh;
meta = {
description = "This is the description of the job.";
license = [ { shortName = "MIT"; } "BSD" ];
homepage = "https://example.com/";
maintainers = [ "alice@example.com" { email = "bob@not.found"; } ];
outPath = "${placeholder "out"}";
};
};
}