From 6d4ccff43c41adaf6e4b2b9bced7243bc2f6e97b Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Tue, 16 Jul 2024 04:22:41 +0200 Subject: [PATCH] hydra-eval-jobset: use nix-eval-jobs instead of hydra-eval-jobs --- flake.nix | 3 +- package.nix | 3 ++ src/script/hydra-eval-jobset | 102 +++++++++++++++++++++-------------- t/evaluator/evaluate-meta.t | 22 ++++++++ t/jobs/meta.nix | 17 ++++++ 5 files changed, 107 insertions(+), 40 deletions(-) create mode 100644 t/evaluator/evaluate-meta.t create mode 100644 t/jobs/meta.nix diff --git a/flake.nix b/flake.nix index d0b3e9d3..859b418c 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ inputs.nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs"; inputs.nix-eval-jobs.inputs.lix.follows = "lix"; - outputs = { self, nixpkgs, lix }: + outputs = { self, nix-eval-jobs, nixpkgs, lix }: let systems = [ "x86_64-linux" "aarch64-linux" ]; forEachSystem = nixpkgs.lib.genAttrs systems; @@ -29,6 +29,7 @@ overlays.default = final: prev: { hydra = final.callPackage ./package.nix { inherit (final.lib) fileset; + nix-eval-jobs = nix-eval-jobs.packages.${final.system}.default; rawSrc = self; }; }; diff --git a/package.nix b/package.nix index 623bebeb..31a55b17 100644 --- a/package.nix +++ b/package.nix @@ -48,6 +48,7 @@ , xz , gnutar , gnused +, nix-eval-jobs , rpm , dpkg @@ -192,6 +193,7 @@ stdenv.mkDerivation (finalAttrs: { openldap postgresql_13 pixz + nix-eval-jobs ]; checkInputs = [ @@ -220,6 +222,7 @@ stdenv.mkDerivation (finalAttrs: { darcs gnused breezy + nix-eval-jobs ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 7ed7ebe8..de365a7d 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -357,22 +357,21 @@ sub evalJobs { my @cmd; if (defined $flakeRef) { - @cmd = ("hydra-eval-jobs", - "--flake", $flakeRef, - "--gc-roots-dir", getGCRootsDir, - "--max-jobs", 1); + @cmd = ("nix-eval-jobs", "--flake", $flakeRef . '#hydraJobs'); } else { my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] or die "cannot find the input containing the job expression\n"; - @cmd = ("hydra-eval-jobs", + @cmd = ("nix-eval-jobs", "<" . $nixExprInputName . "/" . $nixExprPath . ">", - "--gc-roots-dir", getGCRootsDir, - "--max-jobs", 1, inputsToArgs($inputInfo)); } - push @cmd, "--no-allow-import-from-derivation" if $config->{allow_import_from_derivation} // "true" ne "true"; + push @cmd, ("--gc-roots-dir", getGCRootsDir); + push @cmd, ("--max-jobs", 1); + push @cmd, "--meta"; + push @cmd, "--force-recurse"; + push @cmd, ("--option", "allow-import-from-derivation", "false") if $config->{allow_import_from_derivation} // "true" ne "true"; if (defined $ENV{'HYDRA_DEBUG'}) { sub escape { @@ -384,14 +383,23 @@ sub evalJobs { print STDERR "evaluator: @escaped\n"; } - (my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(21600, @cmd); - die "hydra-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) + (my $res, my $jobsJSONLines, my $stderr) = captureStdoutStderr(21600, @cmd); + die "nix-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n") if $res; print STDERR "$stderr"; - return decode_json($jobsJSON); + # XXX: take advantage of nix-eval-jobs's streaming instead of parsing everything in one block at + # the end. + my @jobs; + foreach my $line (split(/\n/, $jobsJSONLines)) { + last if $line eq ""; + + push(@jobs, decode_json($line)); + }; + + return @jobs; } @@ -420,7 +428,7 @@ sub checkBuild { my $firstOutputName = $outputNames[0]; my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName}; - my $jobName = $buildInfo->{jobName} or die; + my $jobName = $buildInfo->{attr} or die; my $drvPath = $buildInfo->{drvPath} or die; my $build; @@ -474,9 +482,30 @@ sub checkBuild { my $time = time(); - sub null { - my ($s) = @_; - return $s eq "" ? undef : $s; + sub getMeta { + my ($s, $def) = @_; + return ($s || "") eq "" ? $def : $s; + } + + sub getMetaStrings { + my ($v, $k, $acc) = @_; + my $t = ref $v; + + if ($t eq 'HASH') { + push @$acc, $v->{$k} if exists $v->{$k}; + } elsif ($t eq 'ARRAY') { + getMetaStrings($_, $k, $acc) foreach @$v; + } elsif (defined $v) { + push @$acc, $v; + } + } + + sub getMetaConcatStrings { + my ($v, $k) = @_; + + my @strings; + getMetaStrings($v, $k, \@strings); + return join(", ", @strings) || undef; } # Add the build to the database. @@ -484,19 +513,19 @@ sub checkBuild { { timestamp => $time , jobset_id => $jobset->id , job => $jobName - , description => null($buildInfo->{description}) - , license => null($buildInfo->{license}) - , homepage => null($buildInfo->{homepage}) - , maintainers => null($buildInfo->{maintainers}) - , maxsilent => $buildInfo->{maxSilent} - , timeout => $buildInfo->{timeout} - , nixname => $buildInfo->{nixName} + , description => getMeta($buildInfo->{meta}->{description}, undef) + , license => getMetaConcatStrings($buildInfo->{meta}->{license}, "shortName") + , homepage => getMeta($buildInfo->{meta}->{homepage}, undef) + , maintainers => getMetaConcatStrings($buildInfo->{meta}->{maintainers}, "email") + , maxsilent => getMeta($buildInfo->{meta}->{maxSilent}, 7200) + , timeout => getMeta($buildInfo->{meta}->{timeout}, 36000) + , nixname => $buildInfo->{name} , drvpath => $drvPath , system => $buildInfo->{system} - , priority => $buildInfo->{schedulingPriority} + , priority => getMeta($buildInfo->{meta}->{schedulingPriority}, 100) , finished => 0 , iscurrent => 1 - , ischannel => $buildInfo->{isChannel} + , ischannel => getMeta($buildInfo->{meta}->{isChannel}, 0) }); $build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} }) @@ -665,7 +694,7 @@ sub checkJobsetWrapped { return; } - # Hash the arguments to hydra-eval-jobs and check the + # Hash the arguments to nix-eval-jobs and check the # JobsetInputHashes to see if the previous evaluation had the same # inputs. If so, bail out. my @args = ($jobset->nixexprinput // "", $jobset->nixexprpath // "", inputsToArgs($inputInfo)); @@ -687,19 +716,18 @@ sub checkJobsetWrapped { # Evaluate the job expression. my $evalStart = clock_gettime(CLOCK_MONOTONIC); - my $jobs = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef); + my @jobs = evalJobs($project->name . ":" . $jobset->name, $inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef); my $evalStop = clock_gettime(CLOCK_MONOTONIC); if ($jobsetsJobset) { - my @keys = keys %$jobs; die "The .jobsets jobset must only have a single job named 'jobsets'" - unless (scalar @keys) == 1 && $keys[0] eq "jobsets"; + unless (scalar @jobs) == 1 && $jobs[0]->{attr} eq "jobsets"; } Net::Statsd::timing("hydra.evaluator.eval_time", int(($evalStop - $evalStart) * 1000)); if ($dryRun) { - foreach my $name (keys %{$jobs}) { - my $job = $jobs->{$name}; + foreach my $job (@jobs) { + my $name = $job->{attr}; if (defined $job->{drvPath}) { print STDERR "good job $name: $job->{drvPath}\n"; } else { @@ -709,11 +737,6 @@ sub checkJobsetWrapped { return; } - die "Jobset contains a job with an empty name. Make sure the jobset evaluates to an attrset of jobs.\n" - if defined $jobs->{""}; - - $jobs->{$_}->{jobName} = $_ for keys %{$jobs}; - my $jobOutPathMap = {}; my $jobsetChanged = 0; my $dbStart = clock_gettime(CLOCK_MONOTONIC); @@ -722,10 +745,10 @@ sub checkJobsetWrapped { # Store the error messages for jobs that failed to evaluate. my $evaluationErrorTime = time; my $evaluationErrorMsg = ""; - foreach my $job (values %{$jobs}) { + foreach my $job (@jobs) { next unless defined $job->{error}; $evaluationErrorMsg .= - ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") . + ($job->{attr} ne "" ? "in job ‘$job->{attr}’" : "at top-level") . ":\n" . $job->{error} . "\n\n"; } setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime); @@ -760,7 +783,7 @@ sub checkJobsetWrapped { }); # Schedule each successfully evaluated job. - foreach my $job (permute(values %{$jobs})) { + foreach my $job (permute(@jobs)) { next if defined $job->{error}; #print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; checkBuild($db, $jobset, $ev, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); @@ -801,7 +824,8 @@ sub checkJobsetWrapped { $drvPathToId{$x->{drvPath}} = $x; } - foreach my $job (values %{$jobs}) { + # XXX: dead code with nix-eval-jobs. To be removed. + foreach my $job (values @jobs) { next unless $job->{constituents}; if (defined $job->{error}) { diff --git a/t/evaluator/evaluate-meta.t b/t/evaluator/evaluate-meta.t new file mode 100644 index 00000000..9f546a7f --- /dev/null +++ b/t/evaluator/evaluate-meta.t @@ -0,0 +1,22 @@ +use feature 'unicode_strings'; +use strict; +use warnings; +use Setup; +use Test2::V0; + +my $ctx = test_context(); + +my $builds = $ctx->makeAndEvaluateJobset( + expression => "meta.nix", + build => 1 +); + +my $build = $builds->{"full-of-meta"}; + +is($build->finished, 1, "Build should be finished."); +is($build->description, "This is the description of the job.", "Wrong description extracted from the build."); +is($build->license, "MIT, BSD", "Wrong licenses extracted from the build."); +is($build->homepage, "https://example.com/", "Wrong homepage extracted from the build."); +is($build->maintainers, 'alice@example.com, bob@not.found', "Wrong maintainers extracted from the build."); + +done_testing; diff --git a/t/jobs/meta.nix b/t/jobs/meta.nix new file mode 100644 index 00000000..9204e384 --- /dev/null +++ b/t/jobs/meta.nix @@ -0,0 +1,17 @@ +with import ./config.nix; +{ + full-of-meta = + mkDerivation { + name = "full-of-meta"; + builder = ./empty-dir-builder.sh; + + meta = { + description = "This is the description of the job."; + license = [ { shortName = "MIT"; } "BSD" ]; + homepage = "https://example.com/"; + maintainers = [ "alice@example.com" { email = "bob@not.found"; } ]; + + outPath = "${placeholder "out"}"; + }; + }; +}