From 15187b059be8ea15423222d7fa81fa938737787e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Feb 2020 15:12:14 +0100 Subject: [PATCH 01/15] Remove hydra-eval-guile-jobs This hasn't been used in a long time (Guix uses its own CI system), and it probably doesn't work anymore. (cherry picked from commit 23c9ca3e94669087d463642baea0cf35a0b8d72f) --- .gitignore | 1 - configure.ac | 13 +- release.nix | 1 - src/lib/Hydra/Controller/Jobset.pm | 5 - src/script/Makefile.am | 6 +- src/script/hydra-eval-guile-jobs.in | 249 ---------------------------- src/script/hydra-eval-jobset | 77 +++------ 7 files changed, 28 insertions(+), 324 deletions(-) delete mode 100644 src/script/hydra-eval-guile-jobs.in diff --git a/.gitignore b/.gitignore index e53ae6a8..5663a9d6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ Makefile.in /aclocal.m4 /missing /install-sh -/src/script/hydra-eval-guile-jobs /src/sql/hydra-postgresql.sql /src/sql/hydra-sqlite.sql /src/sql/tmp.sqlite diff --git a/configure.ac b/configure.ac index f992e49e..f99d4c9d 100644 --- a/configure.ac +++ b/configure.ac @@ -53,15 +53,6 @@ fi PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store]) -PKG_CHECK_MODULES([GUILE], [guile-2.0], [HAVE_GUILE=yes], [HAVE_GUILE=no]) - -if test "x$HAVE_GUILE" = xyes; then - AC_PATH_PROG([GUILE], [guile]) -else - GUILE="guile" -fi -AC_SUBST([GUILE]) - testPath="$(dirname $(type -p expr))" AC_SUBST(testPath) @@ -80,13 +71,11 @@ AC_CONFIG_FILES([ src/lib/Makefile src/root/Makefile src/script/Makefile - src/script/hydra-eval-guile-jobs tests/Makefile tests/jobs/config.nix ]) -AC_CONFIG_COMMANDS([executable-scripts], - [chmod +x src/script/hydra-eval-guile-jobs]) +AC_CONFIG_COMMANDS([executable-scripts], []) AC_CONFIG_HEADER([hydra-config.h]) diff --git a/release.nix b/release.nix index 408e3f69..cf17657e 100644 --- a/release.nix +++ b/release.nix @@ -129,7 +129,6 @@ rec { buildInputs = [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig sqlite libpqxx gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 libxslt - guile # optional, for Guile + Guix support perlDeps perl nix postgresql95 # for running the tests boost diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index fdf5e1c3..91e21dd4 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -223,11 +223,6 @@ sub updateJobset { error($c, "Cannot rename jobset to ‘$jobsetName’ since that identifier is already taken.") if $jobsetName ne $oldName && defined $c->stash->{project}->jobsets->find({ name => $jobsetName }); - # When the expression is in a .scm file, assume it's a Guile + Guix - # build expression. - my $exprType = - $c->stash->{params}->{"nixexprpath"} =~ /.scm$/ ? "guile" : "nix"; - my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c; my $enabled = int($c->stash->{params}->{enabled}); diff --git a/src/script/Makefile.am b/src/script/Makefile.am index 9deb6f29..466d3153 100644 --- a/src/script/Makefile.am +++ b/src/script/Makefile.am @@ -1,6 +1,5 @@ EXTRA_DIST = \ - $(distributable_scripts) \ - hydra-eval-guile-jobs.in + $(distributable_scripts) distributable_scripts = \ hydra-backfill-ids \ @@ -17,5 +16,4 @@ distributable_scripts = \ nix-prefetch-hg bin_SCRIPTS = \ - $(distributable_scripts) \ - hydra-eval-guile-jobs + $(distributable_scripts) diff --git a/src/script/hydra-eval-guile-jobs.in b/src/script/hydra-eval-guile-jobs.in deleted file mode 100644 index 8c5df125..00000000 --- a/src/script/hydra-eval-guile-jobs.in +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/sh -# Aside from this initial boilerplate, this is actually -*- scheme -*- code. -main="(module-ref (resolve-interface '(hydra-eval-guile-jobs)) 'eval-guile-jobs)" - -# Keep the host's GUILE_LOAD_PATH unchanged to allow the installed Guix to -# be used. This moves Guix modules possibly out of control, but solves -# bootstrapping issues. -# -# Use `--fresh-auto-compile' to ignore any available .go, and force -# recompilation. This is because checkouts in the store has mtime set to -# the epoch, and thus .go files look newer, even though they may not -# correspond. - -exec ${GUILE:-@GUILE@} --no-auto-compile --fresh-auto-compile \ - -l "$0" -c "(apply $main (cdr (command-line)))" "$@" -!# -;;; Copyright © 2012, 2013, 2014 Ludovic Courtès -;;; -;;; This file is part of Hydra. -;;; -;;; Hydra is free software: you can redistribute it and/or modify -;;; it under the terms of the GNU General Public License as published by -;;; the Free Software Foundation, either version 3 of the License, or -;;; (at your option) any later version. -;;; -;;; Hydra is distributed in the hope that it will be useful, -;;; but WITHOUT ANY WARRANTY; without even the implied warranty of -;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -;;; GNU General Public License for more details. -;;; -;;; You should have received a copy of the GNU General Public License -;;; along with Hydra. If not, see . - -(define-module (hydra-eval-guile-jobs) - #:use-module (sxml simple) - #:use-module (ice-9 match) - #:use-module (ice-9 regex) - #:use-module (srfi srfi-1) - #:use-module (srfi srfi-11) - #:export (job-evaluations->xml - eval-guile-jobs)) - -(define (guix-variable module name) - "Dynamically link variable NAME under Guix module MODULE and return it. -Note: this is used instead of `@', because when using `@' in an uncompiled -file, Guile tries to load the module directly as it reads the source, which -fails in our case, leading to the creation of empty (guix ...) modules." - ;; TODO: fail with an XML error description - (let ((m (resolve-interface `(guix ,module)))) - (module-ref m name))) - -(define (%derivation-system drv) - ;; XXX: Awful hack to workaround the fact that `derivation-system', which - ;; is a macro, cannot be referred to dynamically. - (struct-ref drv 3)) - -(define strip-store-path - (let* ((store (or (getenv "NIX_STORE_DIR") "/nix/store")) - (store-path-rx - (make-regexp (string-append "^.*" (regexp-quote store) - "/[^-]+-(.+)$")))) - (lambda (path) - (or (and=> (regexp-exec store-path-rx path) - (lambda (match) - (let ((path (match:substring match 1))) - path))) - path)))) - -(define (derivation-path->name drv) - "Return the base name of DRV, sans hash and `.drv' extension." - (let ((d (strip-store-path drv))) - (if (string-suffix? ".drv" d) - (string-drop-right d 4) - d))) - -(define (register-gc-root drv roots-dir) - "Register a permanent garbage collector root under ROOTS-DIR for DRV." - (let ((root (string-append roots-dir "/" (basename drv)))) - (unless (file-exists? root) - (symlink drv root)))) - -(define* (job-evaluations->sxml jobs - #:key gc-roots-dir) - "Return the hydra-eval-jobs SXML form for the result of JOBS, a list of -symbol/thunk pairs." - `(*TOP* - (*PI* xml "version='1.0' encoding='utf-8'") - "\n" - (jobs "\n" - ,@(map (match-lambda - (((? symbol? name) . (? thunk? thunk)) - (let* ((result (save-module-excursion - (lambda () - (set-current-module %user-module) - (with-output-to-port (%make-void-port "w") - thunk)))) - (drv (assoc-ref result 'derivation))) - (define (opt-attr xml-name name) - (match (assoc name result) - ((_ . value) - `((,xml-name ,value))) - (_ - '()))) - - (when gc-roots-dir - ;; Register DRV as a GC root so that it's not collected by - ;; the time 'hydra-queue-runner' attempts to build it. - (register-gc-root drv gc-roots-dir)) - - ;; XXX: Add tags? - `(job (@ (jobName ,name) - (drvPath ,drv) - ,@(opt-attr 'homepage 'home-page) - (license - ,(let loop ((license (assoc-ref result 'license))) - (match license - ((? struct?) - (struct-ref license 0)) - ((l ...) - (string-join (map loop l))) - (_ "")))) - ,@(opt-attr 'description 'description) - (maintainers - ,(string-join (or (assoc-ref result 'maintainers) - '()) - ", ")) - (maxSilent - ,(number->string (or (assoc-ref result - 'max-silent-time) - 3600))) - (timeout - ,(number->string (or (assoc-ref result 'timeout) - 72000))) - (nixName ,(derivation-path->name drv)) - (schedulingPriority - ,(number->string (or (assoc-ref result - 'scheduling-priority) - 10))) - (system - ,(call-with-input-file drv - (compose %derivation-system - (guix-variable 'derivations - 'read-derivation))))) - ;; Resolve Guix modules lazily. - ,(map (match-lambda - ((name . path) - `(output (@ (name ,name) (path ,path))))) - ((guix-variable 'derivations - 'derivation-path->output-paths) - drv)) - - "\n")))) - jobs)))) - -(define* (job-evaluations->xml jobs port - #:key gc-roots-dir) - (set-port-encoding! port "UTF-8") - (sxml->xml (job-evaluations->sxml jobs #:gc-roots-dir gc-roots-dir) - port)) - - -;;; -;;; Command-line entry point. -;;; - -(define (parse-arguments args) - "Traverse ARGS, a list of command-line arguments compatible with -`hydra-eval-jobs', and return the name of the file that defines the jobs, an -expression that returns the entry point in that file (a unary procedure), the -list of name/value pairs passed to that entry point, as well as a GC root -directory or #f." - (define (module-directory dir) - (let ((d (string-append dir "/share/guile/site/2.0"))) - (if (file-exists? d) - d - dir))) - - (let loop ((args args) - (result '()) - (file #f) - (entry 'hydra-jobs) - (roots-dir #f)) - (match args - (() - (if (not file) - (error "hydra-eval-guile-jobs: no expression file given") - (values file entry (reverse result) roots-dir))) - (("-I" name=dir rest ...) - (let* ((dir (match (string-tokenize name=dir - (char-set-complement (char-set - #\=))) - ((_ dir) dir) - ((dir) dir))) - (dir* (module-directory dir))) - (format (current-error-port) "adding `~a' to the load path~%" dir*) - (set! %load-path (cons dir* %load-path)) - (set! %load-compiled-path (cons dir* %load-compiled-path))) - (loop rest result file entry roots-dir)) - (("--argstr" name value rest ...) - (loop rest (alist-cons (string->symbol name) value result) - file entry roots-dir)) - (("--arg" name expr rest ...) - (let ((value (eval (call-with-input-string expr read) - (current-module)))) - (loop rest (alist-cons (string->symbol name) value result) - file entry roots-dir))) - (("--gc-roots-dir" dir rest ...) - (loop rest result file entry dir)) - (("-j" _ rest ...) ; XXX: what's this? - (loop rest result file entry roots-dir)) - (("--entry" expr rest ...) ; entry point, like `guile -e' - (let ((expr (call-with-input-string expr read))) - (loop rest result file expr roots-dir))) - ((file rest ...) ; source file that defines the jobs - (loop rest result file entry roots-dir)) - (_ - (error "hydra-eval-guile-jobs: invalid arguments" args))))) - -(define %user-module - ;; Hydra user module. - ;; TODO: Make it a sandbox. - (let ((m (make-module))) - (beautify-user-module! m) - m)) - -(define (eval-guile-jobs . args) - (setlocale LC_ALL "") - - (let-values (((file entry args gc-roots-dir) - (parse-arguments args))) - - (save-module-excursion - (lambda () - (set-current-module %user-module) - - ;; The standard output must contain only XML. - (with-output-to-port (%make-void-port "w") - (lambda () - (primitive-load file))))) - - (let* ((entry (eval entry %user-module)) - (store ((guix-variable 'store 'open-connection))) - (jobs (entry store args))) - (unless (string? gc-roots-dir) - (format (current-error-port) - "warning: --gc-roots-dir not specified~%")) - - (job-evaluations->xml jobs (current-output-port) - #:gc-roots-dir gc-roots-dir)))) diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 97da1084..6ebd59f1 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -264,53 +264,31 @@ sub fetchInput { sub booleanToString { - my ($exprType, $value) = @_; - my $result; - if ($exprType eq "guile") { - if ($value eq "true") { - $result = "#t"; - } else { - $result = "#f"; - } - $result = $value; - } else { - $result = $value; - } - return $result; + my ($value) = @_; + return $value; } sub buildInputToString { - my ($exprType, $input) = @_; - my $result; - if ($exprType eq "guile") { - $result = "'((file-name . \"" . ${input}->{storePath} . "\")" . - (defined $input->{revision} ? "(revision . \"" . $input->{revision} . "\")" : "") . - (defined $input->{revCount} ? "(revision-count . " . $input->{revCount} . ")" : "") . - (defined $input->{gitTag} ? "(git-tag . \"" . $input->{gitTag} . "\")" : "") . - (defined $input->{shortRev} ? "(short-revision . \"" . $input->{shortRev} . "\")" : "") . - (defined $input->{version} ? "(version . \"" . $input->{version} . "\")" : "") . - ")"; - } else { - $result = "{ outPath = builtins.storePath " . $input->{storePath} . "" . - "; inputType = \"" . $input->{type} . "\"" . - (defined $input->{uri} ? "; uri = \"" . $input->{uri} . "\"" : "") . - (defined $input->{revNumber} ? "; rev = " . $input->{revNumber} . "" : "") . - (defined $input->{revision} ? "; rev = \"" . $input->{revision} . "\"" : "") . - (defined $input->{revCount} ? "; revCount = " . $input->{revCount} . "" : "") . - (defined $input->{gitTag} ? "; gitTag = \"" . $input->{gitTag} . "\"" : "") . - (defined $input->{shortRev} ? "; shortRev = \"" . $input->{shortRev} . "\"" : "") . - (defined $input->{version} ? "; version = \"" . $input->{version} . "\"" : "") . - (defined $input->{outputName} ? "; outputName = \"" . $input->{outputName} . "\"" : "") . - (defined $input->{drvPath} ? "; drvPath = builtins.storePath " . $input->{drvPath} . "" : "") . - ";}"; - } - return $result; + my ($input) = @_; + return + "{ outPath = builtins.storePath " . $input->{storePath} . "" . + "; inputType = \"" . $input->{type} . "\"" . + (defined $input->{uri} ? "; uri = \"" . $input->{uri} . "\"" : "") . + (defined $input->{revNumber} ? "; rev = " . $input->{revNumber} . "" : "") . + (defined $input->{revision} ? "; rev = \"" . $input->{revision} . "\"" : "") . + (defined $input->{revCount} ? "; revCount = " . $input->{revCount} . "" : "") . + (defined $input->{gitTag} ? "; gitTag = \"" . $input->{gitTag} . "\"" : "") . + (defined $input->{shortRev} ? "; shortRev = \"" . $input->{shortRev} . "\"" : "") . + (defined $input->{version} ? "; version = \"" . $input->{version} . "\"" : "") . + (defined $input->{outputName} ? "; outputName = \"" . $input->{outputName} . "\"" : "") . + (defined $input->{drvPath} ? "; drvPath = builtins.storePath " . $input->{drvPath} . "" : "") . + ";}"; } sub inputsToArgs { - my ($inputInfo, $exprType) = @_; + my ($inputInfo) = @_; my @res = (); foreach my $input (sort keys %{$inputInfo}) { @@ -327,14 +305,12 @@ sub inputsToArgs { push @res, "--argstr", $input, $alt->{value}; } elsif ($alt->{type} eq "boolean") { - push @res, "--arg", $input, booleanToString($exprType, $alt->{value}); + push @res, "--arg", $input, booleanToString($alt->{value}); } elsif ($alt->{type} eq "nix") { - die "input type ‘nix’ only supported for Nix-based jobsets\n" unless $exprType eq "nix"; push @res, "--arg", $input, $alt->{value}; } elsif ($alt->{type} eq "eval") { - die "input type ‘eval’ only supported for Nix-based jobsets\n" unless $exprType eq "nix"; my $s = "{ "; # FIXME: escape $_. But dots should not be escaped. $s .= "$_ = builtins.storePath ${\$alt->{jobs}->{$_}}; " @@ -343,7 +319,7 @@ sub inputsToArgs { push @res, "--arg", $input, $s; } else { - push @res, "--arg", $input, buildInputToString($exprType, $alt); + push @res, "--arg", $input, buildInputToString($alt); } } @@ -352,18 +328,16 @@ sub inputsToArgs { sub evalJobs { - my ($inputInfo, $exprType, $nixExprInputName, $nixExprPath) = @_; + my ($inputInfo, $nixExprInputName, $nixExprPath) = @_; my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] or die "cannot find the input containing the job expression\n"; - my $evaluator = ($exprType eq "guile") ? "hydra-eval-guile-jobs" : "hydra-eval-jobs"; - - my @cmd = ($evaluator, + my @cmd = ("hydra-eval-jobs", "<" . $nixExprInputName . "/" . $nixExprPath . ">", "--gc-roots-dir", getGCRootsDir, "-j", 1, - inputsToArgs($inputInfo, $exprType)); + inputsToArgs($inputInfo)); if (defined $ENV{'HYDRA_DEBUG'}) { sub escape { @@ -376,7 +350,7 @@ sub evalJobs { } (my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(21600, @cmd); - die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) + die "hydra-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n") if $res; @@ -595,7 +569,6 @@ sub checkJobsetWrapped { $jobset->discard_changes; $inputInfo->{"declInput"} = [ $declInput ]; } - my $exprType = $jobset->nixexprpath =~ /.scm$/ ? "guile" : "nix"; # Fetch all values for all inputs. my $checkoutStart = clock_gettime(CLOCK_MONOTONIC); @@ -621,7 +594,7 @@ sub checkJobsetWrapped { # Hash the arguments to hydra-eval-jobs and check the # JobsetInputHashes to see if the previous evaluation had the same # inputs. If so, bail out. - my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo, $exprType)); + my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo)); my $argsHash = sha256_hex("@args"); my $prevEval = getPrevJobsetEval($db, $jobset, 0); if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun && !$jobset->forceeval) { @@ -636,7 +609,7 @@ sub checkJobsetWrapped { # Evaluate the job expression. my $evalStart = clock_gettime(CLOCK_MONOTONIC); - my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath); + my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath); my $evalStop = clock_gettime(CLOCK_MONOTONIC); if ($jobsetsJobset) { From 2a50daa3774c521a08c0443242ee3f80715a5189 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2020 16:33:25 +0100 Subject: [PATCH 02/15] Update aggregate handling (cherry picked from commit cf961ac8933b76d9f0a3ac3eb49cc08879f5bcc9) --- src/script/hydra-eval-jobset | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 6ebd59f1..cdb09b74 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -697,7 +697,7 @@ sub checkJobsetWrapped { foreach my $job (values %{$jobs}) { next unless $job->{constituents}; my $x = $drvPathToId{$job->{drvPath}} or die; - foreach my $drvPath (split / /, $job->{constituents}) { + foreach my $drvPath (@{$job->{constituents}}) { my $constituent = $drvPathToId{$drvPath}; if (defined $constituent) { $db->resultset('AggregateConstituents')->update_or_create({aggregate => $x->{id}, constituent => $constituent->{id}}); From b790a007296db1994a514b68d63694f24a39137e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 21:22:52 +0200 Subject: [PATCH 03/15] Disable deprecation warnings (cherry picked from commit 950e8bef6cd90befcf14e96826053d1d154e39fe) --- src/hydra-evaluator/Makefile.am | 2 +- src/hydra-queue-runner/Makefile.am | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hydra-evaluator/Makefile.am b/src/hydra-evaluator/Makefile.am index 161f8567..73638cfe 100644 --- a/src/hydra-evaluator/Makefile.am +++ b/src/hydra-evaluator/Makefile.am @@ -2,4 +2,4 @@ bin_PROGRAMS = hydra-evaluator hydra_evaluator_SOURCES = hydra-evaluator.cc hydra_evaluator_LDADD = $(NIX_LIBS) -lpqxx -hydra_evaluator_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra +hydra_evaluator_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am index b360faed..1726d0df 100644 --- a/src/hydra-queue-runner/Makefile.am +++ b/src/hydra-queue-runner/Makefile.am @@ -4,4 +4,4 @@ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.c builder.cc build-result.cc build-remote.cc \ build-result.hh counter.hh token-server.hh state.hh db.hh hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra +hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations From e4f5156c414050274a9ea6db4eab9a2f334a79d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Dec 2019 22:49:26 +0100 Subject: [PATCH 04/15] Build against nix-master (cherry picked from commit e7f2139e251cb73195eea6fb84e2a6167b4db968) --- src/hydra-eval-jobs/Makefile.am | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 5 +- src/hydra-queue-runner/Makefile.am | 2 +- src/hydra-queue-runner/build-remote.cc | 76 +++++++++-------- src/hydra-queue-runner/build-result.cc | 28 +++---- src/hydra-queue-runner/builder.cc | 44 +++++----- src/hydra-queue-runner/dispatcher.cc | 4 +- src/hydra-queue-runner/hydra-queue-runner.cc | 29 ++++--- src/hydra-queue-runner/queue-monitor.cc | 87 +++++++++++--------- src/hydra-queue-runner/state.hh | 20 ++--- src/hydra-queue-runner/token-server.hh | 2 +- src/libhydra/db.hh | 2 +- src/libhydra/hydra-config.hh | 4 +- 13 files changed, 166 insertions(+), 139 deletions(-) diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am index 7a4e9c91..b41a4eb8 100644 --- a/src/hydra-eval-jobs/Makefile.am +++ b/src/hydra-eval-jobs/Makefile.am @@ -1,5 +1,5 @@ bin_PROGRAMS = hydra-eval-jobs hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc -hydra_eval_jobs_LDADD = $(NIX_LIBS) +hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixrust hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index ce6967b5..8abdea7e 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -147,8 +147,9 @@ static void findJobsWrapped(EvalState & state, JSONObject & top, done. */ auto localStore = state.store.dynamic_pointer_cast(); if (gcRootsDir != "" && localStore) { - Path root = gcRootsDir + "/" + baseNameOf(drvPath); - if (!pathExists(root)) localStore->addPermRoot(drvPath, root, false); + Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); + if (!pathExists(root)) + localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false); } auto res2 = res.object("outputs"); diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am index 1726d0df..af95a3f9 100644 --- a/src/hydra-queue-runner/Makefile.am +++ b/src/hydra-queue-runner/Makefile.am @@ -3,5 +3,5 @@ bin_PROGRAMS = hydra-queue-runner hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \ builder.cc build-result.cc build-remote.cc \ build-result.hh counter.hh token-server.hh state.hh db.hh -hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx +hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lnixrust hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 81692849..6070bd4e 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -82,10 +82,10 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil static void copyClosureTo(std::timed_mutex & sendMutex, ref destStore, - FdSource & from, FdSink & to, const PathSet & paths, + FdSource & from, FdSink & to, const StorePathSet & paths, bool useSubstitutes = false) { - PathSet closure; + StorePathSet closure; for (auto & path : paths) destStore->computeFSClosure(path, closure); @@ -94,20 +94,21 @@ static void copyClosureTo(std::timed_mutex & sendMutex, ref destStore, garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log - to << cmdQueryValidPaths << 1 << useSubstitutes << closure; + to << cmdQueryValidPaths << 1 << useSubstitutes; + writeStorePaths(*destStore, to, closure); to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = readStorePaths(*destStore, from); + auto present = readStorePaths(*destStore, from); if (present.size() == closure.size()) return; - Paths sorted = destStore->topoSortPaths(closure); + auto sorted = destStore->topoSortPaths(closure); - Paths missing; + StorePathSet missing; for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) - if (present.find(*i) == present.end()) missing.push_back(*i); + if (!present.count(*i)) missing.insert(i->clone()); printMsg(lvlDebug, format("sending %1% missing paths") % missing.size()); @@ -131,7 +132,7 @@ void State::buildRemote(ref destStore, { assert(BuildResult::TimedOut == 8); - string base = baseNameOf(step->drvPath); + string base(step->drvPath.to_string()); result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2); AutoDelete autoDelete(result.logFile, false); @@ -217,22 +218,22 @@ void State::buildRemote(ref destStore, outputs of the input derivations. */ updateStep(ssSendingInputs); - PathSet inputs; - BasicDerivation basicDrv(step->drv); + StorePathSet inputs; + BasicDerivation basicDrv(*step->drv); if (sendDerivation) - inputs.insert(step->drvPath); + inputs.insert(step->drvPath.clone()); else - for (auto & p : step->drv.inputSrcs) - inputs.insert(p); + for (auto & p : step->drv->inputSrcs) + inputs.insert(p.clone()); - for (auto & input : step->drv.inputDrvs) { - Derivation drv2 = readDerivation(input.first); + for (auto & input : step->drv->inputDrvs) { + Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first)); for (auto & name : input.second) { auto i = drv2.outputs.find(name); if (i == drv2.outputs.end()) continue; - inputs.insert(i->second.path); - basicDrv.inputSrcs.insert(i->second.path); + inputs.insert(i->second.path.clone()); + basicDrv.inputSrcs.insert(i->second.path.clone()); } } @@ -241,14 +242,15 @@ void State::buildRemote(ref destStore, this will copy the inputs to the binary cache from the local store. */ if (localStore != std::shared_ptr(destStore)) - copyClosure(ref(localStore), destStore, step->drv.inputSrcs, NoRepair, NoCheckSigs); + copyClosure(ref(localStore), destStore, step->drv->inputSrcs, NoRepair, NoCheckSigs); /* Copy the input closure. */ if (!machine->isLocalhost()) { auto mc1 = std::make_shared>(nrStepsWaiting); mc1.reset(); MaintainCount mc2(nrStepsCopyingTo); - printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % step->drvPath % machine->sshName); + printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’", + localStore->printStorePath(step->drvPath), machine->sshName); auto now1 = std::chrono::steady_clock::now(); @@ -272,14 +274,19 @@ void State::buildRemote(ref destStore, logFD = -1; /* Do the build. */ - printMsg(lvlDebug, format("building ‘%1%’ on ‘%2%’") % step->drvPath % machine->sshName); + printMsg(lvlDebug, "building ‘%s’ on ‘%s’", + localStore->printStorePath(step->drvPath), + machine->sshName); updateStep(ssBuilding); - if (sendDerivation) - to << cmdBuildPaths << PathSet({step->drvPath}); - else - to << cmdBuildDerivation << step->drvPath << basicDrv; + if (sendDerivation) { + to << cmdBuildPaths; + writeStorePaths(*localStore, to, singleton(step->drvPath)); + } else { + to << cmdBuildDerivation << localStore->printStorePath(step->drvPath); + writeDerivation(to, *localStore, basicDrv); + } to << maxSilentTime << buildTimeout; if (GET_PROTOCOL_MINOR(remoteVersion) >= 2) to << maxLogSize; @@ -380,7 +387,8 @@ void State::buildRemote(ref destStore, /* If the path was substituted or already valid, then we didn't get a build log. */ if (result.isCached) { - printMsg(lvlInfo, format("outputs of ‘%1%’ substituted or already valid on ‘%2%’") % step->drvPath % machine->sshName); + printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’", + localStore->printStorePath(step->drvPath), machine->sshName); unlink(result.logFile.c_str()); result.logFile = ""; } @@ -395,13 +403,12 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); - PathSet outputs; - for (auto & output : step->drv.outputs) - outputs.insert(output.second.path); + auto outputs = step->drv->outputPaths(); /* Query the size of the output paths. */ size_t totalNarSize = 0; - to << cmdQueryPathInfos << outputs; + to << cmdQueryPathInfos; + writeStorePaths(*localStore, to, outputs); to.flush(); while (true) { if (readString(from) == "") break; @@ -416,8 +423,8 @@ void State::buildRemote(ref destStore, return; } - printMsg(lvlDebug, format("copying outputs of ‘%s’ from ‘%s’ (%d bytes)") - % step->drvPath % machine->sshName % totalNarSize); + printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)", + localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize); /* Block until we have the required amount of memory available, which is twice the NAR size (namely the @@ -431,10 +438,11 @@ void State::buildRemote(ref destStore, auto resMs = std::chrono::duration_cast(resStop - resStart).count(); if (resMs >= 1000) - printMsg(lvlError, format("warning: had to wait %d ms for %d memory tokens for %s") - % resMs % totalNarSize % step->drvPath); + printMsg(lvlError, "warning: had to wait %d ms for %d memory tokens for %s", + resMs, totalNarSize, localStore->printStorePath(step->drvPath)); - to << cmdExportPaths << 0 << outputs; + to << cmdExportPaths << 0; + writeStorePaths(*localStore, to, outputs); to.flush(); destStore->importPaths(from, result.accessor, NoCheckSigs); diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index 25e8c41a..cd8f0a39 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -14,16 +14,14 @@ BuildOutput getBuildOutput(nix::ref store, BuildOutput res; /* Compute the closure size. */ - PathSet outputs; - for (auto & output : drv.outputs) - outputs.insert(output.second.path); - PathSet closure; + auto outputs = drv.outputPaths(); + StorePathSet closure; for (auto & output : outputs) - store->computeFSClosure(output, closure); + store->computeFSClosure(singleton(output), closure); for (auto & path : closure) { auto info = store->queryPathInfo(path); res.closureSize += info->narSize; - if (outputs.find(path) != outputs.end()) res.size += info->narSize; + if (outputs.count(path)) res.size += info->narSize; } /* Get build products. */ @@ -39,11 +37,13 @@ BuildOutput getBuildOutput(nix::ref store, , std::regex::extended); for (auto & output : outputs) { - Path failedFile = output + "/nix-support/failed"; + auto outputS = store->printStorePath(output); + + Path failedFile = outputS + "/nix-support/failed"; if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular) res.failed = true; - Path productsFile = output + "/nix-support/hydra-build-products"; + Path productsFile = outputS + "/nix-support/hydra-build-products"; if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular) continue; @@ -72,7 +72,7 @@ BuildOutput getBuildOutput(nix::ref store, auto st = accessor->stat(product.path); if (st.type == FSAccessor::Type::tMissing) continue; - product.name = product.path == output ? "" : baseNameOf(product.path); + product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path); if (st.type == FSAccessor::Type::tRegular) { product.isRegular = true; @@ -91,14 +91,14 @@ BuildOutput getBuildOutput(nix::ref store, if (!explicitProducts) { for (auto & output : drv.outputs) { BuildProduct product; - product.path = output.second.path; + product.path = store->printStorePath(output.second.path); product.type = "nix-build"; product.subtype = output.first == "out" ? "" : output.first; - product.name = storePathToName(product.path); + product.name = output.second.path.name(); auto st = accessor->stat(product.path); if (st.type == FSAccessor::Type::tMissing) - throw Error(format("getting status of ‘%1%’") % product.path); + throw Error("getting status of ‘%s’", product.path); if (st.type == FSAccessor::Type::tDirectory) res.products.push_back(product); } @@ -106,7 +106,7 @@ BuildOutput getBuildOutput(nix::ref store, /* Get the release name from $output/nix-support/hydra-release-name. */ for (auto & output : outputs) { - Path p = output + "/nix-support/hydra-release-name"; + auto p = store->printStorePath(output) + "/nix-support/hydra-release-name"; if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue; try { res.releaseName = trim(accessor->readFile(p)); @@ -116,7 +116,7 @@ BuildOutput getBuildOutput(nix::ref store, /* Get metrics. */ for (auto & output : outputs) { - Path metricsFile = output + "/nix-support/hydra-metrics"; + auto metricsFile = store->printStorePath(output) + "/nix-support/hydra-metrics"; if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue; for (auto & line : tokenizeString(accessor->readFile(metricsFile), "\n")) { auto fields = tokenizeString>(line); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index edd4b1f7..a0fc01c3 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -18,7 +18,7 @@ void setThreadName(const std::string & name) void State::builder(MachineReservation::ptr reservation) { - setThreadName("bld~" + baseNameOf(reservation->step->drvPath)); + setThreadName("bld~" + std::string(reservation->step->drvPath.to_string())); StepResult res = sRetry; @@ -39,8 +39,10 @@ void State::builder(MachineReservation::ptr reservation) auto destStore = getDestStore(); res = doBuildStep(destStore, reservation, activeStep); } catch (std::exception & e) { - printMsg(lvlError, format("uncaught exception building ‘%1%’ on ‘%2%’: %3%") - % reservation->step->drvPath % reservation->machine->sshName % e.what()); + printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s", + localStore->printStorePath(reservation->step->drvPath), + reservation->machine->sshName, + e.what()); } } @@ -60,7 +62,7 @@ void State::builder(MachineReservation::ptr reservation) nrRetries++; if (step_->tries > maxNrRetries) maxNrRetries = step_->tries; // yeah yeah, not atomic int delta = retryInterval * std::pow(retryBackoff, step_->tries - 1) + (rand() % 10); - printMsg(lvlInfo, format("will retry ‘%1%’ after %2%s") % step->drvPath % delta); + printMsg(lvlInfo, "will retry ‘%s’ after %ss", localStore->printStorePath(step->drvPath), delta); step_->after = std::chrono::system_clock::now() + std::chrono::seconds(delta); } @@ -95,7 +97,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, cancelled (namely if there are no more Builds referring to it). */ BuildID buildId; - Path buildDrvPath; + std::optional buildDrvPath; unsigned int maxSilentTime, buildTimeout; unsigned int repeats = step->isDeterministic ? 1 : 0; @@ -116,7 +118,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, possibility, we retry this step (putting it back in the runnable queue). If there are really no strong pointers to the step, it will be deleted. */ - printMsg(lvlInfo, format("maybe cancelling build step ‘%1%’") % step->drvPath); + printMsg(lvlInfo, "maybe cancelling build step ‘%s’", localStore->printStorePath(step->drvPath)); return sMaybeCancelled; } @@ -138,15 +140,15 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (!build) build = *dependents.begin(); buildId = build->id; - buildDrvPath = build->drvPath; + buildDrvPath = build->drvPath.clone(); maxSilentTime = build->maxSilentTime; buildTimeout = build->buildTimeout; printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - step->drvPath, repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); + localStore->printStorePath(step->drvPath), repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); } - bool quit = buildId == buildOne && step->drvPath == buildDrvPath; + bool quit = buildId == buildOne && step->drvPath == *buildDrvPath; RemoteResult result; BuildOutput res; @@ -166,7 +168,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, try { auto store = destStore.dynamic_pointer_cast(); if (uploadLogsToBinaryCache && store && pathExists(result.logFile)) { - store->upsertFile("log/" + baseNameOf(step->drvPath), readFile(result.logFile), "text/plain; charset=utf-8"); + store->upsertFile("log/" + std::string(step->drvPath.to_string()), readFile(result.logFile), "text/plain; charset=utf-8"); unlink(result.logFile.c_str()); } } catch (...) { @@ -218,7 +220,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (result.stepStatus == bsSuccess) { updateStep(ssPostProcessing); - res = getBuildOutput(destStore, ref(result.accessor), step->drv); + res = getBuildOutput(destStore, ref(result.accessor), *step->drv); } result.accessor = 0; @@ -255,8 +257,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, /* The step had a hopefully temporary failure (e.g. network issue). Retry a number of times. */ if (result.canRetry) { - printMsg(lvlError, format("possibly transient failure building ‘%1%’ on ‘%2%’: %3%") - % step->drvPath % machine->sshName % result.errorMsg); + printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s", + localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg); assert(stepNr); bool retry; { @@ -275,7 +277,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, assert(stepNr); - for (auto & path : step->drv.outputPaths()) + for (auto & path : step->drv->outputPaths()) addRoot(path); /* Register success in the database for all Build objects that @@ -308,7 +310,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, no new referrers can have been added in the meantime or be added afterwards. */ if (direct.empty()) { - printMsg(lvlDebug, format("finishing build step ‘%1%’") % step->drvPath); + printMsg(lvlDebug, "finishing build step ‘%s’", + localStore->printStorePath(step->drvPath)); steps_->erase(step->drvPath); } } @@ -393,7 +396,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, be certain no new referrers can be added. */ if (indirect.empty()) { for (auto & s : steps) { - printMsg(lvlDebug, format("finishing build step ‘%1%’") % s->drvPath); + printMsg(lvlDebug, "finishing build step ‘%s’", + localStore->printStorePath(s->drvPath)); steps_->erase(s->drvPath); } } @@ -437,8 +441,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, /* Remember failed paths in the database so that they won't be built again. */ if (result.stepStatus != bsCachedFailure && result.canCache) - for (auto & path : step->drv.outputPaths()) - txn.parameterized("insert into FailedPaths values ($1)")(path).exec(); + for (auto & path : step->drv->outputPaths()) + txn.parameterized("insert into FailedPaths values ($1)")(localStore->printStorePath(path)).exec(); txn.commit(); } @@ -478,8 +482,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, } -void State::addRoot(const Path & storePath) +void State::addRoot(const StorePath & storePath) { - auto root = rootsDir + "/" + baseNameOf(storePath); + auto root = rootsDir + "/" + std::string(storePath.to_string()); if (!pathExists(root)) writeFile(root, ""); } diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 068d5c57..86596ff5 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -10,7 +10,7 @@ using namespace nix; void State::makeRunnable(Step::ptr step) { - printMsg(lvlChatty, format("step ‘%1%’ is now runnable") % step->drvPath); + printMsg(lvlChatty, "step ‘%s’ is now runnable", localStore->printStorePath(step->drvPath)); { auto step_(step->state.lock()); @@ -248,7 +248,7 @@ system_time State::doDispatch() /* Can this machine do this step? */ if (!mi.machine->supportsStep(step)) { debug("machine '%s' does not support step '%s' (system type '%s')", - mi.machine->sshName, step->drvPath, step->drv.platform); + mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform); continue; } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index f97a1c95..3d5dad4d 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -38,9 +38,9 @@ static uint64_t getMemSize() std::string getEnvOrDie(const std::string & key) { - char * value = getenv(key.c_str()); + auto value = getEnv(key); if (!value) throw Error("environment variable '%s' is not set", key); - return value; + return *value; } @@ -160,7 +160,7 @@ void State::monitorMachinesFile() { string defaultMachinesFile = "/etc/nix/machines"; auto machinesFiles = tokenizeString>( - getEnv("NIX_REMOTE_SYSTEMS", pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); + getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); if (machinesFiles.empty()) { parseMachines("localhost " + @@ -252,10 +252,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID (buildId) (stepNr) (0) // == build - (step->drvPath) + (localStore->printStorePath(step->drvPath)) (status == bsBusy ? 1 : 0) (startTime, startTime != 0) - (step->drv.platform) + (step->drv->platform) ((int) status, status != bsBusy) (propagatedFrom, propagatedFrom != 0) (errorMsg, errorMsg != "") @@ -264,10 +264,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID if (r.affected_rows() == 0) goto restart; - for (auto & output : step->drv.outputs) + for (auto & output : step->drv->outputs) txn.parameterized ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)") - (buildId)(stepNr)(output.first)(output.second.path).exec(); + (buildId)(stepNr)(output.first)(localStore->printStorePath(output.second.path)).exec(); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); @@ -310,7 +310,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result, int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const Path & drvPath, const string & outputName, const Path & storePath) + Build::ptr build, const StorePath & drvPath, const string & outputName, const StorePath & storePath) { restart: auto stepNr = allocBuildStep(txn, build->id); @@ -320,7 +320,7 @@ int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t sto (build->id) (stepNr) (1) // == substitution - (drvPath) + (localStore->printStorePath(drvPath)) (0) (0) (startTime) @@ -330,7 +330,10 @@ int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t sto txn.parameterized ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)") - (build->id)(stepNr)(outputName)(storePath).exec(); + (build->id) + (stepNr) + (outputName) + (localStore->printStorePath(storePath)).exec(); return stepNr; } @@ -450,8 +453,8 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, bool State::checkCachedFailure(Step::ptr step, Connection & conn) { pqxx::work txn(conn); - for (auto & path : step->drv.outputPaths()) - if (!txn.parameterized("select 1 from FailedPaths where path = $1")(path).exec().empty()) + for (auto & path : step->drv->outputPaths()) + if (!txn.parameterized("select 1 from FailedPaths where path = $1")(localStore->printStorePath(path)).exec().empty()) return true; return false; } @@ -763,7 +766,7 @@ void State::run(BuildID buildOne) Store::Params localParams; localParams["max-connections"] = "16"; localParams["max-connection-age"] = "600"; - localStore = openStore(getEnv("NIX_REMOTE"), localParams); + localStore = openStore(getEnv("NIX_REMOTE").value_or(""), localParams); auto storeUri = config->getStrOption("store_uri"); _destStore = storeUri == "" ? localStore : openStore(storeUri); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index e657a4b8..62de134a 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -83,7 +83,7 @@ bool State::getQueuedBuilds(Connection & conn, them yet (since we don't want a long-running transaction). */ std::vector newIDs; std::map newBuildsByID; - std::multimap newBuildsByPath; + std::multimap newBuildsByPath; unsigned int newLastBuildId = lastBuildId; @@ -104,7 +104,7 @@ bool State::getQueuedBuilds(Connection & conn, auto build = std::make_shared(); build->id = id; - build->drvPath = row["drvPath"].as(); + build->drvPath = localStore->parseStorePath(row["drvPath"].as()); build->projectName = row["project"].as(); build->jobsetName = row["jobset"].as(); build->jobName = row["job"].as(); @@ -117,14 +117,14 @@ bool State::getQueuedBuilds(Connection & conn, newIDs.push_back(id); newBuildsByID[id] = build; - newBuildsByPath.emplace(std::make_pair(build->drvPath, id)); + newBuildsByPath.emplace(std::make_pair(build->drvPath.clone(), id)); } } std::set newRunnable; unsigned int nrAdded; std::function createBuild; - std::set finishedDrvs; + std::set finishedDrvs; createBuild = [&](Build::ptr build) { printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName()); @@ -160,7 +160,8 @@ bool State::getQueuedBuilds(Connection & conn, /* Some step previously failed, so mark the build as failed right away. */ - printMsg(lvlError, format("marking build %d as cached failure due to ‘%s’") % build->id % ex.step->drvPath); + printMsg(lvlError, "marking build %d as cached failure due to ‘%s’", + build->id, localStore->printStorePath(ex.step->drvPath)); if (!build->finishedInDB) { auto mc = startDbUpdate(); pqxx::work txn(conn); @@ -171,14 +172,14 @@ bool State::getQueuedBuilds(Connection & conn, auto res = txn.parameterized ("select max(build) from BuildSteps where drvPath = $1 and startTime != 0 and stopTime != 0 and status = 1") - (ex.step->drvPath).exec(); + (localStore->printStorePath(ex.step->drvPath)).exec(); if (!res[0][0].is_null()) propagatedFrom = res[0][0].as(); if (!propagatedFrom) { - for (auto & output : ex.step->drv.outputs) { + for (auto & output : ex.step->drv->outputs) { auto res = txn.parameterized ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1") - (output.second.path).exec(); + (localStore->printStorePath(output.second.path)).exec(); if (!res[0][0].is_null()) { propagatedFrom = res[0][0].as(); break; @@ -217,7 +218,7 @@ bool State::getQueuedBuilds(Connection & conn, /* If we didn't get a step, it means the step's outputs are all valid. So we mark this as a finished, cached build. */ if (!step) { - Derivation drv = readDerivation(build->drvPath); + Derivation drv = readDerivation(*localStore, localStore->printStorePath(build->drvPath)); BuildOutput res = getBuildOutputCached(conn, destStore, drv); for (auto & path : drv.outputPaths()) @@ -227,7 +228,7 @@ bool State::getQueuedBuilds(Connection & conn, auto mc = startDbUpdate(); pqxx::work txn(conn); time_t now = time(0); - printMsg(lvlInfo, format("marking build %1% as succeeded (cached)") % build->id); + printMsg(lvlInfo, "marking build %1% as succeeded (cached)", build->id); markSucceededBuild(txn, build, res, true, now, now); notifyBuildFinished(txn, build->id, {}); txn.commit(); @@ -250,8 +251,8 @@ bool State::getQueuedBuilds(Connection & conn, build->propagatePriorities(); - printMsg(lvlChatty, format("added build %1% (top-level step %2%, %3% new steps)") - % build->id % step->drvPath % newSteps.size()); + printMsg(lvlChatty, "added build %1% (top-level step %2%, %3% new steps)", + build->id, localStore->printStorePath(step->drvPath), newSteps.size()); }; /* Now instantiate build steps for each new build. The builder @@ -271,7 +272,7 @@ bool State::getQueuedBuilds(Connection & conn, try { createBuild(build); } catch (Error & e) { - e.addPrefix(format("while loading build %1%: ") % build->id); + e.addPrefix(fmt("while loading build %1%: ", build->id)); throw; } @@ -358,10 +359,12 @@ void State::processQueueChange(Connection & conn) activeStepState->cancelled = true; if (activeStepState->pid != -1) { printInfo("killing builder process %d of build step ‘%s’", - activeStepState->pid, activeStep->step->drvPath); + activeStepState->pid, + localStore->printStorePath(activeStep->step->drvPath)); if (kill(activeStepState->pid, SIGINT) == -1) printError("error killing build step ‘%s’: %s", - activeStep->step->drvPath, strerror(errno)); + localStore->printStorePath(activeStep->step->drvPath), + strerror(errno)); } } } @@ -370,8 +373,8 @@ void State::processQueueChange(Connection & conn) Step::ptr State::createStep(ref destStore, - Connection & conn, Build::ptr build, const Path & drvPath, - Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, + Connection & conn, Build::ptr build, const StorePath & drvPath, + Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable) { if (finishedDrvs.find(drvPath) != finishedDrvs.end()) return 0; @@ -400,7 +403,7 @@ Step::ptr State::createStep(ref destStore, /* If it doesn't exist, create it. */ if (!step) { step = std::make_shared(); - step->drvPath = drvPath; + step->drvPath = drvPath.clone(); isNew = true; } @@ -414,28 +417,28 @@ Step::ptr State::createStep(ref destStore, if (referringStep) step_->rdeps.push_back(referringStep); - (*steps_)[drvPath] = step; + steps_->insert_or_assign(drvPath.clone(), step); } if (!isNew) return step; - printMsg(lvlDebug, format("considering derivation ‘%1%’") % drvPath); + printMsg(lvlDebug, "considering derivation ‘%1%’", localStore->printStorePath(drvPath)); /* Initialize the step. Note that the step may be visible in ‘steps’ before this point, but that doesn't matter because it's not runnable yet, and other threads won't make it runnable while step->created == false. */ - step->drv = readDerivation(drvPath); - step->parsedDrv = std::make_unique(drvPath, step->drv); + step->drv = std::make_unique(readDerivation(*localStore, localStore->printStorePath(drvPath))); + step->parsedDrv = std::make_unique(drvPath.clone(), *step->drv); step->preferLocalBuild = step->parsedDrv->willBuildLocally(); - step->isDeterministic = get(step->drv.env, "isDetermistic", "0") == "1"; + step->isDeterministic = get(step->drv->env, "isDetermistic").value_or("0") == "1"; - step->systemType = step->drv.platform; + step->systemType = step->drv->platform; { - auto i = step->drv.env.find("requiredSystemFeatures"); + auto i = step->drv->env.find("requiredSystemFeatures"); StringSet features; - if (i != step->drv.env.end()) + if (i != step->drv->env.end()) features = step->requiredSystemFeatures = tokenizeString>(i->second); if (step->preferLocalBuild) features.insert("local"); @@ -451,12 +454,13 @@ Step::ptr State::createStep(ref destStore, /* Are all outputs valid? */ bool valid = true; - PathSet outputs = step->drv.outputPaths(); + auto outputs = step->drv->outputPaths(); DerivationOutputs missing; - for (auto & i : step->drv.outputs) + for (auto & i : step->drv->outputs) if (!destStore->isValidPath(i.second.path)) { valid = false; - missing[i.first] = i.second; + missing.insert_or_assign(i.first, + DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); } /* Try to copy the missing paths from the local store or from @@ -469,7 +473,7 @@ Step::ptr State::createStep(ref destStore, avail++; else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({i.second.path}, infos); + localStore->querySubstitutablePathInfos(singleton(i.second.path), infos); if (infos.size() == 1) avail++; } @@ -482,14 +486,18 @@ Step::ptr State::createStep(ref destStore, time_t startTime = time(0); if (localStore->isValidPath(i.second.path)) - printInfo("copying output ‘%1%’ of ‘%2%’ from local store", i.second.path, drvPath); + printInfo("copying output ‘%1%’ of ‘%2%’ from local store", + localStore->printStorePath(i.second.path), + localStore->printStorePath(drvPath)); else { - printInfo("substituting output ‘%1%’ of ‘%2%’", i.second.path, drvPath); + printInfo("substituting output ‘%1%’ of ‘%2%’", + localStore->printStorePath(i.second.path), + localStore->printStorePath(drvPath)); localStore->ensurePath(i.second.path); // FIXME: should copy directly from substituter to destStore. } - copyClosure(ref(localStore), destStore, {i.second.path}); + copyClosure(ref(localStore), destStore, singleton(i.second.path)); time_t stopTime = time(0); @@ -501,7 +509,10 @@ Step::ptr State::createStep(ref destStore, } } catch (Error & e) { - printError("while copying/substituting output ‘%s’ of ‘%s’: %s", i.second.path, drvPath, e.what()); + printError("while copying/substituting output ‘%s’ of ‘%s’: %s", + localStore->printStorePath(i.second.path), + localStore->printStorePath(drvPath), + e.what()); valid = false; break; } @@ -511,15 +522,15 @@ Step::ptr State::createStep(ref destStore, // FIXME: check whether all outputs are in the binary cache. if (valid) { - finishedDrvs.insert(drvPath); + finishedDrvs.insert(drvPath.clone()); return 0; } /* No, we need to build. */ - printMsg(lvlDebug, format("creating build step ‘%1%’") % drvPath); + printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath)); /* Create steps for the dependencies. */ - for (auto & i : step->drv.inputDrvs) { + for (auto & i : step->drv->inputDrvs) { auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable); if (dep) { auto step_(step->state.lock()); @@ -607,7 +618,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref ("select id, buildStatus, releaseName, closureSize, size from Builds b " "join BuildOutputs o on b.id = o.build " "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1") - (output.second.path).exec(); + (localStore->printStorePath(output.second.path)).exec(); if (r.empty()) continue; BuildID id = r[0][0].as(); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index a95cdb61..de74b768 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -123,8 +123,8 @@ struct Build typedef std::weak_ptr wptr; BuildID id; - nix::Path drvPath; - std::map outputs; + nix::StorePath drvPath; + std::map outputs; std::string projectName, jobsetName, jobName; time_t timestamp; unsigned int maxSilentTime, buildTimeout; @@ -150,8 +150,8 @@ struct Step typedef std::shared_ptr ptr; typedef std::weak_ptr wptr; - nix::Path drvPath; - nix::Derivation drv; + nix::StorePath drvPath; + std::unique_ptr drv; std::unique_ptr parsedDrv; std::set requiredSystemFeatures; bool preferLocalBuild; @@ -252,7 +252,7 @@ struct Machine { /* Check that this machine is of the type required by the step. */ - if (!systemTypes.count(step->drv.platform == "builtin" ? nix::settings.thisSystem : step->drv.platform)) + if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) return false; /* Check that the step requires all mandatory features of this @@ -313,7 +313,7 @@ private: queued builds). Note that these are weak pointers. Steps are kept alive by being reachable from Builds or by being in progress. */ - typedef std::map Steps; + typedef std::map Steps; nix::Sync steps; /* Build steps that have no unbuilt dependencies. */ @@ -454,7 +454,7 @@ private: const std::string & machine); int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const nix::Path & drvPath, const std::string & outputName, const nix::Path & storePath); + Build::ptr build, const nix::StorePath & drvPath, const std::string & outputName, const nix::StorePath & storePath); void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status); @@ -473,8 +473,8 @@ private: const nix::Derivation & drv); Step::ptr createStep(nix::ref store, - Connection & conn, Build::ptr build, const nix::Path & drvPath, - Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, + Connection & conn, Build::ptr build, const nix::StorePath & drvPath, + Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable); Jobset::ptr createJobset(pqxx::work & txn, @@ -523,7 +523,7 @@ private: void dumpStatus(Connection & conn, bool log); - void addRoot(const nix::Path & storePath); + void addRoot(const nix::StorePath & storePath); public: diff --git a/src/hydra-queue-runner/token-server.hh b/src/hydra-queue-runner/token-server.hh index e00004d0..d8004f73 100644 --- a/src/hydra-queue-runner/token-server.hh +++ b/src/hydra-queue-runner/token-server.hh @@ -7,7 +7,7 @@ namespace nix { -MakeError(NoTokens, Error) +MakeError(NoTokens, Error); /* This class hands out tokens. There are only ‘maxTokens’ tokens available. Calling get(N) will return a Token object, representing diff --git a/src/libhydra/db.hh b/src/libhydra/db.hh index 29af7cc2..35d78edf 100644 --- a/src/libhydra/db.hh +++ b/src/libhydra/db.hh @@ -12,7 +12,7 @@ struct Connection : pqxx::connection std::string getFlags() { using namespace nix; - auto s = getEnv("HYDRA_DBI", "dbi:Pg:dbname=hydra;"); + auto s = getEnv("HYDRA_DBI").value_or("dbi:Pg:dbname=hydra;"); std::string prefix = "dbi:Pg:"; if (std::string(s, 0, prefix.size()) != prefix) throw Error("$HYDRA_DBI does not denote a PostgreSQL database"); diff --git a/src/libhydra/hydra-config.hh b/src/libhydra/hydra-config.hh index a4050666..98d73d47 100644 --- a/src/libhydra/hydra-config.hh +++ b/src/libhydra/hydra-config.hh @@ -14,9 +14,9 @@ struct Config /* Read hydra.conf. */ auto hydraConfigFile = getEnv("HYDRA_CONFIG"); - if (pathExists(hydraConfigFile)) { + if (hydraConfigFile && pathExists(*hydraConfigFile)) { - for (auto line : tokenizeString(readFile(hydraConfigFile), "\n")) { + for (auto line : tokenizeString(readFile(*hydraConfigFile), "\n")) { line = trim(string(line, 0, line.find('#'))); auto eq = line.find('='); From 5308e514ad64cde0dafac104c0774a7a77f9517e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Feb 2020 10:26:18 +0100 Subject: [PATCH 05/15] Fix nlohmann_json dependency --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index cf17657e..ce07010a 100644 --- a/release.nix +++ b/release.nix @@ -132,7 +132,7 @@ rec { perlDeps perl nix postgresql95 # for running the tests boost - nlohmann_json + (nlohmann_json.override { multipleHeaders = true; }) ]; hydraPath = lib.makeBinPath ( From adf61e5cf8571a4c50664361c2d8e3b89266e11e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Feb 2020 14:54:21 +0100 Subject: [PATCH 06/15] Fix build (cherry picked from commit 639c660abfd5de62ecfcd8d3cbc2eb6924c7ec75) --- src/hydra-queue-runner/queue-monitor.cc | 7 +++---- src/hydra-queue-runner/state.hh | 6 ++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 62de134a..3c02e1aa 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -102,9 +102,9 @@ bool State::getQueuedBuilds(Connection & conn, if (id > newLastBuildId) newLastBuildId = id; if (builds_->count(id)) continue; - auto build = std::make_shared(); + auto build = std::make_shared( + localStore->parseStorePath(row["drvPath"].as())); build->id = id; - build->drvPath = localStore->parseStorePath(row["drvPath"].as()); build->projectName = row["project"].as(); build->jobsetName = row["jobset"].as(); build->jobName = row["job"].as(); @@ -402,8 +402,7 @@ Step::ptr State::createStep(ref destStore, /* If it doesn't exist, create it. */ if (!step) { - step = std::make_shared(); - step->drvPath = drvPath.clone(); + step = std::make_shared(drvPath.clone()); isNew = true; } diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index de74b768..180907e9 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -136,6 +136,9 @@ struct Build std::atomic_bool finishedInDB{false}; + Build(nix::StorePath && drvPath) : drvPath(std::move(drvPath)) + { } + std::string fullJobName() { return projectName + ":" + jobsetName + ":" + jobName; @@ -201,6 +204,9 @@ struct Step nix::Sync state; + Step(nix::StorePath && drvPath) : drvPath(std::move(drvPath)) + { } + ~Step() { //printMsg(lvlError, format("destroying step %1%") % drvPath); From c642f787ee5f4b1fcb14309b19c5841e2a2e5e75 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Feb 2020 21:10:22 +0100 Subject: [PATCH 07/15] hydra-eval-jobs: Parallelize (cherry picked from commit be8eb9d00d6a3fcac520bc7dfd5740c0b3ade746) --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 542 ++++++++++++++----------- 1 file changed, 315 insertions(+), 227 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 8abdea7e..1514cf4b 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -1,35 +1,63 @@ #include #include -#define GC_LINUX_THREADS 1 -#include - #include "shared.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" #include "util.hh" -#include "json.hh" #include "get-drvs.hh" #include "globals.hh" #include "common-eval-args.hh" +#include "attr-path.hh" +#include "derivations.hh" #include "hydra-config.hh" #include #include +#include + +#include using namespace nix; - static Path gcRootsDir; +static size_t maxMemorySize; +struct MyArgs : MixEvalArgs, MixCommonArgs +{ + Path releaseExpr; + bool dryRun = false; -static void findJobs(EvalState & state, JSONObject & top, - Bindings & autoArgs, Value & v, const string & attrPath); + MyArgs() : MixCommonArgs("hydra-eval-jobs") + { + mkFlag() + .longName("help") + .description("show usage information") + .handler([&]() { + printHelp(programName, std::cout); + throw Exit(); + }); + mkFlag() + .longName("gc-roots-dir") + .description("garbage collector roots directory") + .labels({"path"}) + .dest(&gcRootsDir); -static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) + mkFlag() + .longName("dry-run") + .description("don't create store derivations") + .set(&dryRun, true); + + expectArg("expr", &releaseExpr); + } +}; + +static MyArgs myArgs; + +static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) { Strings res; std::function rec; @@ -54,170 +82,135 @@ static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & return concatStringsSep(", ", res); } - -static std::string lastAttrPath; -static bool comma = false; -static size_t maxHeapSize; - - -struct BailOut { }; - - -bool lte(const std::string & s1, const std::string & s2) +static void worker( + EvalState & state, + Bindings & autoArgs, + AutoCloseFD & to, + AutoCloseFD & from) { - size_t p1 = 0, p2 = 0; + Value vTop; + state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); + + auto vRoot = state.allocValue(); + state.autoCallFunction(autoArgs, vTop, *vRoot); while (true) { - if (p1 == s1.size()) return p2 == s2.size(); - if (p2 == s2.size()) return true; + /* Wait for the master to send us a job name. */ + writeLine(to.get(), "next"); - auto d1 = s1.find('.', p1); - auto d2 = s2.find('.', p2); + auto s = readLine(from.get()); + if (s == "exit") break; + if (!hasPrefix(s, "do ")) abort(); + std::string attrPath(s, 3); - auto c = s1.compare(p1, d1 - p1, s2, p2, d2 - p2); + debug("worker process %d at '%s'", getpid(), attrPath); - if (c < 0) return true; - if (c > 0) return false; + /* Evaluate it and send info back to the master. */ + nlohmann::json reply; - p1 = d1 == std::string::npos ? s1.size() : d1 + 1; - p2 = d2 == std::string::npos ? s2.size() : d2 + 1; - } -} + try { + auto v = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; + + state.forceValue(*v); + + if (auto drv = getDerivation(state, *v, false)) { + + DrvInfo::Outputs outputs = drv->queryOutputs(); + + if (drv->querySystem() == "unknown") + throw EvalError("derivation must have a 'system' attribute"); + + auto drvPath = drv->queryDrvPath(); + + nlohmann::json job; + + job["nixName"] = drv->queryName(); + job["system"] =drv->querySystem(); + job["drvPath"] = drvPath; + job["description"] = drv->queryMetaString("description"); + job["license"] = queryMetaStrings(state, *drv, "license", "shortName"); + job["homepage"] = drv->queryMetaString("homepage"); + job["maintainers"] = queryMetaStrings(state, *drv, "maintainers", "email"); + job["schedulingPriority"] = drv->queryMetaInt("schedulingPriority", 100); + job["timeout"] = drv->queryMetaInt("timeout", 36000); + job["maxSilent"] = drv->queryMetaInt("maxSilent", 7200); + job["isChannel"] = drv->queryMetaBool("isHydraChannel", false); + + /* If this is an aggregate, then get its constituents. */ + auto a = v->attrs->get(state.symbols.create("_hydraAggregate")); + if (a && state.forceBool(*a->value, *a->pos)) { + auto a = v->attrs->get(state.symbols.create("constituents")); + if (!a) + throw EvalError("derivation must have a ‘constituents’ attribute"); -static void findJobsWrapped(EvalState & state, JSONObject & top, - Bindings & autoArgs, Value & vIn, const string & attrPath) -{ - if (lastAttrPath != "" && lte(attrPath, lastAttrPath)) return; + PathSet context; + state.coerceToString(*a->pos, *a->value, context, true, false); + for (auto & i : context) + if (i.at(0) == '!') { + size_t index = i.find("!", 1); + job["constituents"].push_back(string(i, index + 1)); + } - debug(format("at path `%1%'") % attrPath); - - checkInterrupt(); - - Value v; - state.autoCallFunction(autoArgs, vIn, v); - - if (v.type == tAttrs) { - - auto drv = getDerivation(state, v, false); - - if (drv) { - Path drvPath; - - DrvInfo::Outputs outputs = drv->queryOutputs(); - - if (drv->querySystem() == "unknown") - throw EvalError("derivation must have a ‘system’ attribute"); - - if (comma) { std::cout << ","; comma = false; } - - { - auto res = top.object(attrPath); - res.attr("nixName", drv->queryName()); - res.attr("system", drv->querySystem()); - res.attr("drvPath", drvPath = drv->queryDrvPath()); - res.attr("description", drv->queryMetaString("description")); - res.attr("license", queryMetaStrings(state, *drv, "license", "shortName")); - res.attr("homepage", drv->queryMetaString("homepage")); - res.attr("maintainers", queryMetaStrings(state, *drv, "maintainers", "email")); - res.attr("schedulingPriority", drv->queryMetaInt("schedulingPriority", 100)); - res.attr("timeout", drv->queryMetaInt("timeout", 36000)); - res.attr("maxSilent", drv->queryMetaInt("maxSilent", 7200)); - res.attr("isChannel", drv->queryMetaBool("isHydraChannel", false)); - - /* If this is an aggregate, then get its constituents. */ - Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate")); - if (a != v.attrs->end() && state.forceBool(*a->value, *a->pos)) { - Bindings::iterator a = v.attrs->find(state.symbols.create("constituents")); - if (a == v.attrs->end()) - throw EvalError("derivation must have a ‘constituents’ attribute"); - PathSet context; - state.coerceToString(*a->pos, *a->value, context, true, false); - PathSet drvs; - for (auto & i : context) - if (i.at(0) == '!') { - size_t index = i.find("!", 1); - drvs.insert(string(i, index + 1)); + state.forceList(*a->value, *a->pos); + for (unsigned int n = 0; n < a->value->listSize(); ++n) { + auto v = a->value->listElems()[n]; + state.forceValue(*v); + if (v->type == tString) + job["namedConstituents"].push_back(state.forceStringNoCtx(*v)); } - res.attr("constituents", concatStringsSep(" ", drvs)); + } + + /* Register the derivation as a GC root. !!! This + registers roots for jobs that we may have already + done. */ + auto localStore = state.store.dynamic_pointer_cast(); + if (gcRootsDir != "" && localStore) { + Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); + if (!pathExists(root)) + localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false); + } + + nlohmann::json out; + for (auto & j : outputs) + out[j.first] = j.second; + job["outputs"] = std::move(out); + + reply["job"] = std::move(job); } - /* Register the derivation as a GC root. !!! This - registers roots for jobs that we may have already - done. */ - auto localStore = state.store.dynamic_pointer_cast(); - if (gcRootsDir != "" && localStore) { - Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); - if (!pathExists(root)) - localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false); - } - - auto res2 = res.object("outputs"); - for (auto & j : outputs) - res2.attr(j.first, j.second); - - } - - GC_prof_stats_s gc; - GC_get_prof_stats(&gc, sizeof(gc)); - - if (gc.heapsize_full > maxHeapSize) { - printInfo("restarting hydra-eval-jobs after job '%s' because heap size is at %d bytes", attrPath, gc.heapsize_full); - lastAttrPath = attrPath; - throw BailOut(); - } - } - - else { - if (!state.isDerivation(v)) { - for (auto & i : v.attrs->lexicographicOrder()) { + else if (v->type == tAttrs) { + auto attrs = nlohmann::json::array(); + StringSet ss; + for (auto & i : v->attrs->lexicographicOrder()) { std::string name(i->name); - - /* Skip jobs with dots in the name. */ - if (name.find('.') != std::string::npos) { + if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) { printError("skipping job with illegal name '%s'", name); continue; } - - findJobs(state, top, autoArgs, *i->value, - (attrPath.empty() ? "" : attrPath + ".") + name); + attrs.push_back(name); } + reply["attrs"] = std::move(attrs); } + + } catch (EvalError & e) { + reply["error"] = filterANSIEscapes(e.msg(), true); } + + writeLine(to.get(), reply.dump()); + + /* If our RSS exceeds the maximum, exit. The master will + start a new process. */ + struct rusage r; + getrusage(RUSAGE_SELF, &r); + if ((size_t) r.ru_maxrss > maxMemorySize * 1024) break; } - else if (v.type == tNull) { - // allow null values, meaning 'do nothing' - } - - else - throw TypeError(format("unsupported value: %1%") % v); + writeLine(to.get(), "restart"); } - -static void findJobs(EvalState & state, JSONObject & top, - Bindings & autoArgs, Value & v, const string & attrPath) -{ - try { - findJobsWrapped(state, top, autoArgs, v, attrPath); - } catch (EvalError & e) { - if (comma) { std::cout << ","; comma = false; } - auto res = top.object(attrPath); - res.attr("error", filterANSIEscapes(e.msg(), true)); - } -} - - int main(int argc, char * * argv) { - assert(lte("abc", "def")); - assert(lte("abc", "def.foo")); - assert(!lte("def", "abc")); - assert(lte("nixpkgs.hello", "nixpkgs")); - assert(lte("nixpkgs.hello", "nixpkgs.hellooo")); - assert(lte("gitAndTools.git-annex.x86_64-darwin", "gitAndTools.git-annex.x86_64-linux")); - assert(lte("gitAndTools.git-annex.x86_64-linux", "gitAndTools.git-annex-remote-b2.aarch64-linux")); - /* Prevent undeclared dependencies in the evaluation via $NIX_PATH. */ unsetenv("NIX_PATH"); @@ -226,116 +219,211 @@ int main(int argc, char * * argv) auto config = std::make_unique<::Config>(); - auto initialHeapSize = config->getStrOption("evaluator_initial_heap_size", ""); - if (initialHeapSize != "") - setenv("GC_INITIAL_HEAP_SIZE", initialHeapSize.c_str(), 1); - - maxHeapSize = config->getIntOption("evaluator_max_heap_size", 1UL << 30); + auto nrWorkers = config->getIntOption("evaluator_workers", 1); + maxMemorySize = config->getIntOption("evaluator_max_memory_size", 4096); initNix(); initGC(); - /* Read the current heap size, which is the initial heap size. */ - GC_prof_stats_s gc; - GC_get_prof_stats(&gc, sizeof(gc)); - auto initialHeapSizeInt = gc.heapsize_full; - - /* Then make sure the maximum heap size will be bigger than the initial heap size. */ - if (initialHeapSizeInt > maxHeapSize) { - printInfo("warning: evaluator_initial_heap_size (%d) bigger than evaluator_max_heap_size (%d).", initialHeapSizeInt, maxHeapSize); - maxHeapSize = initialHeapSizeInt * 1.1; - printInfo(" evaluator_max_heap_size now set to %d.", maxHeapSize); - } - - Path releaseExpr; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--gc-roots-dir") - gcRootsDir = getArg(*arg, arg, end); - else if (*arg == "--dry-run") - settings.readOnlyMode = true; - else if (*arg != "" && arg->at(0) == '-') - return false; - else - releaseExpr = *arg; - return true; - }); - myArgs.parseCmdline(argvToStrings(argc, argv)); - JSONObject json(std::cout, true); - std::cout.flush(); + /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ + settings.builders = ""; - do { + /* Prevent access to paths outside of the Nix search path and + to the environment. */ + evalSettings.restrictEval = true; - Pipe pipe; - pipe.create(); + if (myArgs.dryRun) settings.readOnlyMode = true; - ProcessOptions options; - options.allowVfork = false; + if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); - GC_atfork_prepare(); + if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); - auto pid = startProcess([&]() { - pipe.readSide = -1; + struct State + { + std::set todo{""}; + std::set active; + nlohmann::json jobs; + std::exception_ptr exc; + }; - GC_atfork_child(); - GC_start_mark_threads(); + std::condition_variable wakeup; - if (lastAttrPath != "") debug("resuming from '%s'", lastAttrPath); + Sync state_; - /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ - settings.builders = ""; + /* Start a handler thread per worker process. */ + auto handler = [&]() + { + try { + pid_t pid = -1; + AutoCloseFD from, to; - /* Prevent access to paths outside of the Nix search path and - to the environment. */ - evalSettings.restrictEval = true; + while (true) { - if (releaseExpr == "") throw UsageError("no expression specified"); + /* Start a new worker process if necessary. */ + if (pid == -1) { + Pipe toPipe, fromPipe; + toPipe.create(); + fromPipe.create(); + pid = startProcess( + [&, + to{std::make_shared(std::move(fromPipe.writeSide))}, + from{std::make_shared(std::move(toPipe.readSide))} + ]() + { + try { + EvalState state(myArgs.searchPath, openStore()); + Bindings & autoArgs = *myArgs.getAutoArgs(state); + worker(state, autoArgs, *to, *from); + } catch (std::exception & e) { + nlohmann::json err; + err["error"] = e.what(); + writeLine(to->get(), err.dump()); + } + }, + ProcessOptions { .allowVfork = false }); + from = std::move(fromPipe.readSide); + to = std::move(toPipe.writeSide); + debug("created worker process %d", pid); + } - if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + /* Check whether the existing worker process is still there. */ + auto s = readLine(from.get()); + if (s == "restart") { + pid = -1; + continue; + } else if (s != "next") { + auto json = nlohmann::json::parse(s); + throw Error("worker error: %s", (std::string) json["error"]); + } - EvalState state(myArgs.searchPath, openStore()); + /* Wait for a job name to become available. */ + std::string attrPath; - Bindings & autoArgs = *myArgs.getAutoArgs(state); + while (true) { + checkInterrupt(); + auto state(state_.lock()); + if ((state->todo.empty() && state->active.empty()) || state->exc) { + writeLine(to.get(), "exit"); + return; + } + if (!state->todo.empty()) { + attrPath = *state->todo.begin(); + state->todo.erase(state->todo.begin()); + state->active.insert(attrPath); + break; + } else + state.wait(wakeup); + } - Value v; - state.evalFile(lookupFileArg(state, releaseExpr), v); + /* Tell the worker to evaluate it. */ + writeLine(to.get(), "do " + attrPath); - comma = lastAttrPath != ""; + /* Wait for the response. */ + auto response = nlohmann::json::parse(readLine(from.get())); - try { - findJobs(state, json, autoArgs, v, ""); - lastAttrPath = ""; - } catch (BailOut &) { } + /* Handle the response. */ + StringSet newAttrs; - writeFull(pipe.writeSide.get(), lastAttrPath); + if (response.find("job") != response.end()) { + auto state(state_.lock()); + state->jobs[attrPath] = response["job"]; + } - exit(0); - }, options); + if (response.find("attrs") != response.end()) { + for (auto & i : response["attrs"]) { + auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i; + newAttrs.insert(s); + } + } - GC_atfork_parent(); + if (response.find("error") != response.end()) { + auto state(state_.lock()); + state->jobs[attrPath]["error"] = response["error"]; + } - pipe.writeSide = -1; + /* Add newly discovered job names to the queue. */ + { + auto state(state_.lock()); + state->active.erase(attrPath); + for (auto & s : newAttrs) + state->todo.insert(s); + wakeup.notify_all(); + } + } + } catch (...) { + auto state(state_.lock()); + state->exc = std::current_exception(); + wakeup.notify_all(); + } + }; - int status; - while (true) { - checkInterrupt(); - if (waitpid(pid, &status, 0) == pid) break; - if (errno != EINTR) continue; + std::vector threads; + for (size_t i = 0; i < nrWorkers; i++) + threads.emplace_back(std::thread(handler)); + + for (auto & thread : threads) + thread.join(); + + auto state(state_.lock()); + + if (state->exc) + std::rethrow_exception(state->exc); + + /* For aggregate jobs that have named consistuents + (i.e. constituents that are a job name rather than a + derivation), look up the referenced job and add it to the + dependencies of the aggregate derivation. */ + auto store = openStore(); + + for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) { + auto jobName = i.key(); + auto & job = i.value(); + + auto named = job.find("namedConstituents"); + if (named == job.end()) continue; + + if (myArgs.dryRun) { + for (std::string jobName2 : *named) { + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + std::string drvPath2 = (*job2)["drvPath"]; + job["constituents"].push_back(drvPath2); + } + } else { + std::string drvPath = job["drvPath"]; + auto drv = readDerivation(*store, drvPath); + + for (std::string jobName2 : *named) { + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + std::string drvPath2 = (*job2)["drvPath"]; + auto drv2 = readDerivation(*store, drvPath2); + job["constituents"].push_back(drvPath2); + drv.inputDrvs[store->parseStorePath(drvPath2)] = {drv2.outputs.begin()->first}; + } + + std::string drvName(store->parseStorePath(drvPath).name()); + assert(hasSuffix(drvName, drvExtension)); + drvName.resize(drvName.size() - drvExtension.size()); + auto h = hashDerivationModulo(*store, drv, true); + auto outPath = store->makeOutputPath("out", h, drvName); + drv.env["out"] = store->printStorePath(outPath); + drv.outputs.insert_or_assign("out", DerivationOutput(outPath.clone(), "", "")); + auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName)); + + debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath); + + job["drvPath"] = newDrvPath; + job["outputs"]["out"] = store->printStorePath(outPath); } - if (status != 0) - throw Exit(WIFEXITED(status) ? WEXITSTATUS(status) : 99); + job.erase("namedConstituents"); + } - maxHeapSize += 64 * 1024 * 1024; - - lastAttrPath = drainFD(pipe.readSide.get()); - } while (lastAttrPath != ""); + std::cout << state->jobs.dump(2) << "\n"; }); } From eb5873ae539affe3e91e04928ab4457446b377b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Feb 2020 11:19:45 +0100 Subject: [PATCH 08/15] Fix build --- release.nix | 6 ++---- src/hydra-eval-jobs/hydra-eval-jobs.cc | 12 ++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/release.nix b/release.nix index ce07010a..a3e2ffb5 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { hydraSrc ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.09-small"; } +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/release-19.09.tar.gz , officialRelease ? false , shell ? false }: @@ -154,9 +154,7 @@ rec { preConfigure = "autoreconf -vfi"; - NIX_LDFLAGS = [ - "-lpthread" - ]; + NIX_LDFLAGS = [ "-lpthread" ]; enableParallelBuilding = true; diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 1514cf4b..deffaeae 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -109,7 +109,7 @@ static void worker( nlohmann::json reply; try { - auto v = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; + auto v = findAlongAttrPath(state, attrPath, autoArgs, *vRoot); state.forceValue(*v); @@ -138,23 +138,23 @@ static void worker( /* If this is an aggregate, then get its constituents. */ auto a = v->attrs->get(state.symbols.create("_hydraAggregate")); - if (a && state.forceBool(*a->value, *a->pos)) { + if (a && state.forceBool(*(*a)->value, *(*a)->pos)) { auto a = v->attrs->get(state.symbols.create("constituents")); if (!a) throw EvalError("derivation must have a ‘constituents’ attribute"); PathSet context; - state.coerceToString(*a->pos, *a->value, context, true, false); + state.coerceToString(*(*a)->pos, *(*a)->value, context, true, false); for (auto & i : context) if (i.at(0) == '!') { size_t index = i.find("!", 1); job["constituents"].push_back(string(i, index + 1)); } - state.forceList(*a->value, *a->pos); - for (unsigned int n = 0; n < a->value->listSize(); ++n) { - auto v = a->value->listElems()[n]; + state.forceList(*(*a)->value, *(*a)->pos); + for (unsigned int n = 0; n < (*a)->value->listSize(); ++n) { + auto v = (*a)->value->listElems()[n]; state.forceValue(*v); if (v->type == tString) job["namedConstituents"].push_back(state.forceStringNoCtx(*v)); From eaa65f51f4e6a63719f5a9f1315a3694a35d127e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 3 Mar 2020 18:17:21 -0500 Subject: [PATCH 09/15] hydra-evaluator: make the logic of the scheduler easier to read --- src/hydra-evaluator/hydra-evaluator.cc | 33 ++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index 091a4e9c..e9103b84 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -129,19 +129,42 @@ struct Evaluator childStarted.notify_one(); } + bool shouldEvaluate(Jobset & jobset) + { + if (jobset.pid != -1) { + // Already running. + return false; + } + + if (jobset.triggerTime == std::numeric_limits::max()) { + // An evaluation of this Jobset is requested + return true; + } + + if (jobset.checkInterval <= 0) { + // Automatic scheduling is disabled. We allow requested + // evaluations, but never schedule start one. + return false; + } + + + if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) { + // Time to schedule a fresh evaluation + return true; + } + + return false; + } + void startEvals(State & state) { std::vector sorted; - time_t now = time(0); - /* Filter out jobsets that have been evaluated recently and have not been triggered. */ for (auto i = state.jobsets.begin(); i != state.jobsets.end(); ++i) if (evalOne || - (i->second.pid == -1 && - (i->second.triggerTime != std::numeric_limits::max() || - (i->second.checkInterval > 0 && i->second.lastCheckedTime + i->second.checkInterval <= now)))) + (i->second.evaluation_style && shouldEvaluate(i->second))) sorted.push_back(i); /* Put jobsets in order of ascending trigger time, last checked From 5fae9d96a25c658e4b3ea4f1c121b8d815ba6492 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 3 Mar 2020 19:53:18 -0500 Subject: [PATCH 10/15] hydra-evaluator: add a 'ONE_AT_A_TIME' evaluator style In the past, jobsets which are automatically evaluated are evaluated regularly, on a schedule. This schedule means a new evaluation is created every checkInterval seconds (assuming something changed.) This model works well for architectures where our build farm can easily keep up with demand. This commit adds a new type of evaluation, called ONE_AT_A_TIME, which only schedules a new evaluation if the previous evaluation of the jobset has no unfinished builds. This model of evaluation lets us have 'low-tier' architectures. For example, we could now have a jobset for ARMv7l builds, where the buildfarm only has a single, underpowered ARMv7l builder. Configuring that jobset as ONE_AT_A_TIME will create an evaluation and then won't schedule another evaluation until every job of the existing evaluation is complete. This way, the cache will have a complete collection of pre-built software for some commits, but the underpowered architecture will never become backlogged in ancient revisions. --- src/hydra-evaluator/hydra-evaluator.cc | 87 ++++++++++++++++++++++++-- src/lib/Hydra/Controller/Jobset.pm | 2 +- src/root/edit-jobset.tt | 1 + src/root/jobset.tt | 2 +- src/sql/hydra.sql | 2 +- 5 files changed, 86 insertions(+), 8 deletions(-) diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index e9103b84..364a5351 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -15,6 +15,13 @@ using namespace nix; typedef std::pair JobsetName; +enum class EvaluationStyle +{ + SCHEDULE = 1, + ONESHOT = 2, + ONE_AT_A_TIME = 3, +}; + struct Evaluator { std::unique_ptr config; @@ -24,6 +31,7 @@ struct Evaluator struct Jobset { JobsetName name; + std::optional evaluation_style; time_t lastCheckedTime, triggerTime; int checkInterval; Pid pid; @@ -60,7 +68,7 @@ struct Evaluator pqxx::work txn(*conn); auto res = txn.parameterized - ("select project, j.name, lastCheckedTime, triggerTime, checkInterval from Jobsets j join Projects p on j.project = p.name " + ("select project, j.name, lastCheckedTime, triggerTime, checkInterval, j.enabled as jobset_enabled from Jobsets j join Projects p on j.project = p.name " "where j.enabled != 0 and p.enabled != 0").exec(); auto state(state_.lock()); @@ -78,6 +86,17 @@ struct Evaluator jobset.lastCheckedTime = row["lastCheckedTime"].as(0); jobset.triggerTime = row["triggerTime"].as(notTriggered); jobset.checkInterval = row["checkInterval"].as(); + switch (row["jobset_enabled"].as(0)) { + case 1: + jobset.evaluation_style = EvaluationStyle::SCHEDULE; + break; + case 2: + jobset.evaluation_style = EvaluationStyle::ONESHOT; + break; + case 3: + jobset.evaluation_style = EvaluationStyle::ONE_AT_A_TIME; + break; + } seen.insert(name); } @@ -133,24 +152,82 @@ struct Evaluator { if (jobset.pid != -1) { // Already running. + debug("shouldEvaluate %s:%s? no: already running", + jobset.name.first, jobset.name.second); return false; } - if (jobset.triggerTime == std::numeric_limits::max()) { + if (jobset.triggerTime != std::numeric_limits::max()) { // An evaluation of this Jobset is requested + debug("shouldEvaluate %s:%s? yes: requested", + jobset.name.first, jobset.name.second); return true; } if (jobset.checkInterval <= 0) { // Automatic scheduling is disabled. We allow requested // evaluations, but never schedule start one. + debug("shouldEvaluate %s:%s? no: checkInterval <= 0", + jobset.name.first, jobset.name.second); return false; } - if (jobset.lastCheckedTime + jobset.checkInterval <= time(0)) { - // Time to schedule a fresh evaluation - return true; + // Time to schedule a fresh evaluation. If the jobset + // is a ONE_AT_A_TIME jobset, ensure the previous jobset + // has no remaining, unfinished work. + + auto conn(dbPool.get()); + + pqxx::work txn(*conn); + + if (jobset.evaluation_style == EvaluationStyle::ONE_AT_A_TIME) { + auto evaluation_res = txn.parameterized + ("select id from JobsetEvals " + "where project = $1 and jobset = $2 " + "order by id desc limit 1") + (jobset.name.first) + (jobset.name.second) + .exec(); + + if (evaluation_res.empty()) { + // First evaluation, so allow scheduling. + debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no prior eval", + jobset.name.first, jobset.name.second); + return true; + } + + auto evaluation_id = evaluation_res[0][0].as(); + + auto unfinished_build_res = txn.parameterized + ("select id from Builds " + "join JobsetEvalMembers " + " on (JobsetEvalMembers.build = Builds.id) " + "where JobsetEvalMembers.eval = $1 " + " and builds.finished = 0 " + " limit 1") + (evaluation_id) + .exec(); + + // If the previous evaluation has no unfinished builds + // schedule! + if (unfinished_build_res.empty()) { + debug("shouldEvaluate(one-at-a-time) %s:%s? yes: no unfinished builds", + jobset.name.first, jobset.name.second); + return true; + } else { + debug("shouldEvaluate(one-at-a-time) %s:%s? no: at least one unfinished build", + jobset.name.first, jobset.name.second); + return false; + } + + + } else { + // EvaluationStyle::ONESHOT, EvaluationStyle::SCHEDULED + debug("shouldEvaluate(oneshot/scheduled) %s:%s? yes: checkInterval elapsed", + jobset.name.first, jobset.name.second); + return true; + } } return false; diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index 91e21dd4..5ce4aab4 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -226,7 +226,7 @@ sub updateJobset { my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c; my $enabled = int($c->stash->{params}->{enabled}); - die if $enabled < 0 || $enabled > 2; + die if $enabled < 0 || $enabled > 3; my $shares = int($c->stash->{params}->{schedulingshares} // 1); error($c, "The number of scheduling shares must be positive.") if $shares <= 0; diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt index 6c380a3a..35ac668f 100644 --- a/src/root/edit-jobset.tt +++ b/src/root/edit-jobset.tt @@ -68,6 +68,7 @@ + diff --git a/src/root/jobset.tt b/src/root/jobset.tt index 9cf1202a..50be0f65 100644 --- a/src/root/jobset.tt +++ b/src/root/jobset.tt @@ -129,7 +129,7 @@ - + diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 8144dd30..a5fdc802 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -61,7 +61,7 @@ create table Jobsets ( errorTime integer, -- timestamp associated with errorMsg lastCheckedTime integer, -- last time the evaluator looked at this jobset triggerTime integer, -- set if we were triggered by a push event - enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot + enabled integer not null default 1, -- 0 = disabled, 1 = enabled, 2 = one-shot, 3 = one-at-a-time enableEmail integer not null default 1, hidden integer not null default 0, emailOverride text not null, From 113a312f6717ae81327b4d5bb91a16ae06e0933e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 3 Mar 2020 22:32:13 -0500 Subject: [PATCH 11/15] handleDeclarativeJobsetBuild: handle errors from readNixFile --- src/lib/Hydra/Helper/AddBuilds.pm | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 40375c6b..296afbc5 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -68,8 +68,14 @@ sub handleDeclarativeJobsetBuild { my $id = $build->id; die "Declarative jobset build $id failed" unless $build->buildstatus == 0; my $declPath = ($build->buildoutputs)[0]->path; - my $declText = readNixFile($declPath) - or die "Couldn't read declarative specification file $declPath: $!"; + my $declText = eval { + readNixFile($declPath) + }; + if ($@) { + print STDERR "ERROR: failed to readNixFile $declPath: ", $@, "\n"; + die; + } + my $declSpec = decode_json($declText); txn_do($db, sub { my @kept = keys %$declSpec; From 117b9ecef18e4bf63b84a9cce0b0d3ae7d8910e9 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 3 Mar 2020 22:36:21 -0500 Subject: [PATCH 12/15] =?UTF-8?q?Nix.pm:=20readNixFile:=20pass=20=C2=AB--e?= =?UTF-8?q?xperimental-features=20nix-command=C2=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Declarative jobsets were broken by the Nix update, causing nix cat-file to break silently. This commit restores declarative jobsets, based on top of a commit making it easier to see what broke. --- src/lib/Hydra/Helper/Nix.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 8ce284ad..5034c81b 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -509,7 +509,8 @@ sub getStoreUri { # Read a file from the (possibly remote) nix store sub readNixFile { my ($path) = @_; - return grab(cmd => ["nix", "cat-store", "--store", getStoreUri(), "$path"]); + return grab(cmd => ["nix", "--experimental-features", "nix-command", + "cat-store", "--store", getStoreUri(), "$path"]); } From 994430b94bb3d237ee8380a0942c58a44dc94c06 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 3 Mar 2020 22:46:32 -0500 Subject: [PATCH 13/15] treewide: allow `nix` command --- src/lib/Hydra/Controller/Build.pm | 6 ++++-- src/script/hydra-eval-jobset | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 22bfd98e..52cb71dd 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -193,7 +193,8 @@ sub checkPath { sub serveFile { my ($c, $path) = @_; - my $res = run(cmd => ["nix", "ls-store", "--store", getStoreUri(), "--json", "$path"]); + my $res = run(cmd => ["nix", "--experimental-features", "nix-command", + "ls-store", "--store", getStoreUri(), "--json", "$path"]); if ($res->{status}) { notFound($c, "File '$path' does not exist.") if $res->{stderr} =~ /does not exist/; @@ -217,7 +218,8 @@ sub serveFile { elsif ($ls->{type} eq "regular") { - $c->stash->{'plain'} = { data => grab(cmd => ["nix", "cat-store", "--store", getStoreUri(), "$path"]) }; + $c->stash->{'plain'} = { data => grab(cmd => ["nix", "--experimental-features", "nix-command", + "cat-store", "--store", getStoreUri(), "$path"]) }; # Detect MIME type. Borrowed from Catalyst::Plugin::Static::Simple. my $type = "text/plain"; diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index cdb09b74..53a16f38 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -82,7 +82,7 @@ sub getPath { my $substituter = $config->{eval_substituter}; - system("nix", "copy", "--from", $substituter, "--", $path) + system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path) if defined $substituter; return isValidPath($path); From 69a6f3448a1cb4252d3468b9bdc7c7e171847c92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Mar 2020 15:16:04 +0100 Subject: [PATCH 14/15] Fix calling job functions Fixes #718. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index deffaeae..f27fde1d 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -109,9 +109,10 @@ static void worker( nlohmann::json reply; try { - auto v = findAlongAttrPath(state, attrPath, autoArgs, *vRoot); + auto vTmp = findAlongAttrPath(state, attrPath, autoArgs, *vRoot); - state.forceValue(*v); + auto v = state.allocValue(); + state.autoCallFunction(autoArgs, *vTmp, *v); if (auto drv = getDerivation(state, *v, false)) { From 123bee1db599818f34f880a756a44339c57c7ade Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Mar 2020 15:16:26 +0100 Subject: [PATCH 15/15] Restore job type checking --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index f27fde1d..4cbe5a1c 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -194,6 +194,11 @@ static void worker( reply["attrs"] = std::move(attrs); } + else if (v->type == tNull) + ; + + else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v)); + } catch (EvalError & e) { reply["error"] = filterANSIEscapes(e.msg(), true); }
State:[% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; END %][% IF jobset.enabled == 0; "Disabled"; ELSIF jobset.enabled == 1; "Enabled"; ELSIF jobset.enabled == 2; "One-shot"; ELSIF jobset.enabled == 3; "One-at-a-time"; END %]
Description: