From d2fc382498b2cf231285c1376f4c1804f651523f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 15 Mar 2009 11:56:11 +0000 Subject: [PATCH] * Register GC roots properly. --- src/c/hydra_eval_jobs.cc | 19 +++++++++++++++++-- src/lib/Hydra/Helper/Nix.pm | 22 +++++++++++++--------- src/script/hydra_build.pl | 5 ++--- src/script/hydra_scheduler.pl | 8 ++++---- src/script/hydra_update_gc_roots.pl | 4 ++-- 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src/c/hydra_eval_jobs.cc b/src/c/hydra_eval_jobs.cc index f7561bef..79dc3a60 100644 --- a/src/c/hydra_eval_jobs.cc +++ b/src/c/hydra_eval_jobs.cc @@ -19,6 +19,9 @@ void printHelp() } +static Path gcRootsDir; + + Expr evalAttr(EvalState & state, Expr e) { return e ? evalExpr(state, e) : e; @@ -97,12 +100,12 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, if (getDerivation(state, e, drv)) { XMLAttrs xmlAttrs; - Path outPath, drvPath; + Path drvPath; xmlAttrs["jobName"] = attrPath; xmlAttrs["nixName"] = drv.name; xmlAttrs["system"] = drv.system; - xmlAttrs["drvPath"] = drv.queryDrvPath(state); + xmlAttrs["drvPath"] = drvPath = drv.queryDrvPath(state); xmlAttrs["outPath"] = drv.queryOutPath(state); xmlAttrs["description"] = drv.queryMetaInfo(state, "description"); xmlAttrs["longDescription"] = drv.queryMetaInfo(state, "longDescription"); @@ -110,6 +113,12 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, xmlAttrs["homepage"] = drv.queryMetaInfo(state, "homepage"); xmlAttrs["schedulingPriority"] = drv.queryMetaInfo(state, "schedulingPriority"); + /* Register the derivation as a GC root. !!! This + registers roots for jobs that we may have already + done. */ + Path root = gcRootsDir + "/" + baseNameOf(drvPath); + if (!pathExists(root)) addPermRoot(drvPath, root, false); + XMLOpenElement _(doc, "job", xmlAttrs); showArgsUsed(doc, argsUsed); } @@ -170,6 +179,10 @@ void run(Strings args) ? (ATermList) autoArgs.get(toATerm(name)) : ATempty, e)); } + else if (arg == "--gc-roots-dir") { + if (i == args.end()) throw UsageError("missing argument"); + gcRootsDir = *i++; + } else if (arg[0] == '-') throw UsageError(format("unknown flag `%1%'") % arg); else @@ -178,6 +191,8 @@ void run(Strings args) if (releaseExpr == "") throw UsageError("no expression specified"); + if (gcRootsDir == "") throw UsageError("--gc-roots-dir not specified"); + store = openStore(); Expr e = parseExprFromFile(state, releaseExpr); diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 1383202d..c6d73fc7 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -9,7 +9,7 @@ our @ISA = qw(Exporter); our @EXPORT = qw( isValidPath queryPathInfo getHydraPath getHydraDBPath openHydraDB - registerRoot getGCRootsDir + registerRoot getGCRootsDir gcRootFor getPrimaryBuildsForReleaseSet getRelease getLatestSuccessfulRelease ); @@ -80,22 +80,26 @@ sub openHydraDB { sub getGCRootsDir { die unless defined $ENV{LOGNAME}; - return "/nix/var/nix/gcroots/per-user/$ENV{LOGNAME}/hydra-roots"; + my $dir = "/nix/var/nix/gcroots/per-user/$ENV{LOGNAME}/hydra-roots"; + mkpath $dir if !-e $dir; + return $dir; +} + + +sub gcRootFor { + my ($path) = @_; + return getGCRootsDir . basename $path; } sub registerRoot { my ($path) = @_; - - my $gcRootsDir = getGCRootsDir; - mkpath($gcRootsDir) if !-e $gcRootsDir; - - my $link = "$gcRootsDir/" . basename $path; - + my $link = gcRootFor $path; + if (!-l $link) { symlink($path, $link) - or die "cannot create symlink in $gcRootsDir to $path"; + or die "cannot create GC root `$link' to `$path'"; } } diff --git a/src/script/hydra_build.pl b/src/script/hydra_build.pl index 490e1352..f1b443bf 100755 --- a/src/script/hydra_build.pl +++ b/src/script/hydra_build.pl @@ -28,8 +28,6 @@ sub doBuild { my $failedDepBuild; my $failedDepStepNr; - registerRoot $outPath; - if (!isValidPath($outPath)) { $isCachedBuild = 0; @@ -75,7 +73,8 @@ sub doBuild { # to get notifications about specific build steps, the # associated log files, etc. my $cmd = "nix-store --max-silent-time 1800 --keep-going --no-build-output " . - "--log-type flat --print-build-trace --realise $drvPath 2>&1"; + "--log-type flat --print-build-trace --realise $drvPath " . + "--add-root " . gcRootFor $outPath . " 2>&1"; my $buildStepNr = 1; diff --git a/src/script/hydra_scheduler.pl b/src/script/hydra_scheduler.pl index 1f5cf60c..27b303f3 100755 --- a/src/script/hydra_scheduler.pl +++ b/src/script/hydra_scheduler.pl @@ -267,9 +267,6 @@ sub checkJob { , sha256hash => $input->{sha256hash} }); } - - # !!! this should really by done by nix-instantiate to prevent a GC race. - registerRoot $drvPath; }); }; @@ -330,9 +327,12 @@ sub checkJobSet { $nixExprPath .= "/" . $jobset->nixexprpath; (my $res, my $jobsXml, my $stderr) = captureStdoutStderr( - "hydra_eval_jobs", $nixExprPath, inputsToArgs($inputInfo)); + "hydra_eval_jobs", $nixExprPath, "--gc-roots-dir", getGCRootsDir, + inputsToArgs($inputInfo)); die "cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res; + print STDERR "$stderr"; + my $jobs = XMLin($jobsXml, ForceArray => ['error', 'job', 'arg'], KeyAttr => [], diff --git a/src/script/hydra_update_gc_roots.pl b/src/script/hydra_update_gc_roots.pl index ca581e88..ea32013c 100755 --- a/src/script/hydra_update_gc_roots.pl +++ b/src/script/hydra_update_gc_roots.pl @@ -83,14 +83,14 @@ my @buildsToKeep = $db->resultset('Builds')->search({finished => 1, keep => 1}, keepBuild $_ foreach @buildsToKeep; -# For scheduled builds, we register the derivation and the output as a GC root. +# For scheduled builds, we register the derivation as a GC root. print STDERR "*** looking for scheduled builds\n"; foreach my $build ($db->resultset('Builds')->search({finished => 0}, {join => 'schedulingInfo'})) { if (isValidPath($build->drvpath)) { print STDERR "keeping scheduled build ", $build->id, " (", strftime("%Y-%m-%d %H:%M:%S", localtime($build->timestamp)), ")\n"; registerRoot $build->drvpath; - registerRoot $build->outpath; + registerRoot $build->outpath if -e $build->outpath; } else { print STDERR "warning: derivation ", $build->drvpath, " has disappeared\n"; }