From fb5f01097b0c72cbd3b9a0541a96d656e95e8a1d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Mon, 14 Jul 2014 13:18:07 +0200
Subject: [PATCH] Fix race between hydra-eval-jobs and hydra-update-gc-roots

If hydra-eval-jobs creates a new root, and hydra-update-gc-roots runs
before hydra-evaluator has had a chance to add the corresponding build
to the database, then hydra-update-gc-roots will remove the root. If
subsequently the Nix garbage collector kicks in, it may remove the
build's .drv file before the build is performed. Since evaluation of
the Nixpkgs and NixOS jobsets nowadays takes a lot of time (e.g. an
hour), the probability of this happening is fairly high.

The quick fix is not to delete roots that are less than a day old. So
long as evaluation doesn't take longer than a day, this should be fine
;-)

Fixes #166.
---
 src/script/hydra-update-gc-roots | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/script/hydra-update-gc-roots b/src/script/hydra-update-gc-roots
index 69e7cf20..4ea91444 100755
--- a/src/script/hydra-update-gc-roots
+++ b/src/script/hydra-update-gc-roots
@@ -2,6 +2,7 @@
 
 use strict;
 use File::Path;
+use File::stat;
 use File::Basename;
 use Nix::Store;
 use Hydra::Schema;
@@ -48,9 +49,7 @@ sub keepBuild {
 }
 
 
-# Read the current GC roots.  We need to do that here so that we don't
-# delete roots that were added while we were determining the desired
-# roots.
+# Read the current GC roots.
 print STDERR "*** reading current roots...\n";
 my $gcRootsDir = getGCRootsDir;
 opendir DIR, $gcRootsDir or die;
@@ -122,14 +121,23 @@ print STDERR "*** removing unneeded GC roots\n";
 
 my $rootsKept = 0;
 my $rootsDeleted = 0;
+my $now = time();
 
 foreach my $link (@roots) {
     next if $link eq "." || $link eq "..";
     my $path = "/nix/store/$link";
     if (!defined $roots{$path}) {
-        print STDERR "removing root $path\n";
-        $rootsDeleted++;
-        unlink "$gcRootsDir/$link" or warn "cannot remove $gcRootsDir/$link";
+        # Don't delete roots that are less than a day old, to prevent
+        # a race where hydra-eval-jobs has added a root but
+        # hydra-evaluator hasn't added them to the database yet.
+        if (lstat($path)->ctime < $now - 24 * 60 * 60) {
+            print STDERR "removing root $path\n";
+            $rootsDeleted++;
+            unlink "$gcRootsDir/$link" or warn "cannot remove $gcRootsDir/$link";
+        } else {
+            print STDERR "NOT removing recent root $path\n";
+            $rootsKept++;
+        }
     } else {
         $rootsKept++;
     }