Merge pull request #1126 from DeterminateSystems/build-localhost-paths

build-remote: copy missing paths from the binary cache to localhost

src/hydra-queue-runner/build-remote.cc

@@ -270,17 +270,24 @@ void State::buildRemote(ref<Store> destStore,
             copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
         }
 
-        /* Copy the input closure. */
-        if (!machine->isLocalhost()) {
+        {
             auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting);
             mc1.reset();
             MaintainCount<counter> mc2(nrStepsCopyingTo);
+
             printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
                 localStore->printStorePath(step->drvPath), machine->sshName);
 
             auto now1 = std::chrono::steady_clock::now();
 
-            copyClosureTo(machine->state->sendLock, destStore, from, to, inputs, true);
+            /* Copy the input closure. */
+            if (machine->isLocalhost()) {
+                StorePathSet closure;
+                destStore->computeFSClosure(inputs, closure);
+                copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
+            } else {
+                copyClosureTo(machine->state->sendLock, destStore, from, to, inputs, true);
+            }
 
             auto now2 = std::chrono::steady_clock::now();
 

t/jobs/dependencies/dependency.nix

@@ -0,0 +1,17 @@
+{ exposeUnderlyingJob, exposeDependentJob }:
+with import ../config.nix;
+let
+  underlyingJob = mkDerivation {
+    name = "underlying-job";
+    builder = ../empty-dir-builder.sh;
+  };
+
+  dependentJob = mkDerivation {
+    name = "dependent-job";
+    builder = ../empty-dir-builder.sh;
+    inherit underlyingJob;
+  };
+in
+(if exposeUnderlyingJob then { inherit underlyingJob; } else { }) //
+(if exposeDependentJob then { inherit dependentJob; } else { }) //
+{ }

t/jobs/dependencies/dependentOnly.nix

@@ -0,0 +1,4 @@
+import ./dependency.nix {
+  exposeUnderlyingJob = false;
+  exposeDependentJob = true;
+}

t/jobs/dependencies/underlyingOnly.nix

@@ -0,0 +1,4 @@
+import ./dependency.nix {
+  exposeUnderlyingJob = true;
+  exposeDependentJob = false;
+}

t/lib/CliRunners.pm

@@ -37,6 +37,8 @@ sub evalSucceeds {
     $jobset->discard_changes;  # refresh from DB
     if ($res) {
         chomp $stdout; chomp $stderr;
+        utf8::decode($stdout) or die "Invalid unicode in stdout.";
+        utf8::decode($stderr) or die "Invalid unicode in stderr.";
         print STDERR "Evaluation unexpectedly failed for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
         print STDERR "STDOUT: $stdout\n" if $stdout ne "";
         print STDERR "STDERR: $stderr\n" if $stderr ne "";
@@ -50,6 +52,8 @@ sub evalFails {
     $jobset->discard_changes;  # refresh from DB
     if (!$res) {
         chomp $stdout; chomp $stderr;
+        utf8::decode($stdout) or die "Invalid unicode in stdout.";
+        utf8::decode($stderr) or die "Invalid unicode in stderr.";
         print STDERR "Evaluation unexpectedly succeeded for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
         print STDERR "STDOUT: $stdout\n" if $stdout ne "";
         print STDERR "STDERR: $stderr\n" if $stderr ne "";
@@ -61,6 +65,8 @@ sub runBuild {
     my ($build) = @_;
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id));
     if ($res) {
+        utf8::decode($stdout) or die "Invalid unicode in stdout.";
+        utf8::decode($stderr) or die "Invalid unicode in stderr.";
         print STDERR "Queue runner stdout: $stdout\n" if $stdout ne "";
         print STDERR "Queue runner stderr: $stderr\n" if $stderr ne "";
     }
@@ -70,6 +76,8 @@ sub runBuild {
 sub sendNotifications() {
     my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-notify", "--queued-only"));
     if ($res) {
+        utf8::decode($stdout) or die "Invalid unicode in stdout.";
+        utf8::decode($stderr) or die "Invalid unicode in stderr.";
         print STDERR "hydra notify stdout: $stdout\n" if $stdout ne "";
         print STDERR "hydra notify stderr: $stderr\n" if $stderr ne "";
     }

t/lib/HydraTestContext.pm

@@ -69,6 +69,7 @@ sub new {
         _db => undef,
         db_handle => $pgsql,
         tmpdir => $dir,
+        nix_state_dir => "$dir/nix/var/nix",
         testdir => abs_path(dirname(__FILE__) . "/.."),
         jobsdir => abs_path(dirname(__FILE__) . "/../jobs")
     };
@@ -115,6 +116,12 @@ sub jobsdir {
     return $self->{jobsdir};
 }
 
+sub nix_state_dir {
+    my ($self) = @_;
+
+    return $self->{nix_state_dir};
+}
+
 # Create a jobset, evaluate it, and optionally build the jobs.
 #
 # In return, you get a hash of all the Builds records, keyed

t/queue-runner/build-locally-with-substitutable-path.t

@@ -0,0 +1,51 @@
+use strict;
+use warnings;
+use Setup;
+use Data::Dumper;
+use Test2::V0;
+my $ctx = test_context(
+    use_external_destination_store => 1
+);
+
+# This test is regarding https://github.com/NixOS/hydra/pull/1126
+#
+# A hydra instance was regularly failing to build derivations with:
+#
+#     possibly transient failure building ‘/nix/store/X.drv’ on ‘localhost’:
+#     dependency '/nix/store/Y' of '/nix/store/Y.drv' does not exist,
+#     and substitution is disabled
+#
+# However it would only fail when building on localhost, and it would only
+# fail if the build output was already in the binary cache.
+#
+# This test replicates this scenario by having two jobs, underlyingJob and
+# dependentJob. dependentJob depends on underlyingJob. We first build
+# underlyingJob and copy it to an external cache. Then forcefully delete
+# the output of underlyingJob, and build dependentJob. In order to pass
+# it must either rebuild underlyingJob or fetch it from the cache.
+
+
+subtest "Building, caching, and then garbage collecting the underlying job" => sub {
+    my $builds = $ctx->makeAndEvaluateJobset(
+        expression => "dependencies/underlyingOnly.nix",
+        build => 1
+    );
+
+    my $path = $builds->{"underlyingJob"}->buildoutputs->find({ name => "out" })->path;
+
+    ok(unlink(Hydra::Helper::Nix::gcRootFor($path)), "Unlinking the GC root for underlying Dependency succeeds");
+
+    (my $ret, my $stdout, my $stderr) = captureStdoutStderr(1, "nix-store", "--delete", $path);
+    is($ret, 0, "Deleting the underlying dependency should succeed");
+};
+
+subtest "Building the dependent job should now succeed, even though we're missing a local dependency" => sub {
+    my $builds = $ctx->makeAndEvaluateJobset(
+        expression => "dependencies/dependentOnly.nix"
+    );
+
+    ok(runBuild($builds->{"dependentJob"}), "building the job should succeed");
+};
+
+
+done_testing;