Merge pull request #1126 from DeterminateSystems/build-localhost-paths

build-remote: copy missing paths from the binary cache to localhost
This commit is contained in:
Graham Christensen 2022-01-21 16:16:33 -05:00 committed by GitHub
commit c280692f91
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 101 additions and 3 deletions

View file

@ -270,17 +270,24 @@ void State::buildRemote(ref<Store> destStore,
copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute); copyPaths(*localStore, *destStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
} }
/* Copy the input closure. */ {
if (!machine->isLocalhost()) {
auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting); auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting);
mc1.reset(); mc1.reset();
MaintainCount<counter> mc2(nrStepsCopyingTo); MaintainCount<counter> mc2(nrStepsCopyingTo);
printMsg(lvlDebug, "sending closure of %s to %s", printMsg(lvlDebug, "sending closure of %s to %s",
localStore->printStorePath(step->drvPath), machine->sshName); localStore->printStorePath(step->drvPath), machine->sshName);
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
/* Copy the input closure. */
if (machine->isLocalhost()) {
StorePathSet closure;
destStore->computeFSClosure(inputs, closure);
copyPaths(*destStore, *localStore, closure, NoRepair, NoCheckSigs, NoSubstitute);
} else {
copyClosureTo(machine->state->sendLock, destStore, from, to, inputs, true); copyClosureTo(machine->state->sendLock, destStore, from, to, inputs, true);
}
auto now2 = std::chrono::steady_clock::now(); auto now2 = std::chrono::steady_clock::now();

View file

@ -0,0 +1,17 @@
{ exposeUnderlyingJob, exposeDependentJob }:
with import ../config.nix;
let
underlyingJob = mkDerivation {
name = "underlying-job";
builder = ../empty-dir-builder.sh;
};
dependentJob = mkDerivation {
name = "dependent-job";
builder = ../empty-dir-builder.sh;
inherit underlyingJob;
};
in
(if exposeUnderlyingJob then { inherit underlyingJob; } else { }) //
(if exposeDependentJob then { inherit dependentJob; } else { }) //
{ }

View file

@ -0,0 +1,4 @@
import ./dependency.nix {
exposeUnderlyingJob = false;
exposeDependentJob = true;
}

View file

@ -0,0 +1,4 @@
import ./dependency.nix {
exposeUnderlyingJob = true;
exposeDependentJob = false;
}

View file

@ -37,6 +37,8 @@ sub evalSucceeds {
$jobset->discard_changes; # refresh from DB $jobset->discard_changes; # refresh from DB
if ($res) { if ($res) {
chomp $stdout; chomp $stderr; chomp $stdout; chomp $stderr;
utf8::decode($stdout) or die "Invalid unicode in stdout.";
utf8::decode($stderr) or die "Invalid unicode in stderr.";
print STDERR "Evaluation unexpectedly failed for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg; print STDERR "Evaluation unexpectedly failed for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
print STDERR "STDOUT: $stdout\n" if $stdout ne ""; print STDERR "STDOUT: $stdout\n" if $stdout ne "";
print STDERR "STDERR: $stderr\n" if $stderr ne ""; print STDERR "STDERR: $stderr\n" if $stderr ne "";
@ -50,6 +52,8 @@ sub evalFails {
$jobset->discard_changes; # refresh from DB $jobset->discard_changes; # refresh from DB
if (!$res) { if (!$res) {
chomp $stdout; chomp $stderr; chomp $stdout; chomp $stderr;
utf8::decode($stdout) or die "Invalid unicode in stdout.";
utf8::decode($stderr) or die "Invalid unicode in stderr.";
print STDERR "Evaluation unexpectedly succeeded for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg; print STDERR "Evaluation unexpectedly succeeded for jobset ".$jobset->project->name.":".$jobset->name.": \n".$jobset->errormsg."\n" if $jobset->errormsg;
print STDERR "STDOUT: $stdout\n" if $stdout ne ""; print STDERR "STDOUT: $stdout\n" if $stdout ne "";
print STDERR "STDERR: $stderr\n" if $stderr ne ""; print STDERR "STDERR: $stderr\n" if $stderr ne "";
@ -61,6 +65,8 @@ sub runBuild {
my ($build) = @_; my ($build) = @_;
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id)); my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-queue-runner", "-vvvv", "--build-one", $build->id));
if ($res) { if ($res) {
utf8::decode($stdout) or die "Invalid unicode in stdout.";
utf8::decode($stderr) or die "Invalid unicode in stderr.";
print STDERR "Queue runner stdout: $stdout\n" if $stdout ne ""; print STDERR "Queue runner stdout: $stdout\n" if $stdout ne "";
print STDERR "Queue runner stderr: $stderr\n" if $stderr ne ""; print STDERR "Queue runner stderr: $stderr\n" if $stderr ne "";
} }
@ -70,6 +76,8 @@ sub runBuild {
sub sendNotifications() { sub sendNotifications() {
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-notify", "--queued-only")); my ($res, $stdout, $stderr) = captureStdoutStderr(60, ("hydra-notify", "--queued-only"));
if ($res) { if ($res) {
utf8::decode($stdout) or die "Invalid unicode in stdout.";
utf8::decode($stderr) or die "Invalid unicode in stderr.";
print STDERR "hydra notify stdout: $stdout\n" if $stdout ne ""; print STDERR "hydra notify stdout: $stdout\n" if $stdout ne "";
print STDERR "hydra notify stderr: $stderr\n" if $stderr ne ""; print STDERR "hydra notify stderr: $stderr\n" if $stderr ne "";
} }

View file

@ -69,6 +69,7 @@ sub new {
_db => undef, _db => undef,
db_handle => $pgsql, db_handle => $pgsql,
tmpdir => $dir, tmpdir => $dir,
nix_state_dir => "$dir/nix/var/nix",
testdir => abs_path(dirname(__FILE__) . "/.."), testdir => abs_path(dirname(__FILE__) . "/.."),
jobsdir => abs_path(dirname(__FILE__) . "/../jobs") jobsdir => abs_path(dirname(__FILE__) . "/../jobs")
}; };
@ -115,6 +116,12 @@ sub jobsdir {
return $self->{jobsdir}; return $self->{jobsdir};
} }
sub nix_state_dir {
my ($self) = @_;
return $self->{nix_state_dir};
}
# Create a jobset, evaluate it, and optionally build the jobs. # Create a jobset, evaluate it, and optionally build the jobs.
# #
# In return, you get a hash of all the Builds records, keyed # In return, you get a hash of all the Builds records, keyed

View file

@ -0,0 +1,51 @@
use strict;
use warnings;
use Setup;
use Data::Dumper;
use Test2::V0;
my $ctx = test_context(
use_external_destination_store => 1
);
# This test is regarding https://github.com/NixOS/hydra/pull/1126
#
# A hydra instance was regularly failing to build derivations with:
#
# possibly transient failure building /nix/store/X.drv on localhost:
# dependency '/nix/store/Y' of '/nix/store/Y.drv' does not exist,
# and substitution is disabled
#
# However it would only fail when building on localhost, and it would only
# fail if the build output was already in the binary cache.
#
# This test replicates this scenario by having two jobs, underlyingJob and
# dependentJob. dependentJob depends on underlyingJob. We first build
# underlyingJob and copy it to an external cache. Then forcefully delete
# the output of underlyingJob, and build dependentJob. In order to pass
# it must either rebuild underlyingJob or fetch it from the cache.
subtest "Building, caching, and then garbage collecting the underlying job" => sub {
my $builds = $ctx->makeAndEvaluateJobset(
expression => "dependencies/underlyingOnly.nix",
build => 1
);
my $path = $builds->{"underlyingJob"}->buildoutputs->find({ name => "out" })->path;
ok(unlink(Hydra::Helper::Nix::gcRootFor($path)), "Unlinking the GC root for underlying Dependency succeeds");
(my $ret, my $stdout, my $stderr) = captureStdoutStderr(1, "nix-store", "--delete", $path);
is($ret, 0, "Deleting the underlying dependency should succeed");
};
subtest "Building the dependent job should now succeed, even though we're missing a local dependency" => sub {
my $builds = $ctx->makeAndEvaluateJobset(
expression => "dependencies/dependentOnly.nix"
);
ok(runBuild($builds->{"dependentJob"}), "building the job should succeed");
};
done_testing;