Support building a derivation if some outputs are already valid
This handles the chroot and build hook cases, which are easy. Supporting the non-chroot-build case will require more work (hash rewriting!). Issue #21.
This commit is contained in:
parent
295027f533
commit
a2785b7391
|
@ -6,6 +6,7 @@ use IO::Handle;
|
||||||
use Nix::Config;
|
use Nix::Config;
|
||||||
use Nix::SSH qw/sshOpts openSSHConnection/;
|
use Nix::SSH qw/sshOpts openSSHConnection/;
|
||||||
use Nix::CopyClosure;
|
use Nix::CopyClosure;
|
||||||
|
use Nix::Store;
|
||||||
no warnings('once');
|
no warnings('once');
|
||||||
|
|
||||||
|
|
||||||
|
@ -18,7 +19,7 @@ no warnings('once');
|
||||||
# less than the maximum load for that machine, we try to get an
|
# less than the maximum load for that machine, we try to get an
|
||||||
# exclusive lock on $currentLoad/$machine-$slot (without blocking).
|
# exclusive lock on $currentLoad/$machine-$slot (without blocking).
|
||||||
# If we get such a lock, we send "accept" to the caller. Otherwise,
|
# If we get such a lock, we send "accept" to the caller. Otherwise,
|
||||||
# we send "postpone" and exit.
|
# we send "postpone" and exit.
|
||||||
# - We release the exclusive lock on $currentLoad/main-lock.
|
# - We release the exclusive lock on $currentLoad/main-lock.
|
||||||
# - We perform the build on $neededSystem.
|
# - We perform the build on $neededSystem.
|
||||||
# - We release the exclusive lock on $currentLoad/$machine-$slot.
|
# - We release the exclusive lock on $currentLoad/$machine-$slot.
|
||||||
|
@ -102,14 +103,14 @@ REQ: while (1) {
|
||||||
sendReply "decline";
|
sendReply "decline";
|
||||||
next;
|
next;
|
||||||
}
|
}
|
||||||
|
|
||||||
# Acquire the exclusive lock on $currentLoad/main-lock.
|
# Acquire the exclusive lock on $currentLoad/main-lock.
|
||||||
mkdir $currentLoad, 0777 or die unless -d $currentLoad;
|
mkdir $currentLoad, 0777 or die unless -d $currentLoad;
|
||||||
my $mainLock = "$currentLoad/main-lock";
|
my $mainLock = "$currentLoad/main-lock";
|
||||||
sysopen MAINLOCK, "$mainLock", O_RDWR|O_CREAT, 0600 or die;
|
sysopen MAINLOCK, "$mainLock", O_RDWR|O_CREAT, 0600 or die;
|
||||||
flock(MAINLOCK, LOCK_EX) or die;
|
flock(MAINLOCK, LOCK_EX) or die;
|
||||||
|
|
||||||
|
|
||||||
while (1) {
|
while (1) {
|
||||||
# Find all machine that can execute this build, i.e., that
|
# Find all machine that can execute this build, i.e., that
|
||||||
# support builds for the given platform and features, and are
|
# support builds for the given platform and features, and are
|
||||||
|
@ -141,7 +142,7 @@ REQ: while (1) {
|
||||||
close $slotLock;
|
close $slotLock;
|
||||||
$slot++;
|
$slot++;
|
||||||
}
|
}
|
||||||
|
|
||||||
push @available, { machine => $cur, load => $load, free => $free }
|
push @available, { machine => $cur, load => $load, free => $free }
|
||||||
if $load < $cur->{maxJobs};
|
if $load < $cur->{maxJobs};
|
||||||
}
|
}
|
||||||
|
@ -160,7 +161,7 @@ REQ: while (1) {
|
||||||
if ($rightType && !$canBuildLocally) {
|
if ($rightType && !$canBuildLocally) {
|
||||||
sendReply "postpone";
|
sendReply "postpone";
|
||||||
} else {
|
} else {
|
||||||
sendReply "decline";
|
sendReply "decline";
|
||||||
}
|
}
|
||||||
close MAINLOCK;
|
close MAINLOCK;
|
||||||
next REQ;
|
next REQ;
|
||||||
|
@ -182,9 +183,9 @@ REQ: while (1) {
|
||||||
|
|
||||||
|
|
||||||
# Select the best available machine and lock a free slot.
|
# Select the best available machine and lock a free slot.
|
||||||
my $selected = $available[0];
|
my $selected = $available[0];
|
||||||
my $machine = $selected->{machine};
|
my $machine = $selected->{machine};
|
||||||
|
|
||||||
$slotLock = openSlotLock($machine, $selected->{free});
|
$slotLock = openSlotLock($machine, $selected->{free});
|
||||||
flock($slotLock, LOCK_EX | LOCK_NB) or die;
|
flock($slotLock, LOCK_EX | LOCK_NB) or die;
|
||||||
utime undef, undef, $slotLock;
|
utime undef, undef, $slotLock;
|
||||||
|
@ -196,7 +197,7 @@ REQ: while (1) {
|
||||||
@sshOpts = ("-i", $machine->{sshKeys}, "-x");
|
@sshOpts = ("-i", $machine->{sshKeys}, "-x");
|
||||||
$hostName = $machine->{hostName};
|
$hostName = $machine->{hostName};
|
||||||
last REQ if openSSHConnection $hostName;
|
last REQ if openSSHConnection $hostName;
|
||||||
|
|
||||||
warn "unable to open SSH connection to $hostName, trying other available machines...\n";
|
warn "unable to open SSH connection to $hostName, trying other available machines...\n";
|
||||||
$machine->{enabled} = 0;
|
$machine->{enabled} = 0;
|
||||||
}
|
}
|
||||||
|
@ -286,10 +287,10 @@ if (system("exec ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store
|
||||||
foreach my $output (@outputs) {
|
foreach my $output (@outputs) {
|
||||||
my $maybeSignRemote = "";
|
my $maybeSignRemote = "";
|
||||||
$maybeSignRemote = "--sign" if $UID != 0;
|
$maybeSignRemote = "--sign" if $UID != 0;
|
||||||
|
next if isValidPath($output);
|
||||||
system("exec ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output'" .
|
system("exec ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output'" .
|
||||||
"| NIX_HELD_LOCKS=$output @bindir@/nix-store --import > /dev/null") == 0
|
"| NIX_HELD_LOCKS=$output @bindir@/nix-store --import > /dev/null") == 0
|
||||||
or die "cannot copy $output from $hostName: $?";
|
or die "cannot copy $output from $hostName: $?";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -756,6 +756,9 @@ private:
|
||||||
/* Referenceable paths (i.e., input and output paths). */
|
/* Referenceable paths (i.e., input and output paths). */
|
||||||
PathSet allPaths;
|
PathSet allPaths;
|
||||||
|
|
||||||
|
/* Outputs that are already valid. */
|
||||||
|
PathSet validPaths;
|
||||||
|
|
||||||
/* User selected for running the builder. */
|
/* User selected for running the builder. */
|
||||||
UserLock buildUser;
|
UserLock buildUser;
|
||||||
|
|
||||||
|
@ -1141,7 +1144,7 @@ void DerivationGoal::tryToBuild()
|
||||||
omitted, but that would be less efficient.) Note that since we
|
omitted, but that would be less efficient.) Note that since we
|
||||||
now hold the locks on the output paths, no other process can
|
now hold the locks on the output paths, no other process can
|
||||||
build this derivation, so no further checks are necessary. */
|
build this derivation, so no further checks are necessary. */
|
||||||
PathSet validPaths = checkPathValidity(true);
|
validPaths = checkPathValidity(true);
|
||||||
if (validPaths.size() == drv.outputs.size()) {
|
if (validPaths.size() == drv.outputs.size()) {
|
||||||
debug(format("skipping build of derivation `%1%', someone beat us to it")
|
debug(format("skipping build of derivation `%1%', someone beat us to it")
|
||||||
% drvPath);
|
% drvPath);
|
||||||
|
@ -1150,22 +1153,16 @@ void DerivationGoal::tryToBuild()
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (validPaths.size() > 0)
|
printMsg(lvlError, format("BLOCKERS: %1%") % showPaths(validPaths));
|
||||||
/* !!! fix this; try to delete valid paths */
|
|
||||||
throw Error(
|
|
||||||
format("derivation `%1%' is blocked by its output paths")
|
|
||||||
% drvPath);
|
|
||||||
|
|
||||||
/* If any of the outputs already exist but are not valid, delete
|
/* If any of the outputs already exist but are not valid, delete
|
||||||
them. */
|
them. */
|
||||||
foreach (DerivationOutputs::iterator, i, drv.outputs) {
|
foreach (DerivationOutputs::iterator, i, drv.outputs) {
|
||||||
Path path = i->second.path;
|
Path path = i->second.path;
|
||||||
if (worker.store.isValidPath(path))
|
if (worker.store.isValidPath(path)) continue;
|
||||||
throw Error(format("obstructed build: path `%1%' exists") % path);
|
if (!pathExists(path)) continue;
|
||||||
if (pathExists(path)) {
|
debug(format("removing unregistered path `%1%'") % path);
|
||||||
debug(format("removing unregistered path `%1%'") % path);
|
deletePathWrapped(path);
|
||||||
deletePathWrapped(path);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Check again whether any output previously failed to build,
|
/* Check again whether any output previously failed to build,
|
||||||
|
@ -1283,6 +1280,10 @@ void DerivationGoal::buildDone()
|
||||||
Path path = i->second.path;
|
Path path = i->second.path;
|
||||||
|
|
||||||
if (useChroot && pathExists(chrootRootDir + path)) {
|
if (useChroot && pathExists(chrootRootDir + path)) {
|
||||||
|
/* Move output paths from the chroot to the Nix store.
|
||||||
|
If the output was already valid, just skip
|
||||||
|
(discard) it. */
|
||||||
|
if (validPaths.find(path) != validPaths.end()) continue;
|
||||||
if (rename((chrootRootDir + path).c_str(), path.c_str()) == -1)
|
if (rename((chrootRootDir + path).c_str(), path.c_str()) == -1)
|
||||||
throw SysError(format("moving build output `%1%' from the chroot to the Nix store") % path);
|
throw SysError(format("moving build output `%1%' from the chroot to the Nix store") % path);
|
||||||
}
|
}
|
||||||
|
@ -1747,6 +1748,9 @@ void DerivationGoal::startBuilder()
|
||||||
#else
|
#else
|
||||||
throw Error("chroot builds are not supported on this platform");
|
throw Error("chroot builds are not supported on this platform");
|
||||||
#endif
|
#endif
|
||||||
|
} else { // !useChroot
|
||||||
|
if (validPaths.size() > 0)
|
||||||
|
throw Error(format("derivation `%1%' is blocked by its output paths %2%") % drvPath % showPaths(validPaths));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue