lix-project/lix
20
58
Fork 31
Code Issues 359 Code Review (Gerrit) Projects 3 Releases Wiki Activity

* Acquire the locks on the output paths before trying to run the build

Browse source 
hook.  This fixes a problem with log files being partially or
  completely filled with 0's because another nix-store process
  truncates the log file.  It should also be more efficient.
This commit is contained in:
Eelco Dolstra 2009-03-18 14:48:42 +00:00
parent 1dcf208f56
commit c183ee5c79
4 changed files with 78 additions and 150 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

208
src/libstore/build.cc
View file

@ -108,7 +108,6 @@ protected:
{ {
nrFailed = 0; nrFailed = 0;
exitCode = ecBusy; exitCode = ecBusy;
forceInputs = false;
} }
virtual ~Goal() virtual ~Goal()
@ -116,8 +115,6 @@ protected:
trace("goal destroyed"); trace("goal destroyed");
} }
bool forceInputs;
public: public:
virtual void work() = 0; virtual void work() = 0;
@ -152,11 +149,6 @@ public:
(important!), etc. */ (important!), etc. */
virtual void cancel() = 0; virtual void cancel() = 0;
void setForceInputs(bool x)
{
forceInputs = x;
}
protected: protected:
void amDone(ExitCode result); void amDone(ExitCode result);
}; };
@ -264,7 +256,7 @@ public:
MakeError(SubstError, Error) MakeError(SubstError, Error)
MakeError(BuildError, Error) MakeError(BuildError, Error) /* denoted a permanent build failure */
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
@ -496,7 +488,7 @@ void UserLock::acquire()
} }
} }
throw BuildError(format("all build users are currently in use; " throw Error(format("all build users are currently in use; "
"consider creating additional users and adding them to the `%1%' group") "consider creating additional users and adding them to the `%1%' group")
% buildUsersGroup); % buildUsersGroup);
} }
@ -689,7 +681,7 @@ private:
void buildDone(); void buildDone();
/* Is the build hook willing to perform the build? */ /* Is the build hook willing to perform the build? */
typedef enum {rpAccept, rpDecline, rpPostpone, rpDone, rpRestart} HookReply; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
HookReply tryBuildHook(); HookReply tryBuildHook();
/* Synchronously wait for a build hook to finish. */ /* Synchronously wait for a build hook to finish. */
@ -838,11 +830,9 @@ void DerivationGoal::haveDerivation()
/* If they are all valid, then we're done. */ /* If they are all valid, then we're done. */
if (invalidOutputs.size() == 0) { if (invalidOutputs.size() == 0) {
if(!forceInputs) {
amDone(ecSuccess); amDone(ecSuccess);
return; return;
} }
}
/* If this is a fixed-output derivation, it is possible that some /* If this is a fixed-output derivation, it is possible that some
other goal is already building the output paths. (The case other goal is already building the output paths. (The case
@ -883,55 +873,21 @@ void DerivationGoal::outputsSubstituted()
nrFailed = 0; nrFailed = 0;
if (checkPathValidity(false).size() == 0) { if (checkPathValidity(false).size() == 0) {
if (! forceInputs){
amDone(ecSuccess); amDone(ecSuccess);
return; return;
} }
}
/* Otherwise, at least one of the output paths could not be /* Otherwise, at least one of the output paths could not be
produced using a substitute. So we have to build instead. */ produced using a substitute. So we have to build instead. */
/* The inputs must be built before we can build this goal. */ /* The inputs must be built before we can build this goal. */
/* !!! but if possible, only install the paths that we need */ foreach (DerivationInputs::iterator, i, drv.inputDrvs)
for (DerivationInputs::iterator i = drv.inputDrvs.begin(); addWaitee(worker.makeDerivationGoal(i->first));
i != drv.inputDrvs.end(); ++i){
GoalPtr newGoal = worker.makeDerivationGoal(i->first);
newGoal->setForceInputs(forceInputs);
addWaitee(newGoal);
}
for (PathSet::iterator i = drv.inputSrcs.begin(); for (PathSet::iterator i = drv.inputSrcs.begin();
i != drv.inputSrcs.end(); ++i) i != drv.inputSrcs.end(); ++i)
addWaitee(worker.makeSubstitutionGoal(*i)); addWaitee(worker.makeSubstitutionGoal(*i));
/* Actually, I do some work twice just to be on the safe side */
string s = drv.env["exportBuildReferencesGraph"];
Strings ss = tokenizeString(s);
if (ss.size() % 2 !=0)
throw BuildError(format("odd number of tokens in `exportBuildReferencesGraph': `%1%'") % s);
for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
string fileName = *i++;
Path storePath=*i++;
if (!isInStore(storePath))
throw BuildError(format("`exportBuildReferencesGraph' contains a non-store path `%1%'")
% storePath);
storePath = toStorePath(storePath);
if (!worker.store.isValidPath(storePath))
throw BuildError(format("`exportBuildReferencesGraph' contains an invalid path `%1%'")
% storePath);
/* Build-time closure should be in dependencies
* We really want just derivation, its closure
* and outputs. Looks like we should build it.
* */
GoalPtr newGoal = worker.makeDerivationGoal(storePath);
newGoal->setForceInputs(true);
addWaitee(newGoal);
}
state = &DerivationGoal::inputsRealised; state = &DerivationGoal::inputsRealised;
} }
@ -943,18 +899,49 @@ void DerivationGoal::inputsRealised()
if (nrFailed != 0) { if (nrFailed != 0) {
printMsg(lvlError, printMsg(lvlError,
format("cannot build derivation `%1%': " format("cannot build derivation `%1%': "
"%2% inputs could not be realised") "%2% dependencies couldn't be built")
% drvPath % nrFailed); % drvPath % nrFailed);
amDone(ecFailed); amDone(ecFailed);
return; return;
} }
/* Maybe we just wanted to force build of inputs */ /* Gather information necessary for computing the closure and/or
if (checkPathValidity(false).size() == 0) { running the build hook. */
amDone(ecSuccess);
return; /* The outputs are referenceable paths. */
foreach (DerivationOutputs::iterator, i, drv.outputs) {
debug(format("building path `%1%'") % i->second.path);
allPaths.insert(i->second.path);
} }
/* Determine the full set of input paths. */
/* First, the input derivations. */
foreach (DerivationInputs::iterator, i, drv.inputDrvs) {
/* Add the relevant output closures of the input derivation
`*i' as input paths. Only add the closures of output paths
that are specified as inputs. */
assert(worker.store.isValidPath(i->first));
Derivation inDrv = derivationFromPath(i->first);
foreach (StringSet::iterator, j, i->second)
if (inDrv.outputs.find(*j) != inDrv.outputs.end())
computeFSClosure(inDrv.outputs[*j].path, inputPaths);
else
throw Error(
format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'")
% drvPath % *j % i->first);
}
/* Second, the input sources. */
foreach (PathSet::iterator, i, drv.inputSrcs)
computeFSClosure(*i, inputPaths);
/* !!! if `*i' is a derivation, then add the closure of every
output in the dependency graph to `inputPaths'. */
debug(format("added input paths %1%") % showPaths(inputPaths));
allPaths.insert(inputPaths.begin(), inputPaths.end());
/* Okay, try to build. Note that here we don't wait for a build /* Okay, try to build. Note that here we don't wait for a build
slot to become available, since we don't need one if there is a slot to become available, since we don't need one if there is a
build hook. */ build hook. */
@ -967,6 +954,20 @@ void DerivationGoal::tryToBuild()
{ {
trace("trying to build"); trace("trying to build");
/* Acquire locks on the output paths. After acquiring the locks,
it might turn out that somebody else has performed the build
already, and we're done. If not, we can proceed. */
switch (prepareBuild()) {
case prDone:
amDone(ecSuccess);
return;
case prRestart:
worker.waitForAnyGoal(shared_from_this());
return;
case prProceed:
break;
}
try { try {
/* Is the build hook willing to accept this job? */ /* Is the build hook willing to accept this job? */
@ -980,38 +981,19 @@ void DerivationGoal::tryToBuild()
case rpPostpone: case rpPostpone:
/* Not now; wait until at least one child finishes. */ /* Not now; wait until at least one child finishes. */
worker.waitForChildTermination(shared_from_this()); worker.waitForChildTermination(shared_from_this());
outputLocks.unlock();
return; return;
case rpDecline: case rpDecline:
/* We should do it ourselves. */ /* We should do it ourselves. */
break; break;
case rpDone:
/* Somebody else did it. */
amDone(ecSuccess);
return;
case rpRestart:
/* Somebody else is building this output path.
Restart from haveDerivation(). */
state = &DerivationGoal::haveDerivation;
worker.waitForAnyGoal(shared_from_this());
return;
} }
usingBuildHook = false;
/* Make sure that we are allowed to start a build. */ /* Make sure that we are allowed to start a build. */
if (!worker.canBuildMore()) { if (!worker.canBuildMore()) {
worker.waitForBuildSlot(shared_from_this()); worker.waitForBuildSlot(shared_from_this());
return; outputLocks.unlock();
}
/* Acquire locks and such. If we then see that the build has
been done by somebody else, we're done. */
usingBuildHook = false;
PrepareBuildReply preply = prepareBuild();
if (preply == prDone) {
amDone(ecSuccess);
return;
} else if (preply == prRestart) {
state = &DerivationGoal::haveDerivation;
worker.waitForAnyGoal(shared_from_this());
return; return;
} }
@ -1022,14 +1004,13 @@ void DerivationGoal::tryToBuild()
printMsg(lvlError, e.msg()); printMsg(lvlError, e.msg());
outputLocks.unlock(); outputLocks.unlock();
buildUser.release(); buildUser.release();
if (printBuildTrace) { if (printBuildTrace)
if (usingBuildHook) if (usingBuildHook)
printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%") printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%")
% drvPath % drv.outputs["out"].path % 0 % e.msg()); % drvPath % drv.outputs["out"].path % 0 % e.msg());
else else
printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%") printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
% drvPath % drv.outputs["out"].path % 0 % e.msg()); % drvPath % drv.outputs["out"].path % 0 % e.msg());
}
amDone(ecFailed); amDone(ecFailed);
return; return;
} }
@ -1069,8 +1050,7 @@ void DerivationGoal::buildDone()
malicious user from leaving behind a process that keeps files malicious user from leaving behind a process that keeps files
open and modifies them after they have been chown'ed to open and modifies them after they have been chown'ed to
root. */ root. */
if (buildUser.enabled()) if (buildUser.enabled()) buildUser.kill();
buildUser.kill();
try { try {
@ -1140,7 +1120,7 @@ void DerivationGoal::buildDone()
% drvPath % drv.outputs["out"].path % status % e.msg()); % drvPath % drv.outputs["out"].path % status % e.msg());
else else
printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%") printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
% drvPath % drv.outputs["out"].path % status % e.msg()); % drvPath % drv.outputs["out"].path % 1 % e.msg());
} }
amDone(ecFailed); amDone(ecFailed);
return; return;
@ -1296,16 +1276,6 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
else if (reply == "accept") { else if (reply == "accept") {
/* Acquire locks and such. If we then see that the output
paths are now valid, we're done. */
PrepareBuildReply preply = prepareBuild();
if (preply == prDone || preply == prRestart) {
/* Tell the hook to exit. */
writeLine(toHook.writeSide, "cancel");
terminateBuildHook();
return preply == prDone ? rpDone : rpRestart;
}
printMsg(lvlInfo, format("running hook to build path(s) %1%") printMsg(lvlInfo, format("running hook to build path(s) %1%")
% showPaths(outputPaths(drv.outputs))); % showPaths(outputPaths(drv.outputs)));
@ -1390,8 +1360,7 @@ DerivationGoal::PrepareBuildReply DerivationGoal::prepareBuild()
has locked the output since we checked in haveDerivation(). has locked the output since we checked in haveDerivation().
(It can't happen between here and the lockPaths() call below (It can't happen between here and the lockPaths() call below
because we're not allowing multi-threading.) */ because we're not allowing multi-threading.) */
for (DerivationOutputs::iterator i = drv.outputs.begin(); foreach (DerivationOutputs::iterator, i, drv.outputs)
i != drv.outputs.end(); ++i)
if (pathIsLockedByMe(i->second.path)) { if (pathIsLockedByMe(i->second.path)) {
debug(format("restarting derivation `%1%' because `%2%' is locked by another goal") debug(format("restarting derivation `%1%' because `%2%' is locked by another goal")
% drvPath % i->second.path); % drvPath % i->second.path);
@ -1425,66 +1394,23 @@ DerivationGoal::PrepareBuildReply DerivationGoal::prepareBuild()
if (validPaths.size() > 0) { if (validPaths.size() > 0) {
/* !!! fix this; try to delete valid paths */ /* !!! fix this; try to delete valid paths */
throw BuildError( throw Error(
format("derivation `%1%' is blocked by its output paths") format("derivation `%1%' is blocked by its output paths")
% drvPath); % drvPath);
} }
/* If any of the outputs already exist but are not registered, /* If any of the outputs already exist but are not registered,
delete them. */ delete them. */
for (DerivationOutputs::iterator i = drv.outputs.begin(); foreach (DerivationOutputs::iterator, i, drv.outputs) {
i != drv.outputs.end(); ++i)
{
Path path = i->second.path; Path path = i->second.path;
if (worker.store.isValidPath(path)) if (worker.store.isValidPath(path))
throw BuildError(format("obstructed build: path `%1%' exists") % path); throw Error(format("obstructed build: path `%1%' exists") % path);
if (pathExists(path)) { if (pathExists(path)) {
debug(format("removing unregistered path `%1%'") % path); debug(format("removing unregistered path `%1%'") % path);
deletePathWrapped(path); deletePathWrapped(path);
} }
} }
/* Gather information necessary for computing the closure and/or
running the build hook. */
/* The outputs are referenceable paths. */
for (DerivationOutputs::iterator i = drv.outputs.begin();
i != drv.outputs.end(); ++i)
{
debug(format("building path `%1%'") % i->second.path);
allPaths.insert(i->second.path);
}
/* Determine the full set of input paths. */
/* First, the input derivations. */
for (DerivationInputs::iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i)
{
/* Add the relevant output closures of the input derivation
`*i' as input paths. Only add the closures of output paths
that are specified as inputs. */
assert(worker.store.isValidPath(i->first));
Derivation inDrv = derivationFromPath(i->first);
for (StringSet::iterator j = i->second.begin();
j != i->second.end(); ++j)
if (inDrv.outputs.find(*j) != inDrv.outputs.end())
computeFSClosure(inDrv.outputs[*j].path, inputPaths);
else
throw BuildError(
format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'")
% drvPath % *j % i->first);
}
/* Second, the input sources. */
for (PathSet::iterator i = drv.inputSrcs.begin();
i != drv.inputSrcs.end(); ++i)
computeFSClosure(*i, inputPaths);
debug(format("added input paths %1%") % showPaths(inputPaths));
allPaths.insert(inputPaths.begin(), inputPaths.end());
return prProceed; return prProceed;
} }
@ -1508,7 +1434,7 @@ void DerivationGoal::startBuilder()
&& !(drv.platform == "i686-linux" && thisSystem == "x86_64-linux") && !(drv.platform == "i686-linux" && thisSystem == "x86_64-linux")
#endif #endif
) )
throw BuildError( throw Error(
format("a `%1%' is required to build `%3%', but I am a `%2%'") format("a `%1%' is required to build `%3%', but I am a `%2%'")
% drv.platform % thisSystem % drvPath); % drv.platform % thisSystem % drvPath);

4
tests/dependencies.sh
View file

@ -1,5 +1,7 @@
source common.sh source common.sh
clearStore
drvPath=$($nixinstantiate dependencies.nix) drvPath=$($nixinstantiate dependencies.nix)
echo "derivation is $drvPath" echo "derivation is $drvPath"
@ -13,7 +15,7 @@ if test -n "$dot"; then
$dot < $TEST_ROOT/graph $dot < $TEST_ROOT/graph
fi fi
outPath=$($nixstore -rvv "$drvPath") outPath=$($nixstore -rvv "$drvPath") || fail "build failed"
# Test Graphviz graph generation. # Test Graphviz graph generation.
$nixstore -q --graph "$outPath" > $TEST_ROOT/graph $nixstore -q --graph "$outPath" > $TEST_ROOT/graph

6
tests/gc-concurrent.sh
View file

@ -1,6 +1,6 @@
source common.sh source common.sh
$NIX_BIN_DIR/nix-collect-garbage -vvvvv $NIX_BIN_DIR/nix-collect-garbage
drvPath1=$($nixinstantiate gc-concurrent.nix -A test1) drvPath1=$($nixinstantiate gc-concurrent.nix -A test1)
outPath1=$($nixstore -q $drvPath1) outPath1=$($nixstore -q $drvPath1)
@ -28,7 +28,7 @@ pid2=$!
# Run the garbage collector while the build is running. # Run the garbage collector while the build is running.
sleep 4 sleep 4
$NIX_BIN_DIR/nix-collect-garbage -vvvvv $NIX_BIN_DIR/nix-collect-garbage
# Wait for build #1/#2 to finish. # Wait for build #1/#2 to finish.
echo waiting for pid $pid1 to finish... echo waiting for pid $pid1 to finish...
@ -53,6 +53,6 @@ rm -f "$NIX_STATE_DIR"/gcroots/foo*
! test -e $outPath3.lock ! test -e $outPath3.lock
# If we run the collector now, it should delete outPath1/2. # If we run the collector now, it should delete outPath1/2.
$NIX_BIN_DIR/nix-collect-garbage -vvvvv $NIX_BIN_DIR/nix-collect-garbage
! test -e $outPath1 ! test -e $outPath1
! test -e $outPath2 ! test -e $outPath2

2
tests/parallel.sh
View file

@ -2,7 +2,7 @@ source common.sh
clearStore clearStore
outPath=$($nixbuild -vv -j10000 parallel.nix) outPath=$($nixbuild -j10000 parallel.nix)
echo "output path is $outPath" echo "output path is $outPath"