* Added a switch `--fallback'. From the manual:

Whenever Nix attempts to realise a derivation for which a closure is
  already known, but this closure cannot be realised, fall back on
  normalising the derivation.

  The most common scenario in which this is useful is when we have
  registered substitutes in order to perform binary distribution from,
  say, a network repository.  If the repository is down, the
  realisation of the derivation will fail.  When this option is
  specified, Nix will build the derivation instead.  Thus, binary
  installation falls back on a source installation.  This option is
  not the default since it is generally not desirable for a transient
  failure in obtaining the substitutes to lead to a full build from
  source (with the related consumption of resources).
This commit is contained in:
Eelco Dolstra 2004-06-28 10:42:57 +00:00
parent b113edeab7
commit 91dc023665
17 changed files with 185 additions and 58 deletions

View file

@ -15,3 +15,4 @@
<arg><option>-k</option></arg>
<arg><option>--keep-failed</option></arg>
<arg><option>-K</option></arg>
<arg><option>--fallback</option></arg>

View file

@ -147,3 +147,28 @@
</listitem>
</varlistentry>
<varlistentry>
<term><option>--fallback</option></term>
<listitem>
<para>
Whenever Nix attempts to realise a derivation for which a
closure is already known, but this closure cannot be realised,
fall back on normalising the derivation.
</para>
<para>
The most common scenario in which this is useful is when we have
registered substitutes in order to perform binary distribution
from, say, a network repository. If the repository is down, the
realisation of the derivation will fail. When this option is
specified, Nix will build the derivation instead. Thus,
binary installation falls back on a source installation. This
option is not the default since it is generally not desirable
for a transient failure in obtaining the substitutes to lead to
a full build from source (with the related consumption of
resources).
</para>
</listitem>
</varlistentry>

View file

@ -139,6 +139,8 @@ static void initAndRun(int argc, char * * argv)
keepFailed = true;
else if (arg == "--keep-going" || arg == "-k")
keepGoing = true;
else if (arg == "--fallback")
tryFallback = true;
else if (arg == "--max-jobs" || arg == "-j") {
++i;
if (i == args.end()) throw UsageError("`--max-jobs' requires an argument");

View file

@ -366,9 +366,12 @@ void Database::setString(const Transaction & txn, TableId table,
void Database::setStrings(const Transaction & txn, TableId table,
const string & key, const Strings & data)
const string & key, const Strings & data, bool deleteEmpty)
{
setString(txn, table, key, packStrings(data));
if (deleteEmpty && data.size() == 0)
delPair(txn, table, key);
else
setString(txn, table, key, packStrings(data));
}

View file

@ -76,7 +76,8 @@ public:
const string & key, const string & data);
void setStrings(const Transaction & txn, TableId table,
const string & key, const Strings & data);
const string & key, const Strings & data,
bool deleteEmpty = true);
void delPair(const Transaction & txn, TableId table,
const string & key);

View file

@ -10,6 +10,8 @@ bool keepFailed = false;
bool keepGoing = false;
bool tryFallback = false;
Verbosity buildVerbosity = lvlDebug;
unsigned int maxBuildJobs = 1;

View file

@ -33,6 +33,10 @@ extern bool keepFailed;
of the same goal) fails. */
extern bool keepGoing;
/* Whether, if we cannot realise the known closure corresponding to a
derivation, we should try to normalise the derivation instead. */
extern bool tryFallback;
/* Verbosity level for build output. */
extern Verbosity buildVerbosity;

View file

@ -201,10 +201,27 @@ void Goal::waiteeDone(GoalPtr waitee, bool success)
{
assert(waitees.find(waitee) != waitees.end());
waitees.erase(waitee);
assert(nrWaitees > 0);
if (!success) ++nrFailed;
if (!--nrWaitees || (!success && !keepGoing))
assert(nrWaitees > 0);
if (!--nrWaitees || (!success && !keepGoing)) {
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (Goals::iterator i = waitees.begin(); i != waitees.end(); ++i) {
GoalPtr goal = *i;
WeakGoals waiters2;
for (WeakGoals::iterator j = goal->waiters.begin();
j != goal->waiters.end(); ++j)
if (j->lock() != shared_from_this())
waiters2.insert(*j);
goal->waiters = waiters2;
}
waitees.clear();
worker.wakeUp(shared_from_this());
}
}
@ -271,6 +288,17 @@ const char * * strings2CharPtrs(const Strings & ss)
}
/* Should only be called after an expression has been normalised. */
Path queryNormalForm(const Path & nePath)
{
StoreExpr ne = storeExprFromPath(nePath);
if (ne.type == StoreExpr::neClosure) return nePath;
Path nfPath;
if (!querySuccessor(nePath, nfPath)) abort();
return nfPath;
}
//////////////////////////////////////////////////////////////////////
@ -472,13 +500,7 @@ void NormalisationGoal::inputNormalised()
/* Inputs must also be realised before we can build this goal. */
for (PathSet::iterator i = expr.derivation.inputs.begin();
i != expr.derivation.inputs.end(); ++i)
{
Path neInput = *i, nfInput;
if (querySuccessor(neInput, nfInput))
neInput = nfInput;
/* Otherwise the input must be a closure. */
addWaitee(worker.makeRealisationGoal(neInput));
}
addWaitee(worker.makeRealisationGoal(queryNormalForm(*i)));
resetWaitees(expr.derivation.inputs.size());
@ -829,8 +851,8 @@ bool NormalisationGoal::prepareBuild()
i != expr.derivation.inputs.end(); ++i)
{
checkInterrupt();
Path nePath = *i, nfPath;
if (!querySuccessor(nePath, nfPath)) nfPath = nePath;
Path nePath = *i;
Path nfPath = queryNormalForm(nePath);
inputNFs.insert(nfPath);
if (nfPath != nePath) inputSucs[nePath] = nfPath;
/* !!! nfPath should be a root of the garbage collector while
@ -1174,9 +1196,15 @@ string NormalisationGoal::name()
class RealisationGoal : public Goal
{
private:
/* The path of the closure store expression. */
/* The path of the store expression. */
Path nePath;
/* The normal form. */
Path nfPath;
/* Whether we should try to delete a broken successor mapping. */
bool tryFallback;
/* The store expression stored at nePath. */
StoreExpr expr;
@ -1191,9 +1219,12 @@ public:
/* The states. */
void init();
void isNormalised();
void haveStoreExpr();
void elemFinished();
void fallBack(const format & error);
string name();
};
@ -1202,6 +1233,7 @@ RealisationGoal::RealisationGoal(const Path & _nePath, Worker & _worker)
: Goal(_worker)
{
nePath = _nePath;
tryFallback = ::tryFallback;
state = &RealisationGoal::init;
}
@ -1221,11 +1253,36 @@ void RealisationGoal::init()
{
trace("init");
/* The first thing to do is to make sure that the store expression
exists. If it doesn't, it may be created through a
substitute. */
if (querySuccessor(nePath, nfPath)) {
isNormalised();
return;
}
/* First normalise the expression (which is a no-op if the
expression is already a closure). */
resetWaitees(1);
addWaitee(worker.makeSubstitutionGoal(nePath));
addWaitee(worker.makeNormalisationGoal(nePath));
/* Since there is no successor right now, the normalisation goal
will perform an actual build. So there is no sense in trying a
fallback if the realisation of the closure fails (it can't
really fail). */
tryFallback = false;
state = &RealisationGoal::isNormalised;
}
void RealisationGoal::isNormalised()
{
trace("has been normalised");
nfPath = queryNormalForm(nePath);
/* Now make sure that the store expression exists. If it doesn't,
it may be created through a substitute. */
resetWaitees(1);
addWaitee(worker.makeSubstitutionGoal(nfPath));
state = &RealisationGoal::haveStoreExpr;
}
@ -1236,21 +1293,18 @@ void RealisationGoal::haveStoreExpr()
trace("loading store expression");
if (nrFailed != 0) {
printMsg(lvlError,
format("cannot realise missing store expression `%1%'")
% nePath);
amDone(false);
fallBack(format("cannot realise closure `%1%' since that file is missing") % nfPath);
return;
}
assert(isValidPath(nePath));
assert(isValidPath(nfPath));
/* Get the store expression. */
expr = storeExprFromPath(nePath);
expr = storeExprFromPath(nfPath);
/* If this is a normal form (i.e., a closure) we are also done. */
if (expr.type != StoreExpr::neClosure)
throw Error(format("expected closure in `%1%'") % nePath);
throw Error(format("expected closure in `%1%'") % nfPath);
/* Each path in the closure should exist, or should be creatable
through a substitute. */
@ -1269,12 +1323,11 @@ void RealisationGoal::elemFinished()
trace("all closure elements present");
if (nrFailed != 0) {
printMsg(lvlError,
fallBack(
format("cannot realise closure `%1%': "
"%2% closure element(s) are not present "
"and could not be substituted")
% nePath % nrFailed);
amDone(false);
% nfPath % nrFailed);
return;
}
@ -1282,6 +1335,21 @@ void RealisationGoal::elemFinished()
}
void RealisationGoal::fallBack(const format & error)
{
if (tryFallback && nePath != nfPath) {
printMsg(lvlError, format("%1%; trying to normalise derivation instead")
% error);
tryFallback = false;
unregisterSuccessor(nePath);
init();
} else {
printMsg(lvlError, format("%1%; maybe `--fallback' will help") % error);
amDone(false);
}
}
string RealisationGoal::name()
{
return (format("realisation of `%1%'") % nePath).str();
@ -1409,8 +1477,7 @@ void SubstitutionGoal::exprNormalised()
}
/* Realise the substitute store expression. */
if (!querySuccessor(sub.storeExpr, nfSub))
nfSub = sub.storeExpr;
nfSub = queryNormalForm(sub.storeExpr);
addWaitee(worker.makeRealisationGoal(nfSub));
resetWaitees(1);
@ -1869,19 +1936,19 @@ Path normaliseStoreExpr(const Path & nePath)
if (!worker.run(worker.makeNormalisationGoal(nePath)))
throw Error(format("normalisation of store expression `%1%' failed") % nePath);
Path nfPath;
if (!querySuccessor(nePath, nfPath)) abort();
return nfPath;
return queryNormalForm(nePath);
}
void realiseClosure(const Path & nePath)
Path realiseStoreExpr(const Path & nePath)
{
startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath);
startNest(nest, lvlDebug, format("realising `%1%'") % nePath);
Worker worker;
if (!worker.run(worker.makeRealisationGoal(nePath)))
throw Error(format("realisation of closure `%1%' failed") % nePath);
throw Error(format("realisation of store expressions `%1%' failed") % nePath);
return queryNormalForm(nePath);
}

View file

@ -10,13 +10,13 @@
successor is known. */
Path normaliseStoreExpr(const Path & nePath);
/* Realise a closure store expression in the file system.
The pending paths are those that are already being realised. This
prevents infinite recursion for paths realised through a substitute
(since when we build the substitute, we would first try to realise
its output paths through substitutes... kaboom!). */
void realiseClosure(const Path & nePath);
/* Realise a store expression. If the expression is a derivation, it
is first normalised into a closure. The closure is then realised
in the file system (i.e., it is ensured that each path in the
closure exists in the file system, if necessary by using the
substitute mechanism). Returns the normal form of the expression
(i.e., its closure expression). */
Path realiseStoreExpr(const Path & nePath);
/* Ensure that a path exists, possibly by instantiating it by
realising a substitute. */

View file

@ -237,6 +237,30 @@ void registerSuccessor(const Transaction & txn,
}
void unregisterSuccessor(const Path & srcPath)
{
assertStorePath(srcPath);
Transaction txn(nixDB);
Path sucPath;
if (!nixDB.queryString(txn, dbSuccessors, srcPath, sucPath)) {
txn.abort();
return;
}
nixDB.delPair(txn, dbSuccessors, srcPath);
Paths revs;
nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs);
Paths::iterator i = find(revs.begin(), revs.end(), srcPath);
assert(i != revs.end());
revs.erase(i);
nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs);
txn.commit();
}
bool querySuccessor(const Path & srcPath, Path & sucPath)
{
return nixDB.queryString(noTxn, dbSuccessors, srcPath, sucPath);
@ -294,10 +318,7 @@ static void writeSubstitutes(const Transaction & txn,
ss.push_back(packStrings(ss2));
}
if (ss.size() == 0)
nixDB.delPair(txn, dbSubstitutes, srcPath);
else
nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
}

View file

@ -54,6 +54,9 @@ void copyPath(const Path & src, const Path & dst);
void registerSuccessor(const Transaction & txn,
const Path & srcPath, const Path & sucPath);
/* Remove a successor mapping. */
void unregisterSuccessor(const Path & srcPath);
/* Return the predecessors of the Nix expression stored at the given
path. */
bool querySuccessor(const Path & srcPath, Path & sucPath);

View file

@ -213,8 +213,7 @@ void createUserEnv(EvalState & state, const DrvInfos & drvs,
/* Realise the resulting store expression. */
debug(format("realising user environment"));
Path nfPath = normaliseStoreExpr(topLevelDrv.drvPath);
realiseClosure(nfPath);
Path nfPath = realiseStoreExpr(topLevelDrv.drvPath);
/* Switch the current user environment to the output path. */
debug(format("switching to new user environment"));

View file

@ -26,8 +26,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); i++)
{
Path nfPath = normaliseStoreExpr(*i);
realiseClosure(nfPath);
Path nfPath = realiseStoreExpr(*i);
cout << format("%1%\n") % (string) nfPath;
}
}
@ -58,8 +57,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
Path maybeNormalise(const Path & ne, bool normalise, bool realise)
{
if (realise) {
Path ne2 = normaliseStoreExpr(ne);
realiseClosure(ne2);
Path ne2 = realiseStoreExpr(ne);
return normalise ? ne2 : ne;
} else
return normalise ? normaliseStoreExpr(ne) : ne;

View file

@ -19,10 +19,10 @@ parallel.sh: parallel.nix
build-hook.sh: build-hook.nix
substitutes.sh: substitutes.nix substituter.nix
substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix
fall-back.sh: fall-back.nix
fallback.sh: fallback.nix
TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \
build-hook.sh substitutes.sh substitutes2.sh
build-hook.sh substitutes.sh substitutes2.sh fallback.sh verify.sh
XFAIL_TESTS =
@ -36,4 +36,4 @@ EXTRA_DIST = $(TESTS) \
build-hook.nix.in build-hook.hook.sh \
substitutes.nix.in substituter.nix.in substituter.builder.sh \
substitutes2.nix.in substituter2.nix.in substituter2.builder.sh \
fall-back.nix.in
fallback.nix.in

View file

@ -7,7 +7,7 @@ suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store
(echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute
$TOP/src/nix-store/nix-store --successor $storeExpr $suc
outPath=$($TOP/src/nix-store/nix-store -qnf "$storeExpr")
outPath=$($TOP/src/nix-store/nix-store -qnf --fallback "$storeExpr")
echo "output path is $outPath"

1
tests/verify.sh Normal file
View file

@ -0,0 +1 @@
$TOP/src/nix-store/nix-store --verify