diff --git a/doc/manual/opt-common-syn.xml b/doc/manual/opt-common-syn.xml
index f90041863..f848ad249 100644
--- a/doc/manual/opt-common-syn.xml
+++ b/doc/manual/opt-common-syn.xml
@@ -15,3 +15,4 @@
+
diff --git a/doc/manual/opt-common.xml b/doc/manual/opt-common.xml
index 158e282fd..db8a60eca 100644
--- a/doc/manual/opt-common.xml
+++ b/doc/manual/opt-common.xml
@@ -147,3 +147,28 @@
+
+
+
+
+
+ Whenever Nix attempts to realise a derivation for which a
+ closure is already known, but this closure cannot be realised,
+ fall back on normalising the derivation.
+
+
+
+ The most common scenario in which this is useful is when we have
+ registered substitutes in order to perform binary distribution
+ from, say, a network repository. If the repository is down, the
+ realisation of the derivation will fail. When this option is
+ specified, Nix will build the derivation instead. Thus,
+ binary installation falls back on a source installation. This
+ option is not the default since it is generally not desirable
+ for a transient failure in obtaining the substitutes to lead to
+ a full build from source (with the related consumption of
+ resources).
+
+
+
+
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 13ad4fede..6aad03a37 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -139,6 +139,8 @@ static void initAndRun(int argc, char * * argv)
keepFailed = true;
else if (arg == "--keep-going" || arg == "-k")
keepGoing = true;
+ else if (arg == "--fallback")
+ tryFallback = true;
else if (arg == "--max-jobs" || arg == "-j") {
++i;
if (i == args.end()) throw UsageError("`--max-jobs' requires an argument");
diff --git a/src/libstore/db.cc b/src/libstore/db.cc
index e792a371b..5c8e7edec 100644
--- a/src/libstore/db.cc
+++ b/src/libstore/db.cc
@@ -366,9 +366,12 @@ void Database::setString(const Transaction & txn, TableId table,
void Database::setStrings(const Transaction & txn, TableId table,
- const string & key, const Strings & data)
+ const string & key, const Strings & data, bool deleteEmpty)
{
- setString(txn, table, key, packStrings(data));
+ if (deleteEmpty && data.size() == 0)
+ delPair(txn, table, key);
+ else
+ setString(txn, table, key, packStrings(data));
}
diff --git a/src/libstore/db.hh b/src/libstore/db.hh
index 1c681b9b5..bbeabfc7d 100644
--- a/src/libstore/db.hh
+++ b/src/libstore/db.hh
@@ -76,7 +76,8 @@ public:
const string & key, const string & data);
void setStrings(const Transaction & txn, TableId table,
- const string & key, const Strings & data);
+ const string & key, const Strings & data,
+ bool deleteEmpty = true);
void delPair(const Transaction & txn, TableId table,
const string & key);
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index e7b32244b..aad26501b 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -10,6 +10,8 @@ bool keepFailed = false;
bool keepGoing = false;
+bool tryFallback = false;
+
Verbosity buildVerbosity = lvlDebug;
unsigned int maxBuildJobs = 1;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index cef4f704e..7f88d5c53 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -33,6 +33,10 @@ extern bool keepFailed;
of the same goal) fails. */
extern bool keepGoing;
+/* Whether, if we cannot realise the known closure corresponding to a
+ derivation, we should try to normalise the derivation instead. */
+extern bool tryFallback;
+
/* Verbosity level for build output. */
extern Verbosity buildVerbosity;
diff --git a/src/libstore/normalise.cc b/src/libstore/normalise.cc
index a38bee60f..6fc3bdfc3 100644
--- a/src/libstore/normalise.cc
+++ b/src/libstore/normalise.cc
@@ -201,10 +201,27 @@ void Goal::waiteeDone(GoalPtr waitee, bool success)
{
assert(waitees.find(waitee) != waitees.end());
waitees.erase(waitee);
- assert(nrWaitees > 0);
+
if (!success) ++nrFailed;
- if (!--nrWaitees || (!success && !keepGoing))
+
+ assert(nrWaitees > 0);
+ if (!--nrWaitees || (!success && !keepGoing)) {
+
+ /* If we failed and keepGoing is not set, we remove all
+ remaining waitees. */
+ for (Goals::iterator i = waitees.begin(); i != waitees.end(); ++i) {
+ GoalPtr goal = *i;
+ WeakGoals waiters2;
+ for (WeakGoals::iterator j = goal->waiters.begin();
+ j != goal->waiters.end(); ++j)
+ if (j->lock() != shared_from_this())
+ waiters2.insert(*j);
+ goal->waiters = waiters2;
+ }
+ waitees.clear();
+
worker.wakeUp(shared_from_this());
+ }
}
@@ -271,6 +288,17 @@ const char * * strings2CharPtrs(const Strings & ss)
}
+/* Should only be called after an expression has been normalised. */
+Path queryNormalForm(const Path & nePath)
+{
+ StoreExpr ne = storeExprFromPath(nePath);
+ if (ne.type == StoreExpr::neClosure) return nePath;
+ Path nfPath;
+ if (!querySuccessor(nePath, nfPath)) abort();
+ return nfPath;
+}
+
+
//////////////////////////////////////////////////////////////////////
@@ -472,13 +500,7 @@ void NormalisationGoal::inputNormalised()
/* Inputs must also be realised before we can build this goal. */
for (PathSet::iterator i = expr.derivation.inputs.begin();
i != expr.derivation.inputs.end(); ++i)
- {
- Path neInput = *i, nfInput;
- if (querySuccessor(neInput, nfInput))
- neInput = nfInput;
- /* Otherwise the input must be a closure. */
- addWaitee(worker.makeRealisationGoal(neInput));
- }
+ addWaitee(worker.makeRealisationGoal(queryNormalForm(*i)));
resetWaitees(expr.derivation.inputs.size());
@@ -829,8 +851,8 @@ bool NormalisationGoal::prepareBuild()
i != expr.derivation.inputs.end(); ++i)
{
checkInterrupt();
- Path nePath = *i, nfPath;
- if (!querySuccessor(nePath, nfPath)) nfPath = nePath;
+ Path nePath = *i;
+ Path nfPath = queryNormalForm(nePath);
inputNFs.insert(nfPath);
if (nfPath != nePath) inputSucs[nePath] = nfPath;
/* !!! nfPath should be a root of the garbage collector while
@@ -1174,9 +1196,15 @@ string NormalisationGoal::name()
class RealisationGoal : public Goal
{
private:
- /* The path of the closure store expression. */
+ /* The path of the store expression. */
Path nePath;
+ /* The normal form. */
+ Path nfPath;
+
+ /* Whether we should try to delete a broken successor mapping. */
+ bool tryFallback;
+
/* The store expression stored at nePath. */
StoreExpr expr;
@@ -1191,9 +1219,12 @@ public:
/* The states. */
void init();
+ void isNormalised();
void haveStoreExpr();
void elemFinished();
+ void fallBack(const format & error);
+
string name();
};
@@ -1202,6 +1233,7 @@ RealisationGoal::RealisationGoal(const Path & _nePath, Worker & _worker)
: Goal(_worker)
{
nePath = _nePath;
+ tryFallback = ::tryFallback;
state = &RealisationGoal::init;
}
@@ -1221,11 +1253,36 @@ void RealisationGoal::init()
{
trace("init");
- /* The first thing to do is to make sure that the store expression
- exists. If it doesn't, it may be created through a
- substitute. */
+ if (querySuccessor(nePath, nfPath)) {
+ isNormalised();
+ return;
+ }
+
+ /* First normalise the expression (which is a no-op if the
+ expression is already a closure). */
resetWaitees(1);
- addWaitee(worker.makeSubstitutionGoal(nePath));
+ addWaitee(worker.makeNormalisationGoal(nePath));
+
+ /* Since there is no successor right now, the normalisation goal
+ will perform an actual build. So there is no sense in trying a
+ fallback if the realisation of the closure fails (it can't
+ really fail). */
+ tryFallback = false;
+
+ state = &RealisationGoal::isNormalised;
+}
+
+
+void RealisationGoal::isNormalised()
+{
+ trace("has been normalised");
+
+ nfPath = queryNormalForm(nePath);
+
+ /* Now make sure that the store expression exists. If it doesn't,
+ it may be created through a substitute. */
+ resetWaitees(1);
+ addWaitee(worker.makeSubstitutionGoal(nfPath));
state = &RealisationGoal::haveStoreExpr;
}
@@ -1236,21 +1293,18 @@ void RealisationGoal::haveStoreExpr()
trace("loading store expression");
if (nrFailed != 0) {
- printMsg(lvlError,
- format("cannot realise missing store expression `%1%'")
- % nePath);
- amDone(false);
+ fallBack(format("cannot realise closure `%1%' since that file is missing") % nfPath);
return;
}
- assert(isValidPath(nePath));
+ assert(isValidPath(nfPath));
/* Get the store expression. */
- expr = storeExprFromPath(nePath);
+ expr = storeExprFromPath(nfPath);
/* If this is a normal form (i.e., a closure) we are also done. */
if (expr.type != StoreExpr::neClosure)
- throw Error(format("expected closure in `%1%'") % nePath);
+ throw Error(format("expected closure in `%1%'") % nfPath);
/* Each path in the closure should exist, or should be creatable
through a substitute. */
@@ -1269,12 +1323,11 @@ void RealisationGoal::elemFinished()
trace("all closure elements present");
if (nrFailed != 0) {
- printMsg(lvlError,
+ fallBack(
format("cannot realise closure `%1%': "
"%2% closure element(s) are not present "
"and could not be substituted")
- % nePath % nrFailed);
- amDone(false);
+ % nfPath % nrFailed);
return;
}
@@ -1282,6 +1335,21 @@ void RealisationGoal::elemFinished()
}
+void RealisationGoal::fallBack(const format & error)
+{
+ if (tryFallback && nePath != nfPath) {
+ printMsg(lvlError, format("%1%; trying to normalise derivation instead")
+ % error);
+ tryFallback = false;
+ unregisterSuccessor(nePath);
+ init();
+ } else {
+ printMsg(lvlError, format("%1%; maybe `--fallback' will help") % error);
+ amDone(false);
+ }
+}
+
+
string RealisationGoal::name()
{
return (format("realisation of `%1%'") % nePath).str();
@@ -1409,8 +1477,7 @@ void SubstitutionGoal::exprNormalised()
}
/* Realise the substitute store expression. */
- if (!querySuccessor(sub.storeExpr, nfSub))
- nfSub = sub.storeExpr;
+ nfSub = queryNormalForm(sub.storeExpr);
addWaitee(worker.makeRealisationGoal(nfSub));
resetWaitees(1);
@@ -1869,19 +1936,19 @@ Path normaliseStoreExpr(const Path & nePath)
if (!worker.run(worker.makeNormalisationGoal(nePath)))
throw Error(format("normalisation of store expression `%1%' failed") % nePath);
- Path nfPath;
- if (!querySuccessor(nePath, nfPath)) abort();
- return nfPath;
+ return queryNormalForm(nePath);
}
-void realiseClosure(const Path & nePath)
+Path realiseStoreExpr(const Path & nePath)
{
- startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath);
+ startNest(nest, lvlDebug, format("realising `%1%'") % nePath);
Worker worker;
if (!worker.run(worker.makeRealisationGoal(nePath)))
- throw Error(format("realisation of closure `%1%' failed") % nePath);
+ throw Error(format("realisation of store expressions `%1%' failed") % nePath);
+
+ return queryNormalForm(nePath);
}
diff --git a/src/libstore/normalise.hh b/src/libstore/normalise.hh
index bbde545c4..43be136e5 100644
--- a/src/libstore/normalise.hh
+++ b/src/libstore/normalise.hh
@@ -10,13 +10,13 @@
successor is known. */
Path normaliseStoreExpr(const Path & nePath);
-/* Realise a closure store expression in the file system.
-
- The pending paths are those that are already being realised. This
- prevents infinite recursion for paths realised through a substitute
- (since when we build the substitute, we would first try to realise
- its output paths through substitutes... kaboom!). */
-void realiseClosure(const Path & nePath);
+/* Realise a store expression. If the expression is a derivation, it
+ is first normalised into a closure. The closure is then realised
+ in the file system (i.e., it is ensured that each path in the
+ closure exists in the file system, if necessary by using the
+ substitute mechanism). Returns the normal form of the expression
+ (i.e., its closure expression). */
+Path realiseStoreExpr(const Path & nePath);
/* Ensure that a path exists, possibly by instantiating it by
realising a substitute. */
diff --git a/src/libstore/store.cc b/src/libstore/store.cc
index 9677f8422..44b3a29e3 100644
--- a/src/libstore/store.cc
+++ b/src/libstore/store.cc
@@ -237,6 +237,30 @@ void registerSuccessor(const Transaction & txn,
}
+void unregisterSuccessor(const Path & srcPath)
+{
+ assertStorePath(srcPath);
+
+ Transaction txn(nixDB);
+
+ Path sucPath;
+ if (!nixDB.queryString(txn, dbSuccessors, srcPath, sucPath)) {
+ txn.abort();
+ return;
+ }
+ nixDB.delPair(txn, dbSuccessors, srcPath);
+
+ Paths revs;
+ nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs);
+ Paths::iterator i = find(revs.begin(), revs.end(), srcPath);
+ assert(i != revs.end());
+ revs.erase(i);
+ nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs);
+
+ txn.commit();
+}
+
+
bool querySuccessor(const Path & srcPath, Path & sucPath)
{
return nixDB.queryString(noTxn, dbSuccessors, srcPath, sucPath);
@@ -294,10 +318,7 @@ static void writeSubstitutes(const Transaction & txn,
ss.push_back(packStrings(ss2));
}
- if (ss.size() == 0)
- nixDB.delPair(txn, dbSubstitutes, srcPath);
- else
- nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
+ nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
}
diff --git a/src/libstore/store.hh b/src/libstore/store.hh
index 40d1859e5..68f7d6190 100644
--- a/src/libstore/store.hh
+++ b/src/libstore/store.hh
@@ -54,6 +54,9 @@ void copyPath(const Path & src, const Path & dst);
void registerSuccessor(const Transaction & txn,
const Path & srcPath, const Path & sucPath);
+/* Remove a successor mapping. */
+void unregisterSuccessor(const Path & srcPath);
+
/* Return the predecessors of the Nix expression stored at the given
path. */
bool querySuccessor(const Path & srcPath, Path & sucPath);
diff --git a/src/nix-env/main.cc b/src/nix-env/main.cc
index 50ff0b19e..c57f03cce 100644
--- a/src/nix-env/main.cc
+++ b/src/nix-env/main.cc
@@ -213,8 +213,7 @@ void createUserEnv(EvalState & state, const DrvInfos & drvs,
/* Realise the resulting store expression. */
debug(format("realising user environment"));
- Path nfPath = normaliseStoreExpr(topLevelDrv.drvPath);
- realiseClosure(nfPath);
+ Path nfPath = realiseStoreExpr(topLevelDrv.drvPath);
/* Switch the current user environment to the output path. */
debug(format("switching to new user environment"));
diff --git a/src/nix-store/main.cc b/src/nix-store/main.cc
index 2f38edf77..e83f9133f 100644
--- a/src/nix-store/main.cc
+++ b/src/nix-store/main.cc
@@ -26,8 +26,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); i++)
{
- Path nfPath = normaliseStoreExpr(*i);
- realiseClosure(nfPath);
+ Path nfPath = realiseStoreExpr(*i);
cout << format("%1%\n") % (string) nfPath;
}
}
@@ -58,8 +57,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
Path maybeNormalise(const Path & ne, bool normalise, bool realise)
{
if (realise) {
- Path ne2 = normaliseStoreExpr(ne);
- realiseClosure(ne2);
+ Path ne2 = realiseStoreExpr(ne);
return normalise ? ne2 : ne;
} else
return normalise ? normaliseStoreExpr(ne) : ne;
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 19122cd95..9e1d6a198 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -19,10 +19,10 @@ parallel.sh: parallel.nix
build-hook.sh: build-hook.nix
substitutes.sh: substitutes.nix substituter.nix
substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix
-fall-back.sh: fall-back.nix
+fallback.sh: fallback.nix
TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \
- build-hook.sh substitutes.sh substitutes2.sh
+ build-hook.sh substitutes.sh substitutes2.sh fallback.sh verify.sh
XFAIL_TESTS =
@@ -36,4 +36,4 @@ EXTRA_DIST = $(TESTS) \
build-hook.nix.in build-hook.hook.sh \
substitutes.nix.in substituter.nix.in substituter.builder.sh \
substitutes2.nix.in substituter2.nix.in substituter2.builder.sh \
- fall-back.nix.in
+ fallback.nix.in
diff --git a/tests/fall-back.nix.in b/tests/fallback.nix.in
similarity index 100%
rename from tests/fall-back.nix.in
rename to tests/fallback.nix.in
diff --git a/tests/fall-back.sh b/tests/fallback.sh
similarity index 88%
rename from tests/fall-back.sh
rename to tests/fallback.sh
index e4a394217..5799775eb 100644
--- a/tests/fall-back.sh
+++ b/tests/fallback.sh
@@ -7,7 +7,7 @@ suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store
(echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute
$TOP/src/nix-store/nix-store --successor $storeExpr $suc
-outPath=$($TOP/src/nix-store/nix-store -qnf "$storeExpr")
+outPath=$($TOP/src/nix-store/nix-store -qnf --fallback "$storeExpr")
echo "output path is $outPath"
diff --git a/tests/verify.sh b/tests/verify.sh
new file mode 100644
index 000000000..ede3e7d74
--- /dev/null
+++ b/tests/verify.sh
@@ -0,0 +1 @@
+$TOP/src/nix-store/nix-store --verify