From 66c51dc21558c6ac5149c5158df7e5b580029e84 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 29 Jan 2008 18:17:36 +0000 Subject: [PATCH] * nix-store --dump-db / --load-db to dump/load the Nix DB. * nix-store --register-validity: option to supply the content hash of each path. * Removed compatibility with Nix <= 0.7 stores. --- doc/manual/release-notes.xml | 13 +++++ src/libstore/build.cc | 34 +---------- src/libstore/local-store.cc | 106 +++++------------------------------ src/libstore/local-store.hh | 2 + src/libstore/remote-store.cc | 6 ++ src/libstore/remote-store.hh | 2 + src/libstore/store-api.cc | 38 ++++++++++++- src/libstore/store-api.hh | 9 ++- src/nix-store/nix-store.cc | 57 +++++++++++++++---- 9 files changed, 130 insertions(+), 137 deletions(-) diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index 656a28fca..906a772c8 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -6,6 +6,19 @@ + + +
Release 0.12 (TBA) + + + + nix-store --dump-db / --load-db. + + + +
+ +
Release 0.11 (December 31, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index af54b161e..a4f9c469c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1263,34 +1263,6 @@ string showPaths(const PathSet & paths) } -/* Return a string accepted by `nix-store --register-validity' that - registers the specified paths as valid. Note: it's the - responsibility of the caller to provide a closure. */ -static string makeValidityRegistration(const PathSet & paths, - bool showDerivers) -{ - string s = ""; - - for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) { - s += *i + "\n"; - - Path deriver = showDerivers ? store->queryDeriver(*i) : ""; - s += deriver + "\n"; - - PathSet references; - store->queryReferences(*i, references); - - s += (format("%1%\n") % references.size()).str(); - - for (PathSet::iterator j = references.begin(); - j != references.end(); ++j) - s += *j + "\n"; - } - - return s; -} - - DerivationGoal::HookReply DerivationGoal::tryBuildHook() { if (!useBuildHook) return rpDecline; @@ -1417,7 +1389,7 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook() /* The `references' file has exactly the format accepted by `nix-store --register-validity'. */ writeStringToFile(referencesFN, - makeValidityRegistration(allInputs, true)); + makeValidityRegistration(allInputs, true, false)); /* Tell the hook to proceed. */ writeLine(toHook.writeSide, "okay"); @@ -1662,7 +1634,7 @@ void DerivationGoal::startBuilder() /* !!! in secure Nix, the writing should be done on the build uid for security (maybe). */ writeStringToFile(tmpDir + "/" + fileName, - makeValidityRegistration(refs, false)); + makeValidityRegistration(refs, false, false)); } // The same for derivations @@ -1701,7 +1673,7 @@ void DerivationGoal::startBuilder() /* !!! in secure Nix, the writing should be done on the build uid for security (maybe). */ writeStringToFile(tmpDir + "/" + fileName, - makeValidityRegistration(refs, false)); + makeValidityRegistration(refs, false, false)); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7b3b7355e..12a73b70c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -56,7 +56,6 @@ static TableId dbReferrers = 0; static TableId dbDerivers = 0; -static void upgradeStore07(); static void upgradeStore09(); static void upgradeStore11(); @@ -128,12 +127,12 @@ LocalStore::LocalStore(bool reserveSpace) % curSchema % nixSchemaVersion); if (curSchema < nixSchemaVersion) { + if (curSchema == 0) /* new store */ + curSchema = nixSchemaVersion; if (curSchema <= 1) - upgradeStore07(); - if (curSchema == 2) - upgradeStore09(); - if (curSchema == 3) - upgradeStore11(); + throw Error("your Nix store is no longer supported"); + if (curSchema <= 2) upgradeStore09(); + if (curSchema <= 3) upgradeStore11(); writeFile(schemaFN, (format("%1%") % nixSchemaVersion).str()); } } @@ -261,6 +260,14 @@ bool LocalStore::isValidPath(const Path & path) } +PathSet LocalStore::queryValidPaths() +{ + Paths paths; + nixDB.enumTable(noTxn, dbValidPaths, paths); + return PathSet(paths.begin(), paths.end()); +} + + static string addPrefix(const string & prefix, const string & s) { return prefix + string(1, (char) 0) + s; @@ -1069,93 +1076,6 @@ void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) } -/* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */ -static void upgradeStore07() -{ - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - - Transaction txn(nixDB); - - Paths validPaths2; - nixDB.enumTable(txn, dbValidPaths, validPaths2); - PathSet validPaths(validPaths2.begin(), validPaths2.end()); - - std::cerr << "hashing paths..."; - int n = 0; - for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) { - checkInterrupt(); - string s; - nixDB.queryString(txn, dbValidPaths, *i, s); - if (s == "") { - Hash hash = hashPath(htSHA256, *i); - setHash(txn, *i, hash); - std::cerr << "."; - if (++n % 1000 == 0) { - txn.commit(); - txn.begin(nixDB); - } - } - } - std::cerr << std::endl; - - txn.commit(); - - txn.begin(nixDB); - - std::cerr << "processing closures..."; - for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) { - checkInterrupt(); - if (i->size() > 6 && string(*i, i->size() - 6) == ".store") { - ATerm t = ATreadFromNamedFile(i->c_str()); - if (!t) throw Error(format("cannot read aterm from `%1%'") % *i); - - ATermList roots, elems; - if (!matchOldClosure(t, roots, elems)) continue; - - for (ATermIterator j(elems); j; ++j) { - - ATerm path2; - ATermList references2; - if (!matchOldClosureElem(*j, path2, references2)) continue; - - Path path = aterm2String(path2); - if (validPaths.find(path) == validPaths.end()) - /* Skip this path; it's invalid. This is a normal - condition (Nix <= 0.7 did not enforce closure - on closure store expressions). */ - continue; - - PathSet references; - for (ATermIterator k(references2); k; ++k) { - Path reference = aterm2String(*k); - if (validPaths.find(reference) == validPaths.end()) - /* Bad reference. Set it anyway and let the - user fix it. */ - printMsg(lvlError, format("closure `%1%' contains reference from `%2%' " - "to invalid path `%3%' (run `nix-store --verify')") - % *i % path % reference); - references.insert(reference); - } - - PathSet prevReferences; - queryReferences(txn, path, prevReferences); - if (prevReferences.size() > 0 && references != prevReferences) - printMsg(lvlError, format("warning: conflicting references for `%1%'") % path); - - if (references != prevReferences) - setReferences(txn, path, references); - } - - std::cerr << "."; - } - } - std::cerr << std::endl; - - /* !!! maybe this transaction is way too big */ - txn.commit(); -} - - /* Upgrade from schema 2 (0.8 <= Nix <= 0.9) to schema 3 (Nix >= 0.10). The only thing to do here is to upgrade the old `referer' table (which causes quadratic complexity in some cases) to the new diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6c366167f..9e6130013 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -59,6 +59,8 @@ public: bool isValidPath(const Path & path); + PathSet queryValidPaths(); + Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cc847c050..88a2fca8a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -185,6 +185,12 @@ bool RemoteStore::isValidPath(const Path & path) } +PathSet RemoteStore::queryValidPaths() +{ + throw Error("not implemented"); +} + + bool RemoteStore::hasSubstitutes(const Path & path) { writeInt(wopHasSubstitutes, to); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 01cac93a4..1501591e2 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -27,6 +27,8 @@ public: bool isValidPath(const Path & path); + PathSet queryValidPaths(); + Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 22a66ccab..b5bc85e18 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -151,11 +151,47 @@ Path computeStorePathForText(const string & suffix, const string & s, } -ValidPathInfo decodeValidPathInfo(std::istream & str) +/* Return a string accepted by decodeValidPathInfo() that + registers the specified paths as valid. Note: it's the + responsibility of the caller to provide a closure. */ +string makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash) +{ + string s = ""; + + for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) { + s += *i + "\n"; + + if (showHash) + s += printHash(store->queryPathHash(*i)) + "\n"; + + Path deriver = showDerivers ? store->queryDeriver(*i) : ""; + s += deriver + "\n"; + + PathSet references; + store->queryReferences(*i, references); + + s += (format("%1%\n") % references.size()).str(); + + for (PathSet::iterator j = references.begin(); + j != references.end(); ++j) + s += *j + "\n"; + } + + return s; +} + + +ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) { ValidPathInfo info; getline(str, info.path); if (str.eof()) { info.path = ""; return info; } + if (hashGiven) { + string s; + getline(str, s); + info.hash = parseHash(htSHA256, s); + } getline(str, info.deriver); string s; int n; getline(str, s); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e44259dda..39a7c9b70 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -35,6 +35,9 @@ public: /* Checks whether a path is valid. */ virtual bool isValidPath(const Path & path) = 0; + /* Query the set of valid paths. */ + virtual PathSet queryValidPaths() = 0; + /* Queries the hash of a valid path. */ virtual Hash queryPathHash(const Path & path) = 0; @@ -249,6 +252,9 @@ extern boost::shared_ptr<StoreAPI> store; boost::shared_ptr<StoreAPI> openStore(bool reserveSpace = true); +string makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash); + struct ValidPathInfo { Path path; @@ -257,7 +263,8 @@ struct ValidPathInfo PathSet references; }; -ValidPathInfo decodeValidPathInfo(std::istream & str); +ValidPathInfo decodeValidPathInfo(std::istream & str, + bool hashGiven = false); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f0e36463d..17b3c18fa 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -401,26 +401,31 @@ static void opReadLog(Strings opFlags, Strings opArgs) } -static void opRegisterValidity(Strings opFlags, Strings opArgs) +static void opDumpDB(Strings opFlags, Strings opArgs) { - bool reregister = false; // !!! maybe this should be the default - - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--reregister") reregister = true; - else throw UsageError(format("unknown flag `%1%'") % *i); + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + PathSet validPaths = store->queryValidPaths(); + /* !!! this isn't streamy; makeValidityRegistration() builds a + potentially gigantic string. */ + cout << makeValidityRegistration(validPaths, true, true); +} - if (!opArgs.empty()) throw UsageError("no arguments expected"); +static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) +{ ValidPathInfos infos; while (1) { - ValidPathInfo info = decodeValidPathInfo(cin); + ValidPathInfo info = decodeValidPathInfo(cin, hashGiven); if (info.path == "") break; if (!store->isValidPath(info.path) || reregister) { /* !!! races */ - canonicalisePathMetaData(info.path); - info.hash = hashPath(htSHA256, info.path); + if (canonicalise) + canonicalisePathMetaData(info.path); + if (!hashGiven) + info.hash = hashPath(htSHA256, info.path); infos.push_back(info); } } @@ -432,6 +437,32 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs) } +static void opLoadDB(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + registerValidity(true, true, false); +} + + +static void opRegisterValidity(Strings opFlags, Strings opArgs) +{ + bool reregister = false; // !!! maybe this should be the default + bool hashGiven = false; + + for (Strings::iterator i = opFlags.begin(); + i != opFlags.end(); ++i) + if (*i == "--reregister") reregister = true; + else if (*i == "--hash-given") hashGiven = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + registerValidity(reregister, hashGiven, true); +} + + static void opCheckValidity(Strings opFlags, Strings opArgs) { bool printInvalid = false; @@ -681,6 +712,10 @@ void run(Strings args) op = opQuery; else if (arg == "--read-log" || arg == "-l") op = opReadLog; + else if (arg == "--dump-db") + op = opDumpDB; + else if (arg == "--load-db") + op = opLoadDB; else if (arg == "--register-validity") op = opRegisterValidity; else if (arg == "--check-validity")