From 619310571002fc74e428824bd603604d1055b61b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 15:02:52 -0400 Subject: [PATCH] Automatically optimise the Nix store when a new path is added Auto-optimisation is enabled by default. It can be turned off by setting auto-optimise-store to false in nix.conf. --- doc/manual/conf-file.xml | 15 ++++++++++++++- src/libstore/build.cc | 4 ++++ src/libstore/local-store.cc | 7 +++++++ src/libstore/local-store.hh | 7 +++++++ src/libstore/optimise-store.cc | 22 +++++++++++++--------- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 1b19e56b5..c095a001c 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -337,7 +337,20 @@ build-use-chroot = /dev /proc /bin true. - + + + auto-optimise-store + + If set to true (the default), + Nix automatically detects files in the store that have identical + contents, and replaces them with hard links to a single copy. + This saves disk space. If set to false, you + can still run nix-store --optimise to get rid + of duplicate files. + + + + diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 26268f6dd..a3bde3462 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2093,6 +2093,8 @@ void DerivationGoal::computeClosure() if (allowed.find(*i) == allowed.end()) throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); } + + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() } /* Register each output path as valid, and register the sets of @@ -2546,6 +2548,8 @@ void SubstitutionGoal::finished() HashResult hash = hashPath(htSHA256, storePath); + worker.store.optimisePath(storePath); // FIXME: combine with hashPath() + ValidPathInfo info2; info2.path = storePath; info2.hash = hash.first; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e009191b6..05b2b9c6e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -209,6 +209,7 @@ LocalStore::LocalStore(bool reserveSpace) /* Create missing state directories if they don't already exist. */ createDirs(nixStore); + createDirs(linksDir = nixStore + "/.links"); Path profilesDir = nixStateDir + "/profiles"; createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/temproots"); @@ -1116,6 +1117,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, hash.second = dump.size(); } else hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; @@ -1170,6 +1173,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, canonicalisePathMetaData(dstPath); HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); ValidPathInfo info; info.path = dstPath; @@ -1405,6 +1410,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* !!! if we were clever, we could prevent the hashPath() here. */ HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1bb47fb3b..7d30a2d40 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -85,6 +85,8 @@ private: typedef std::map RunningSubstituters; RunningSubstituters runningSubstituters; + + Path linksDir; public: @@ -169,6 +171,9 @@ public: files with the same contents. */ void optimiseStore(OptimiseStats & stats); + /* Optimise a single store path. */ + void optimisePath(const Path & path); + /* Check the integrity of the Nix store. */ void verifyStore(bool checkContents); @@ -267,6 +272,8 @@ private: Path importPath(bool requireSignature, Source & source); void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + + void optimisePath_(OptimiseStats & stats, const Path & path); }; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 0893db9d3..a7aa14fb4 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -49,10 +49,7 @@ struct MakeImmutable }; -const string linksDir = ".links"; - - -static void hashAndLink(OptimiseStats & stats, const Path & path) +void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) @@ -61,7 +58,7 @@ static void hashAndLink(OptimiseStats & stats, const Path & path) if (S_ISDIR(st.st_mode)) { Strings names = readDirectory(path); foreach (Strings::iterator, i, names) - hashAndLink(stats, path + "/" + *i); + optimisePath_(stats, path + "/" + *i); return; } @@ -91,7 +88,7 @@ static void hashAndLink(OptimiseStats & stats, const Path & path) printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); /* Check if this is a known hash. */ - Path linkPath = nixStore + "/" + linksDir + "/" + printHash32(hash); + Path linkPath = linksDir + "/" + printHash32(hash); if (!pathExists(linkPath)) { /* Nope, create a hard link in the links directory. */ @@ -177,15 +174,22 @@ static void hashAndLink(OptimiseStats & stats, const Path & path) void LocalStore::optimiseStore(OptimiseStats & stats) { - createDirs(nixStore + "/" + linksDir); - PathSet paths = queryValidPaths(); foreach (PathSet::iterator, i, paths) { addTempRoot(*i); if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - hashAndLink(stats, *i); + optimisePath_(stats, *i); + } +} + + +void LocalStore::optimisePath(const Path & path) +{ + if (queryBoolSetting("auto-optimise-store", true)) { + OptimiseStats stats; + optimisePath_(stats, path); } }