Merge branch 'master' into no-manifests

This commit is contained in:
Eelco Dolstra 2012-07-26 15:14:33 -04:00
commit 8c79100839
50 changed files with 343 additions and 307 deletions

4
.gitignore vendored
View file

@ -79,13 +79,9 @@ Makefile.in
/src/libexpr/parser-tab.cc /src/libexpr/parser-tab.cc
/src/libexpr/parser-tab.hh /src/libexpr/parser-tab.hh
/src/libexpr/parser-tab.output /src/libexpr/parser-tab.output
/src/libexpr/nixexpr-ast.hh
/src/libexpr/nixexpr-ast.cc
/src/libexpr/nix.tbl /src/libexpr/nix.tbl
# /src/libstore/ # /src/libstore/
/src/libstore/derivations-ast.cc
/src/libstore/derivations-ast.hh
/src/libstore/schema.sql.hh /src/libstore/schema.sql.hh
# /src/nix-env/ # /src/nix-env/

View file

@ -115,6 +115,23 @@ AC_CHECK_HEADERS([sys/mount.h], [], [],
]) ])
# Check for lutimes, optionally used for changing the mtime of
# symlinks.
AC_CHECK_FUNCS([lutimes])
# Check whether the store optimiser can optimise symlinks.
AC_MSG_CHECKING([whether it is possible to create a link to a symlink])
ln -s bla tmp_link
if ln tmp_link tmp_link2 2> /dev/null; then
AC_MSG_RESULT(yes)
AC_DEFINE(CAN_LINK_SYMLINK, 1, [Whether link() works on symlinks.])
else
AC_MSG_RESULT(no)
fi
rm -f tmp_link tmp_link2
# Check for <locale>. # Check for <locale>.
AC_LANG_PUSH(C++) AC_LANG_PUSH(C++)
AC_CHECK_HEADERS([locale]) AC_CHECK_HEADERS([locale])

View file

@ -1,6 +1,7 @@
all-local: config.nix all-local: config.nix
files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix \
imported-drv-to-derivation.nix
install-exec-local: install-exec-local:
$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs $(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs

View file

@ -0,0 +1,21 @@
attrs @ { drvPath, outputs, ... }:
let
commonAttrs = (builtins.listToAttrs outputsList) //
{ all = map (x: x.value) outputsList;
inherit drvPath;
type = "derivation";
};
outputToAttrListElement = outputName:
{ name = outputName;
value = commonAttrs // {
outPath = builtins.getAttr outputName attrs;
inherit outputName;
};
};
outputsList = map outputToAttrListElement outputs;
in (builtins.head outputsList).value

View file

@ -357,7 +357,20 @@ build-use-chroot = /dev /proc /bin</programlisting>
<literal>true</literal>.</para></listitem> <literal>true</literal>.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry><term><literal>auto-optimise-store</literal></term>
<listitem><para>If set to <literal>true</literal> (the default),
Nix automatically detects files in the store that have identical
contents, and replaces them with hard links to a single copy.
This saves disk space. If set to <literal>false</literal>, you
can still run <command>nix-store --optimise</command> to get rid
of duplicate files.</para></listitem>
</varlistentry>
</variablelist> </variablelist>
</para> </para>

View file

@ -8,7 +8,7 @@
<!--==================================================================--> <!--==================================================================-->
<section xml:id="ssec-relnotes-1.1"><title>Release 1.1 (TBA)</title> <section xml:id="ssec-relnotes-1.1"><title>Release 1.1 (July 18, 2012)</title>
<para>This release has the following improvements:</para> <para>This release has the following improvements:</para>

View file

@ -156,10 +156,6 @@ let
rpm_fedora13x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora13x86_64) 50; rpm_fedora13x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora13x86_64) 50;
rpm_fedora16i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora16i386) 50; rpm_fedora16i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora16i386) 50;
rpm_fedora16x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora16x86_64) 50; rpm_fedora16x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora16x86_64) 50;
rpm_opensuse103i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.opensuse103i386) 40;
rpm_opensuse110i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.opensuse110i386) 50;
rpm_opensuse110x86_64 = makeRPM_x86_64 (diskImageFuns: diskImageFuns.opensuse110x86_64) 50;
deb_debian60i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian60i386) 50; deb_debian60i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian60i386) 50;

View file

@ -123,6 +123,10 @@ EOF
elsif ($arg eq "--show-trace") { elsif ($arg eq "--show-trace") {
push @instArgs, $arg; push @instArgs, $arg;
} }
elsif ($arg eq "-") {
@exprs = ("-");
}
elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") { elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") {
push @buildArgs, $arg; push @buildArgs, $arg;

View file

@ -19,9 +19,9 @@ fi
export PATH="$HOME/.nix-profile/bin:$PATH" export PATH="$HOME/.nix-profile/bin:$PATH"
# Subscribe the root user to the NixOS channel by default. # Subscribe the root user to the Nixpkgs channel by default.
if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then
echo "http://nixos.org/releases/nixos/channels/nixos-unstable nixos" > $HOME/.nix-channels echo "http://nixos.org/releases/nixos/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels
fi fi
# Create the per-user garbage collector roots directory. # Create the per-user garbage collector roots directory.
@ -43,7 +43,7 @@ fi
# Set up secure multi-user builds: non-root users build through the # Set up secure multi-user builds: non-root users build through the
# Nix daemon. # Nix daemon.
if test "$USER" != root; then if [ "$USER" != root -a -e @localstatedir@/nix/daemon-socket/socket ]; then
export NIX_REMOTE=daemon export NIX_REMOTE=daemon
else else
unset NIX_REMOTE unset NIX_REMOTE

View file

@ -1,20 +1,13 @@
#ifndef __ATTR_PATH_H #pragma once
#define __ATTR_PATH_H
#include "eval.hh" #include "eval.hh"
#include <string> #include <string>
#include <map> #include <map>
namespace nix { namespace nix {
void findAlongAttrPath(EvalState & state, const string & attrPath, void findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Expr * e, Value & v); Bindings & autoArgs, Expr * e, Value & v);
} }
#endif /* !__ATTR_PATH_H */

View file

@ -1,9 +1,7 @@
#ifndef __COMMON_OPTS_H #pragma once
#define __COMMON_OPTS_H
#include "eval.hh" #include "eval.hh"
namespace nix { namespace nix {
/* Some common option parsing between nix-env and nix-instantiate. */ /* Some common option parsing between nix-env and nix-instantiate. */
@ -17,6 +15,3 @@ bool parseSearchPathArg(const string & arg, Strings::iterator & i,
Path lookupFileArg(EvalState & state, string s); Path lookupFileArg(EvalState & state, string s);
} }
#endif /* !__COMMON_OPTS_H */

View file

@ -1,5 +1,4 @@
#ifndef __EVAL_INLINE_H #pragma once
#define __EVAL_INLINE_H
#include "eval.hh" #include "eval.hh"
@ -8,7 +7,6 @@
namespace nix { namespace nix {
LocalNoInlineNoReturn(void throwEvalError(const char * s)) LocalNoInlineNoReturn(void throwEvalError(const char * s))
{ {
throw EvalError(s); throw EvalError(s);
@ -55,7 +53,4 @@ inline void EvalState::forceList(Value & v)
throwTypeError("value is %1% while a list was expected", showType(v)); throwTypeError("value is %1% while a list was expected", showType(v));
} }
} }
#endif /* !__EVAL_INLINE_H */

View file

@ -1,5 +1,4 @@
#ifndef __EVAL_H #pragma once
#define __EVAL_H
#include "value.hh" #include "value.hh"
#include "nixexpr.hh" #include "nixexpr.hh"
@ -256,6 +255,3 @@ string showType(const Value & v);
} }
#endif /* !__EVAL_H */

View file

@ -1,5 +1,4 @@
#ifndef __GET_DRVS_H #pragma once
#define __GET_DRVS_H
#include "eval.hh" #include "eval.hh"
@ -83,6 +82,3 @@ void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
} }
#endif /* !__GET_DRVS_H */

View file

@ -1,12 +1,9 @@
#ifndef __NAMES_H #pragma once
#define __NAMES_H
#include "types.hh" #include "types.hh"
namespace nix { namespace nix {
struct DrvName struct DrvName
{ {
string fullName; string fullName;
@ -19,15 +16,9 @@ struct DrvName
bool matches(DrvName & n); bool matches(DrvName & n);
}; };
typedef list<DrvName> DrvNames; typedef list<DrvName> DrvNames;
int compareVersions(const string & v1, const string & v2); int compareVersions(const string & v1, const string & v2);
DrvNames drvNamesFromArgs(const Strings & opArgs); DrvNames drvNamesFromArgs(const Strings & opArgs);
} }
#endif /* !__NAMES_H */

View file

@ -1,5 +1,4 @@
#ifndef __NIXEXPR_H #pragma once
#define __NIXEXPR_H
#include "value.hh" #include "value.hh"
#include "symbol-table.hh" #include "symbol-table.hh"
@ -290,6 +289,3 @@ struct StaticEnv
} }
#endif /* !__NIXEXPR_H */

View file

@ -65,7 +65,31 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
} }
} }
state.evalFile(path, v); if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) {
Derivation drv = parseDerivation(readFile(path));
Value & w = *state.allocValue();
state.mkAttrs(w, 1 + drv.outputs.size());
mkString(*state.allocAttr(w, state.sDrvPath), path, singleton<PathSet>("=" + path));
state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size());
unsigned int outputs_index = 0;
Value * outputsVal = w.attrs->find(state.symbols.create("outputs"))->value;
foreach (DerivationOutputs::iterator, i, drv.outputs) {
mkString(*state.allocAttr(w, state.symbols.create(i->first)),
i->second.path, singleton<PathSet>("!" + i->first + "!" + path));
mkString(*(outputsVal->list.elems[outputs_index++] = state.allocValue()),
i->first);
}
w.attrs->sort();
Value fun;
state.mkThunk_(fun,
state.parseExprFromFile(state.findFile("nix/imported-drv-to-derivation.nix")));
state.forceFunction(fun);
mkApp(v, fun, w);
state.forceAttrs(v);
} else {
state.evalFile(path, v);
}
} }

View file

@ -1,5 +1,4 @@
#ifndef __SYMBOL_TABLE_H #pragma once
#define __SYMBOL_TABLE_H
#include "config.h" #include "config.h"
@ -88,5 +87,3 @@ public:
}; };
} }
#endif /* !__SYMBOL_TABLE_H */

View file

@ -1,5 +1,4 @@
#ifndef __VALUE_TO_XML_H #pragma once
#define __VALUE_TO_XML_H
#include "nixexpr.hh" #include "nixexpr.hh"
#include "eval.hh" #include "eval.hh"
@ -13,5 +12,3 @@ void printValueAsXML(EvalState & state, bool strict, bool location,
Value & v, std::ostream & out, PathSet & context); Value & v, std::ostream & out, PathSet & context);
} }
#endif /* !__VALUE_TO_XML_H */

View file

@ -1,5 +1,4 @@
#ifndef __VALUE_H #pragma once
#define __VALUE_H
#include "symbol-table.hh" #include "symbol-table.hh"
@ -151,5 +150,3 @@ void mkPath(Value & v, const char * s);
} }
#endif /* !__VALUE_H */

View file

@ -1,5 +1,4 @@
#ifndef __SHARED_H #pragma once
#define __SHARED_H
#include "util.hh" #include "util.hh"
@ -54,6 +53,3 @@ extern int exitCode;
extern char * * argvSaved; extern char * * argvSaved;
} }
#endif /* !__SHARED_H */

View file

@ -1533,7 +1533,7 @@ void DerivationGoal::startBuilder()
/* Create a temporary directory where the build will take /* Create a temporary directory where the build will take
place. */ place. */
tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false); tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false, 0700);
/* For convenience, set an environment pointing to the top build /* For convenience, set an environment pointing to the top build
directory. */ directory. */
@ -2099,6 +2099,8 @@ void DerivationGoal::computeClosure()
if (allowed.find(*i) == allowed.end()) if (allowed.find(*i) == allowed.end())
throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
} }
worker.store.optimisePath(path); // FIXME: combine with scanForReferences()
} }
/* Register each output path as valid, and register the sets of /* Register each output path as valid, and register the sets of
@ -2182,6 +2184,7 @@ void DerivationGoal::deleteTmpDir(bool force)
% drvPath % tmpDir); % drvPath % tmpDir);
if (buildUser.enabled() && !amPrivileged()) if (buildUser.enabled() && !amPrivileged())
getOwnership(tmpDir); getOwnership(tmpDir);
chmod(tmpDir.c_str(), 0755);
} }
else else
deletePathWrapped(tmpDir); deletePathWrapped(tmpDir);
@ -2562,6 +2565,8 @@ void SubstitutionGoal::finished()
HashResult hash = hashPath(htSHA256, storePath); HashResult hash = hashPath(htSHA256, storePath);
worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
ValidPathInfo info2; ValidPathInfo info2;
info2.path = storePath; info2.path = storePath;
info2.hash = hash.first; info2.hash = hash.first;

View file

@ -1,5 +1,4 @@
#ifndef __DERIVATIONS_H #pragma once
#define __DERIVATIONS_H
#include <map> #include <map>
@ -81,6 +80,3 @@ typedef std::map<Path, Hash> DrvHashes;
extern DrvHashes drvHashes; extern DrvHashes drvHashes;
} }
#endif /* !__DERIVATIONS_H */

View file

@ -436,6 +436,8 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
{ {
checkInterrupt(); checkInterrupt();
if (path == linksDir) return true;
struct stat st; struct stat st;
if (lstat(path.c_str(), &st)) { if (lstat(path.c_str(), &st)) {
if (errno == ENOENT) return true; if (errno == ENOENT) return true;
@ -569,6 +571,37 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
} }
/* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases
the link count. */
void LocalStore::removeUnusedLinks()
{
AutoCloseDir dir = opendir(linksDir.c_str());
if (!dir) throw SysError(format("opening directory `%1%'") % linksDir);
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir)) {
checkInterrupt();
string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name;
struct stat st;
if (lstat(path.c_str(), &st) == -1)
throw SysError(format("statting `%1%'") % path);
if (st.st_nlink != 1) continue;
printMsg(lvlTalkative, format("deleting unused link `%1%'") % path);
if (unlink(path.c_str()) == -1)
throw SysError(format("deleting `%1%'") % path);
}
}
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{ {
GCState state(results); GCState state(results);
@ -682,6 +715,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
released. */ released. */
foreach (PathSet::iterator, i, state.invalidated) foreach (PathSet::iterator, i, state.invalidated)
deleteGarbage(state, *i); deleteGarbage(state, *i);
/* Clean up the links directory. */
printMsg(lvlError, format("deleting unused links..."));
removeUnusedLinks();
} }

View file

@ -1,5 +1,4 @@
#ifndef __GLOBALS_H #pragma once
#define __GLOBALS_H
#include "types.hh" #include "types.hh"
@ -118,6 +117,3 @@ void setDefaultsFromEnvironment();
} }
#endif /* !__GLOBALS_H */

View file

@ -12,6 +12,7 @@
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/time.h>
#include <unistd.h> #include <unistd.h>
#include <utime.h> #include <utime.h>
#include <fcntl.h> #include <fcntl.h>
@ -208,6 +209,7 @@ LocalStore::LocalStore(bool reserveSpace)
/* Create missing state directories if they don't already exist. */ /* Create missing state directories if they don't already exist. */
createDirs(nixStore); createDirs(nixStore);
createDirs(linksDir = nixStore + "/.links");
Path profilesDir = nixStateDir + "/profiles"; Path profilesDir = nixStateDir + "/profiles";
createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/profiles");
createDirs(nixStateDir + "/temproots"); createDirs(nixStateDir + "/temproots");
@ -444,7 +446,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
throw SysError(format("changing owner of `%1%' to %2%") throw SysError(format("changing owner of `%1%' to %2%")
% path % geteuid()); % path % geteuid());
} }
if (!S_ISLNK(st.st_mode)) { if (!S_ISLNK(st.st_mode)) {
/* Mask out all type related bits. */ /* Mask out all type related bits. */
@ -458,14 +460,20 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
} }
if (st.st_mtime != mtimeStore) { }
struct utimbuf utimbuf;
utimbuf.actime = st.st_atime; if (st.st_mtime != mtimeStore) {
utimbuf.modtime = mtimeStore; struct timeval times[2];
if (utime(path.c_str(), &utimbuf) == -1) times[0].tv_sec = st.st_atime;
throw SysError(format("changing modification time of `%1%'") % path); times[0].tv_usec = 0;
} times[1].tv_sec = mtimeStore;
times[1].tv_usec = 0;
#if HAVE_LUTIMES
if (lutimes(path.c_str(), times) == -1)
#else
if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
#endif
throw SysError(format("changing modification time of `%1%'") % path);
} }
if (recurse && S_ISDIR(st.st_mode)) { if (recurse && S_ISDIR(st.st_mode)) {
@ -1134,6 +1142,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
hash.second = dump.size(); hash.second = dump.size();
} else } else
hash = hashPath(htSHA256, dstPath); hash = hashPath(htSHA256, dstPath);
optimisePath(dstPath); // FIXME: combine with hashPath()
ValidPathInfo info; ValidPathInfo info;
info.path = dstPath; info.path = dstPath;
@ -1188,6 +1198,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
canonicalisePathMetaData(dstPath); canonicalisePathMetaData(dstPath);
HashResult hash = hashPath(htSHA256, dstPath); HashResult hash = hashPath(htSHA256, dstPath);
optimisePath(dstPath);
ValidPathInfo info; ValidPathInfo info;
info.path = dstPath; info.path = dstPath;
@ -1423,6 +1435,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
/* !!! if we were clever, we could prevent the hashPath() /* !!! if we were clever, we could prevent the hashPath()
here. */ here. */
HashResult hash = hashPath(htSHA256, dstPath); HashResult hash = hashPath(htSHA256, dstPath);
optimisePath(dstPath); // FIXME: combine with hashPath()
ValidPathInfo info; ValidPathInfo info;
info.path = dstPath; info.path = dstPath;

View file

@ -1,5 +1,4 @@
#ifndef __LOCAL_STORE_H #pragma once
#define __LOCAL_STORE_H
#include <string> #include <string>
@ -86,6 +85,8 @@ private:
typedef std::map<Path, RunningSubstituter> RunningSubstituters; typedef std::map<Path, RunningSubstituter> RunningSubstituters;
RunningSubstituters runningSubstituters; RunningSubstituters runningSubstituters;
Path linksDir;
public: public:
@ -168,8 +169,11 @@ public:
/* Optimise the disk space usage of the Nix store by hard-linking /* Optimise the disk space usage of the Nix store by hard-linking
files with the same contents. */ files with the same contents. */
void optimiseStore(bool dryRun, OptimiseStats & stats); void optimiseStore(OptimiseStats & stats);
/* Optimise a single store path. */
void optimisePath(const Path & path);
/* Check the integrity of the Nix store. */ /* Check the integrity of the Nix store. */
void verifyStore(bool checkContents); void verifyStore(bool checkContents);
@ -260,6 +264,8 @@ private:
int openGCLock(LockType lockType); int openGCLock(LockType lockType);
void removeUnusedLinks();
void startSubstituter(const Path & substituter, void startSubstituter(const Path & substituter,
RunningSubstituter & runningSubstituter); RunningSubstituter & runningSubstituter);
@ -268,6 +274,8 @@ private:
Path importPath(bool requireSignature, Source & source); Path importPath(bool requireSignature, Source & source);
void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
void optimisePath_(OptimiseStats & stats, const Path & path);
}; };
@ -302,6 +310,3 @@ void deletePathWrapped(const Path & path,
void deletePathWrapped(const Path & path); void deletePathWrapped(const Path & path);
} }
#endif /* !__LOCAL_STORE_H */

View file

@ -1,5 +1,4 @@
#ifndef __MISC_H #pragma once
#define __MISC_H
#include "derivations.hh" #include "derivations.hh"
@ -35,6 +34,3 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
} }
#endif /* !__MISC_H */

View file

@ -1,6 +1,7 @@
#include "util.hh" #include "util.hh"
#include "local-store.hh" #include "local-store.hh"
#include "immutable.hh" #include "immutable.hh"
#include "globals.hh"
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
@ -12,9 +13,6 @@
namespace nix { namespace nix {
typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath;
static void makeWritable(const Path & path) static void makeWritable(const Path & path)
{ {
struct stat st; struct stat st;
@ -51,140 +49,152 @@ struct MakeImmutable
}; };
static void hashAndLink(bool dryRun, HashToPath & hashToPath, void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
OptimiseStats & stats, const Path & path)
{ {
struct stat st; struct stat st;
if (lstat(path.c_str(), &st)) if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path); throw SysError(format("getting attributes of path `%1%'") % path);
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path);
foreach (Strings::iterator, i, names)
optimisePath_(stats, path + "/" + *i);
return;
}
/* We can hard link regular files and maybe symlinks. */
if (!S_ISREG(st.st_mode)
#if CAN_LINK_SYMLINK
x
&& !S_ISLNK(st.st_mode)
#endif
) return;
/* Sometimes SNAFUs can cause files in the Nix store to be /* Sometimes SNAFUs can cause files in the Nix store to be
modified, in particular when running programs as root under modified, in particular when running programs as root under
NixOS (example: $fontconfig/var/cache being modified). Skip NixOS (example: $fontconfig/var/cache being modified). Skip
those files. */ those files. FIXME: check the modification time. */
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
return; return;
} }
/* We can hard link regular files and symlinks. */ /* Hash the file. Note that hashPath() returns the hash over the
if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { NAR serialisation, which includes the execute bit on the file.
Thus, executable and non-executable files with the same
contents *won't* be linked (which is good because otherwise the
permissions would be screwed up).
/* Hash the file. Note that hashPath() returns the hash over Also note that if `path' is a symlink, then we're hashing the
the NAR serialisation, which includes the execute bit on contents of the symlink (i.e. the result of readlink()), not
the file. Thus, executable and non-executable files with the contents of the target (which may not even exist). */
the same contents *won't* be linked (which is good because Hash hash = hashPath(htSHA256, path).first;
otherwise the permissions would be screwed up). stats.totalFiles++;
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
Also note that if `path' is a symlink, then we're hashing /* Check if this is a known hash. */
the contents of the symlink (i.e. the result of Path linkPath = linksDir + "/" + printHash32(hash);
readlink()), not the contents of the target (which may not
even exist). */
Hash hash = hashPath(htSHA256, path).first;
stats.totalFiles++;
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
std::pair<Path, ino_t> prevPath = hashToPath[hash]; if (!pathExists(linkPath)) {
/* Nope, create a hard link in the links directory. */
if (prevPath.first == "") { makeMutable(path);
hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino); MakeImmutable mk1(path);
return;
}
/* Yes! We've seen a file with the same contents. Replace
the current file with a hard link to that file. */
stats.sameContents++;
if (prevPath.second == st.st_ino) {
printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first);
return;
}
if (!dryRun) {
printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first);
Path tempLink = (format("%1%.tmp-%2%-%3%") if (link(path.c_str(), linkPath.c_str()) == -1)
% path % getpid() % rand()).str(); throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
/* Make the containing directory writable, but only if return;
it's not the store itself (we don't want or need to
mess with its permissions). */
bool mustToggle = !isStorePath(path);
if (mustToggle) makeWritable(dirOf(path));
/* When we're done, make the directory read-only again and
reset its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
/* If prevPath is immutable, we can't create hard links
to it, so make it mutable first (and make it immutable
again when we're done). We also have to make path
mutable, otherwise rename() will fail to delete it. */
makeMutable(prevPath.first);
MakeImmutable mk1(prevPath.first);
makeMutable(path);
MakeImmutable mk2(path);
if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) {
/* Too many links to the same file (>= 32000 on
most file systems). This is likely to happen
with empty files. Just start over, creating
links to the current file. */
printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
return;
}
throw SysError(format("cannot link `%1%' to `%2%'")
% tempLink % prevPath.first);
}
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (errno == EMLINK) {
/* Some filesystems generate too many links on the
rename, rather than on the original link.
(Probably it temporarily increases the st_nlink
field before decreasing it again.) */
printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
/* Unlink the temp link. */
if (unlink(tempLink.c_str()) == -1)
printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
return;
}
throw SysError(format("cannot rename `%1%' to `%2%'")
% tempLink % path);
}
} else
printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
stats.filesLinked++;
stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks;
} }
if (S_ISDIR(st.st_mode)) { /* Yes! We've seen a file with the same contents. Replace the
Strings names = readDirectory(path); current file with a hard link to that file. */
foreach (Strings::iterator, i, names) struct stat stLink;
hashAndLink(dryRun, hashToPath, stats, path + "/" + *i); if (lstat(linkPath.c_str(), &stLink))
throw SysError(format("getting attributes of path `%1%'") % linkPath);
stats.sameContents++;
if (st.st_ino == stLink.st_ino) {
printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath);
return;
} }
printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath);
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
% nixStore % getpid() % rand()).str();
/* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its
permissions). */
bool mustToggle = !isStorePath(path);
if (mustToggle) makeWritable(dirOf(path));
/* When we're done, make the directory read-only again and reset
its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
/* If linkPath is immutable, we can't create hard links to it,
so make it mutable first (and make it immutable again when
we're done). We also have to make path mutable, otherwise
rename() will fail to delete it. */
makeMutable(linkPath);
MakeImmutable mk1(linkPath);
makeMutable(path);
MakeImmutable mk2(path);
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
if (errno == EMLINK) {
/* Too many links to the same file (>= 32000 on most file
systems). This is likely to happen with empty files.
Just shrug and ignore. */
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
return;
}
throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
}
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (errno == EMLINK) {
/* Some filesystems generate too many links on the rename,
rather than on the original link. (Probably it
temporarily increases the st_nlink field before
decreasing it again.) */
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
/* Unlink the temp link. */
if (unlink(tempLink.c_str()) == -1)
printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
return;
}
throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
}
stats.filesLinked++;
stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks;
} }
void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) void LocalStore::optimiseStore(OptimiseStats & stats)
{ {
HashToPath hashToPath;
PathSet paths = queryAllValidPaths(); PathSet paths = queryAllValidPaths();
foreach (PathSet::iterator, i, paths) { foreach (PathSet::iterator, i, paths) {
addTempRoot(*i); addTempRoot(*i);
if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
hashAndLink(dryRun, hashToPath, stats, *i); optimisePath_(stats, *i);
}
}
void LocalStore::optimisePath(const Path & path)
{
if (queryBoolSetting("auto-optimise-store", true)) {
OptimiseStats stats;
optimisePath_(stats, path);
} }
} }

View file

@ -1,5 +1,4 @@
#ifndef __PATHLOCKS_H #pragma once
#define __PATHLOCKS_H
#include "types.hh" #include "types.hh"
@ -44,6 +43,3 @@ bool pathIsLockedByMe(const Path & path);
} }
#endif /* !__PATHLOCKS_H */

View file

@ -1,5 +1,4 @@
#ifndef __REFERENCES_H #pragma once
#define __REFERENCES_H
#include "types.hh" #include "types.hh"
#include "hash.hh" #include "hash.hh"
@ -10,5 +9,3 @@ PathSet scanForReferences(const Path & path, const PathSet & refs,
HashResult & hash); HashResult & hash);
} }
#endif /* !__REFERENCES_H */

View file

@ -1,5 +1,4 @@
#ifndef __REMOTE_STORE_H #pragma once
#define __REMOTE_STORE_H
#include <string> #include <string>
@ -103,6 +102,3 @@ private:
} }
#endif /* !__REMOTE_STORE_H */

View file

@ -1,5 +1,4 @@
#ifndef __STOREAPI_H #pragma once
#define __STOREAPI_H
#include "hash.hh" #include "hash.hh"
#include "serialise.hh" #include "serialise.hh"
@ -362,6 +361,3 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */
} }
#endif /* !__STOREAPI_H */

View file

@ -1,6 +1,4 @@
#ifndef __WORKER_PROTOCOL_H #pragma once
#define __WORKER_PROTOCOL_H
namespace nix { namespace nix {
@ -67,6 +65,3 @@ template<class T> T readStorePaths(Source & from);
} }
#endif /* !__WORKER_PROTOCOL_H */

View file

@ -1,5 +1,4 @@
#ifndef __ARCHIVE_H #pragma once
#define __ARCHIVE_H
#include "types.hh" #include "types.hh"
#include "serialise.hh" #include "serialise.hh"
@ -74,6 +73,3 @@ void restorePath(const Path & path, Source & source);
} }
#endif /* !__ARCHIVE_H */

View file

@ -1,5 +1,4 @@
#ifndef __HASH_H #pragma once
#define __HASH_H
#include "types.hh" #include "types.hh"
#include "serialise.hh" #include "serialise.hh"
@ -109,6 +108,3 @@ public:
} }
#endif /* !__HASH_H */

View file

@ -1,5 +1,4 @@
#ifndef __IMMUTABLE_H #pragma once
#define __IMMUTABLE_H
#include <types.hh> #include <types.hh>
@ -15,5 +14,3 @@ void makeImmutable(const Path & path);
void makeMutable(const Path & path); void makeMutable(const Path & path);
} }
#endif /* !__IMMUTABLE_H */

View file

@ -1,5 +1,4 @@
#ifndef __SERIALISE_H #pragma once
#define __SERIALISE_H
#include "types.hh" #include "types.hh"
@ -130,6 +129,3 @@ MakeError(SerialisationError, Error)
} }
#endif /* !__SERIALISE_H */

View file

@ -1,5 +1,4 @@
#ifndef __TYPES_H #pragma once
#define __TYPES_H
#include <string> #include <string>
#include <list> #include <list>
@ -74,6 +73,3 @@ typedef enum {
} }
#endif /* !__TYPES_H */

View file

@ -380,7 +380,7 @@ static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
Path createTempDir(const Path & tmpRoot, const Path & prefix, Path createTempDir(const Path & tmpRoot, const Path & prefix,
bool includePid, bool useGlobalCounter) bool includePid, bool useGlobalCounter, mode_t mode)
{ {
static int globalCounter = 0; static int globalCounter = 0;
int localCounter = 0; int localCounter = 0;
@ -389,7 +389,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
while (1) { while (1) {
checkInterrupt(); checkInterrupt();
Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
if (mkdir(tmpDir.c_str(), 0777) == 0) { if (mkdir(tmpDir.c_str(), mode) == 0) {
/* Explicitly set the group of the directory. This is to /* Explicitly set the group of the directory. This is to
work around around problems caused by BSD's group work around around problems caused by BSD's group
ownership semantics (directories inherit the group of ownership semantics (directories inherit the group of

View file

@ -1,5 +1,4 @@
#ifndef __UTIL_H #pragma once
#define __UTIL_H
#include "types.hh" #include "types.hh"
@ -89,7 +88,7 @@ void makePathReadOnly(const Path & path);
/* Create a temporary directory. */ /* Create a temporary directory. */
Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
bool includePid = true, bool useGlobalCounter = true); bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
/* Create a directory and all its parents, if necessary. Returns the /* Create a directory and all its parents, if necessary. Returns the
list of created directories, in order of creation. */ list of created directories, in order of creation. */
@ -333,6 +332,3 @@ void ignoreException();
} }
#endif /* !__UTIL_H */

View file

@ -1,5 +1,4 @@
#ifndef __XML_WRITER_H #pragma once
#define __XML_WRITER_H
#include <iostream> #include <iostream>
#include <string> #include <string>
@ -70,6 +69,3 @@ public:
} }
#endif /* !__XML_WRITER_H */

View file

@ -1,5 +1,4 @@
#ifndef __PROFILES_H #pragma once
#define __PROFILES_H
#include "types.hh" #include "types.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
@ -54,6 +53,3 @@ void lockProfile(PathLocks & lock, const Path & profile);
string optimisticLockProfile(const Path & profile); string optimisticLockProfile(const Path & profile);
} }
#endif /* !__PROFILES_H */

View file

@ -1,5 +1,4 @@
#ifndef __USER_ENV_H #pragma once
#define __USER_ENV_H
#include "get-drvs.hh" #include "get-drvs.hh"
@ -12,9 +11,3 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
const string & lockToken); const string & lockToken);
} }
#endif /* !__USER_ENV_H */

View file

@ -1,5 +1,4 @@
#ifndef __DOTGRAPH_H #pragma once
#define __DOTGRAPH_H
#include "types.hh" #include "types.hh"
@ -8,5 +7,3 @@ namespace nix {
void printDotGraph(const PathSet & roots); void printDotGraph(const PathSet & roots);
} }
#endif /* !__DOTGRAPH_H */

View file

@ -746,18 +746,12 @@ static void showOptimiseStats(OptimiseStats & stats)
files with the same contents. */ files with the same contents. */
static void opOptimise(Strings opFlags, Strings opArgs) static void opOptimise(Strings opFlags, Strings opArgs)
{ {
if (!opArgs.empty()) if (!opArgs.empty() || !opFlags.empty())
throw UsageError("no arguments expected"); throw UsageError("no arguments expected");
bool dryRun = false;
foreach (Strings::iterator, i, opFlags)
if (*i == "--dry-run") dryRun = true;
else throw UsageError(format("unknown flag `%1%'") % *i);
OptimiseStats stats; OptimiseStats stats;
try { try {
ensureLocalStore().optimiseStore(dryRun, stats); ensureLocalStore().optimiseStore(stats);
} catch (...) { } catch (...) {
showOptimiseStats(stats); showOptimiseStats(stats);
throw; throw;

View file

@ -1,5 +1,4 @@
#ifndef __XMLGRAPH_H #pragma once
#define __XMLGRAPH_H
#include "types.hh" #include "types.hh"
@ -8,5 +7,3 @@ namespace nix {
void printXmlGraph(const PathSet & roots); void printXmlGraph(const PathSet & roots);
} }
#endif /* !__XMLGRAPH_H */

View file

@ -9,7 +9,7 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
remote-store.sh export.sh export-graph.sh negative-caching.sh \ remote-store.sh export.sh export-graph.sh negative-caching.sh \
binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \
multiple-outputs.sh import-derivation.sh fetchurl.sh multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh
XFAIL_TESTS = XFAIL_TESTS =

26
tests/optimise-store.sh Normal file
View file

@ -0,0 +1,26 @@
source common.sh
clearStore
outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link)
outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link)
inode1="$(perl -e "print ((lstat('$outPath1/foo'))[1])")"
inode2="$(perl -e "print ((lstat('$outPath2/foo'))[1])")"
if [ "$inode1" != "$inode2" ]; then
echo "inodes do not match"
exit 1
fi
nlink="$(perl -e "print ((lstat('$outPath1/foo'))[3])")"
if [ "$nlink" != 3 ]; then
echo "link count incorrect"
exit 1
fi
nix-store --gc
if [ -n "$(ls $NIX_STORE_DIR/.links)" ]; then
echo ".links directory not empty after GC"
exit 1
fi

View file

@ -1 +1 @@
1.1 1.2