From 58337e0e6122a97061dcf803954f72469f67afca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jul 2012 11:51:27 -0400 Subject: [PATCH 01/19] Set release date --- doc/manual/release-notes.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index 69ab0874b..ed06b638a 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -8,7 +8,7 @@ -
Release 1.1 (TBA) +
Release 1.1 (July 18, 2012) This release has the following improvements: From b7fd2c28224a69476434d69b5d9da3d150c07226 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jul 2012 14:59:03 -0400 Subject: [PATCH 02/19] Use "#pragma once" to prevent repeated header file inclusion --- .gitignore | 4 ---- src/libexpr/attr-path.hh | 9 +-------- src/libexpr/common-opts.hh | 7 +------ src/libexpr/eval-inline.hh | 7 +------ src/libexpr/eval.hh | 6 +----- src/libexpr/get-drvs.hh | 6 +----- src/libexpr/names.hh | 11 +---------- src/libexpr/nixexpr.hh | 6 +----- src/libexpr/symbol-table.hh | 5 +---- src/libexpr/value-to-xml.hh | 5 +---- src/libexpr/value.hh | 5 +---- src/libmain/shared.hh | 6 +----- src/libstore/derivations.hh | 6 +----- src/libstore/globals.hh | 6 +----- src/libstore/local-store.hh | 6 +----- src/libstore/misc.hh | 6 +----- src/libstore/pathlocks.hh | 6 +----- src/libstore/references.hh | 5 +---- src/libstore/remote-store.hh | 6 +----- src/libstore/store-api.hh | 6 +----- src/libstore/worker-protocol.hh | 7 +------ src/libutil/archive.hh | 6 +----- src/libutil/hash.hh | 6 +----- src/libutil/immutable.hh | 5 +---- src/libutil/serialise.hh | 6 +----- src/libutil/types.hh | 6 +----- src/libutil/util.hh | 6 +----- src/libutil/xml-writer.hh | 6 +----- src/nix-env/profiles.hh | 6 +----- src/nix-env/user-env.hh | 9 +-------- src/nix-store/dotgraph.hh | 5 +---- src/nix-store/xmlgraph.hh | 5 +---- 32 files changed, 31 insertions(+), 166 deletions(-) diff --git a/.gitignore b/.gitignore index d7f151507..3fb96f124 100644 --- a/.gitignore +++ b/.gitignore @@ -78,13 +78,9 @@ Makefile.in /src/libexpr/parser-tab.cc /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output -/src/libexpr/nixexpr-ast.hh -/src/libexpr/nixexpr-ast.cc /src/libexpr/nix.tbl # /src/libstore/ -/src/libstore/derivations-ast.cc -/src/libstore/derivations-ast.hh /src/libstore/schema.sql.hh # /src/nix-env/ diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index b106da5ef..d3aad7461 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -1,20 +1,13 @@ -#ifndef __ATTR_PATH_H -#define __ATTR_PATH_H +#pragma once #include "eval.hh" #include #include - namespace nix { - void findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Expr * e, Value & v); - } - - -#endif /* !__ATTR_PATH_H */ diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh index c28641e90..e2e3fe771 100644 --- a/src/libexpr/common-opts.hh +++ b/src/libexpr/common-opts.hh @@ -1,9 +1,7 @@ -#ifndef __COMMON_OPTS_H -#define __COMMON_OPTS_H +#pragma once #include "eval.hh" - namespace nix { /* Some common option parsing between nix-env and nix-instantiate. */ @@ -17,6 +15,3 @@ bool parseSearchPathArg(const string & arg, Strings::iterator & i, Path lookupFileArg(EvalState & state, string s); } - - -#endif /* !__COMMON_OPTS_H */ diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 6026a7d11..57a9e4c63 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -1,5 +1,4 @@ -#ifndef __EVAL_INLINE_H -#define __EVAL_INLINE_H +#pragma once #include "eval.hh" @@ -8,7 +7,6 @@ namespace nix { - LocalNoInlineNoReturn(void throwEvalError(const char * s)) { throw EvalError(s); @@ -55,7 +53,4 @@ inline void EvalState::forceList(Value & v) throwTypeError("value is %1% while a list was expected", showType(v)); } - } - -#endif /* !__EVAL_INLINE_H */ diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index bab9303b0..5103ae8ce 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,5 +1,4 @@ -#ifndef __EVAL_H -#define __EVAL_H +#pragma once #include "value.hh" #include "nixexpr.hh" @@ -256,6 +255,3 @@ string showType(const Value & v); } - - -#endif /* !__EVAL_H */ diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 2d260c57b..879dc8dbb 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -1,5 +1,4 @@ -#ifndef __GET_DRVS_H -#define __GET_DRVS_H +#pragma once #include "eval.hh" @@ -78,6 +77,3 @@ void getDerivations(EvalState & state, Value & v, const string & pathPrefix, } - - -#endif /* !__GET_DRVS_H */ diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh index e189302d6..ebe113e82 100644 --- a/src/libexpr/names.hh +++ b/src/libexpr/names.hh @@ -1,12 +1,9 @@ -#ifndef __NAMES_H -#define __NAMES_H +#pragma once #include "types.hh" - namespace nix { - struct DrvName { string fullName; @@ -19,15 +16,9 @@ struct DrvName bool matches(DrvName & n); }; - typedef list DrvNames; - int compareVersions(const string & v1, const string & v2); DrvNames drvNamesFromArgs(const Strings & opArgs); - } - - -#endif /* !__NAMES_H */ diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 6eb771a72..4c1a0bb2d 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -1,5 +1,4 @@ -#ifndef __NIXEXPR_H -#define __NIXEXPR_H +#pragma once #include "value.hh" #include "symbol-table.hh" @@ -290,6 +289,3 @@ struct StaticEnv } - - -#endif /* !__NIXEXPR_H */ diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 976117a20..143fc495b 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -1,5 +1,4 @@ -#ifndef __SYMBOL_TABLE_H -#define __SYMBOL_TABLE_H +#pragma once #include "config.h" @@ -88,5 +87,3 @@ public: }; } - -#endif /* !__SYMBOL_TABLE_H */ diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 3ebc989ff..97657327e 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -1,5 +1,4 @@ -#ifndef __VALUE_TO_XML_H -#define __VALUE_TO_XML_H +#pragma once #include "nixexpr.hh" #include "eval.hh" @@ -13,5 +12,3 @@ void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); } - -#endif /* !__VALUE_TO_XML_H */ diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 41512d40b..c9ec236c4 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,5 +1,4 @@ -#ifndef __VALUE_H -#define __VALUE_H +#pragma once #include "symbol-table.hh" @@ -151,5 +150,3 @@ void mkPath(Value & v, const char * s); } - -#endif /* !__VALUE_H */ diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index d198df70d..7849e10e3 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,5 +1,4 @@ -#ifndef __SHARED_H -#define __SHARED_H +#pragma once #include "util.hh" @@ -54,6 +53,3 @@ extern int exitCode; extern char * * argvSaved; } - - -#endif /* !__SHARED_H */ diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 27e471d88..8f22b4afa 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,5 +1,4 @@ -#ifndef __DERIVATIONS_H -#define __DERIVATIONS_H +#pragma once #include @@ -81,6 +80,3 @@ typedef std::map DrvHashes; extern DrvHashes drvHashes; } - - -#endif /* !__DERIVATIONS_H */ diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 12a9b9ca1..1c0877a5e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,5 +1,4 @@ -#ifndef __GLOBALS_H -#define __GLOBALS_H +#pragma once #include "types.hh" @@ -118,6 +117,3 @@ void setDefaultsFromEnvironment(); } - - -#endif /* !__GLOBALS_H */ diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 65ee029c2..eb7705219 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,5 +1,4 @@ -#ifndef __LOCAL_STORE_H -#define __LOCAL_STORE_H +#pragma once #include @@ -302,6 +301,3 @@ void deletePathWrapped(const Path & path, void deletePathWrapped(const Path & path); } - - -#endif /* !__LOCAL_STORE_H */ diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh index 850c12af4..fe0bbdd79 100644 --- a/src/libstore/misc.hh +++ b/src/libstore/misc.hh @@ -1,5 +1,4 @@ -#ifndef __MISC_H -#define __MISC_H +#pragma once #include "derivations.hh" @@ -35,6 +34,3 @@ void queryMissing(StoreAPI & store, const PathSet & targets, } - - -#endif /* !__MISC_H */ diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 57ca1584a..8a6b1450d 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,5 +1,4 @@ -#ifndef __PATHLOCKS_H -#define __PATHLOCKS_H +#pragma once #include "types.hh" @@ -44,6 +43,3 @@ bool pathIsLockedByMe(const Path & path); } - - -#endif /* !__PATHLOCKS_H */ diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 158c08a77..013809d12 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,5 +1,4 @@ -#ifndef __REFERENCES_H -#define __REFERENCES_H +#pragma once #include "types.hh" #include "hash.hh" @@ -10,5 +9,3 @@ PathSet scanForReferences(const Path & path, const PathSet & refs, HashResult & hash); } - -#endif /* !__REFERENCES_H */ diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f0e5dbf76..c57b49ce1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,5 +1,4 @@ -#ifndef __REMOTE_STORE_H -#define __REMOTE_STORE_H +#pragma once #include @@ -101,6 +100,3 @@ private: } - - -#endif /* !__REMOTE_STORE_H */ diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0ab15c380..9ba67852e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,4 @@ -#ifndef __STOREAPI_H -#define __STOREAPI_H +#pragma once #include "hash.hh" #include "serialise.hh" @@ -356,6 +355,3 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */ } - - -#endif /* !__STOREAPI_H */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b08410fa1..501c0b3db 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,6 +1,4 @@ -#ifndef __WORKER_PROTOCOL_H -#define __WORKER_PROTOCOL_H - +#pragma once namespace nix { @@ -64,6 +62,3 @@ template T readStorePaths(Source & from); } - - -#endif /* !__WORKER_PROTOCOL_H */ diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index fff620313..ccac92074 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -1,5 +1,4 @@ -#ifndef __ARCHIVE_H -#define __ARCHIVE_H +#pragma once #include "types.hh" #include "serialise.hh" @@ -74,6 +73,3 @@ void restorePath(const Path & path, Source & source); } - - -#endif /* !__ARCHIVE_H */ diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index e0b6478cc..781f51742 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -1,5 +1,4 @@ -#ifndef __HASH_H -#define __HASH_H +#pragma once #include "types.hh" #include "serialise.hh" @@ -109,6 +108,3 @@ public: } - - -#endif /* !__HASH_H */ diff --git a/src/libutil/immutable.hh b/src/libutil/immutable.hh index 5a42a4610..8af419004 100644 --- a/src/libutil/immutable.hh +++ b/src/libutil/immutable.hh @@ -1,5 +1,4 @@ -#ifndef __IMMUTABLE_H -#define __IMMUTABLE_H +#pragma once #include @@ -15,5 +14,3 @@ void makeImmutable(const Path & path); void makeMutable(const Path & path); } - -#endif /* !__IMMUTABLE_H */ diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index ded4b12a0..42dd27117 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,5 +1,4 @@ -#ifndef __SERIALISE_H -#define __SERIALISE_H +#pragma once #include "types.hh" @@ -130,6 +129,3 @@ MakeError(SerialisationError, Error) } - - -#endif /* !__SERIALISE_H */ diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 844ad6f76..165a46fa2 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,5 +1,4 @@ -#ifndef __TYPES_H -#define __TYPES_H +#pragma once #include #include @@ -74,6 +73,3 @@ typedef enum { } - - -#endif /* !__TYPES_H */ diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ee0f3862a..362d0f65e 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,5 +1,4 @@ -#ifndef __UTIL_H -#define __UTIL_H +#pragma once #include "types.hh" @@ -333,6 +332,3 @@ void ignoreException(); } - - -#endif /* !__UTIL_H */ diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index e5cc5f8c5..fee2eb495 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -1,5 +1,4 @@ -#ifndef __XML_WRITER_H -#define __XML_WRITER_H +#pragma once #include #include @@ -70,6 +69,3 @@ public: } - - -#endif /* !__XML_WRITER_H */ diff --git a/src/nix-env/profiles.hh b/src/nix-env/profiles.hh index a64258dae..30d2376d9 100644 --- a/src/nix-env/profiles.hh +++ b/src/nix-env/profiles.hh @@ -1,5 +1,4 @@ -#ifndef __PROFILES_H -#define __PROFILES_H +#pragma once #include "types.hh" #include "pathlocks.hh" @@ -54,6 +53,3 @@ void lockProfile(PathLocks & lock, const Path & profile); string optimisticLockProfile(const Path & profile); } - - -#endif /* !__PROFILES_H */ diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh index 4125d8217..f188efe9b 100644 --- a/src/nix-env/user-env.hh +++ b/src/nix-env/user-env.hh @@ -1,5 +1,4 @@ -#ifndef __USER_ENV_H -#define __USER_ENV_H +#pragma once #include "get-drvs.hh" @@ -12,9 +11,3 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, const string & lockToken); } - -#endif /* !__USER_ENV_H */ - - - - diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index 2318e2fde..68410d841 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -1,5 +1,4 @@ -#ifndef __DOTGRAPH_H -#define __DOTGRAPH_H +#pragma once #include "types.hh" @@ -8,5 +7,3 @@ namespace nix { void printDotGraph(const PathSet & roots); } - -#endif /* !__DOTGRAPH_H */ diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/xmlgraph.hh index 2f9908c43..c2216c5a4 100644 --- a/src/nix-store/xmlgraph.hh +++ b/src/nix-store/xmlgraph.hh @@ -1,5 +1,4 @@ -#ifndef __XMLGRAPH_H -#define __XMLGRAPH_H +#pragma once #include "types.hh" @@ -8,5 +7,3 @@ namespace nix { void printXmlGraph(const PathSet & roots); } - -#endif /* !__XMLGRAPH_H */ From 98193bb440561875d2829f9dd542e38972dbcf63 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jul 2012 16:50:56 -0400 Subject: [PATCH 03/19] Remove RPM builds that don't evaluate --- release.nix | 4 ---- 1 file changed, 4 deletions(-) diff --git a/release.nix b/release.nix index 5f20920f3..47d296c62 100644 --- a/release.nix +++ b/release.nix @@ -153,10 +153,6 @@ let rpm_fedora13x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora13x86_64) 50; rpm_fedora16i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora16i386) 50; rpm_fedora16x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora16x86_64) 50; - - rpm_opensuse103i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.opensuse103i386) 40; - rpm_opensuse110i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.opensuse110i386) 50; - rpm_opensuse110x86_64 = makeRPM_x86_64 (diskImageFuns: diskImageFuns.opensuse110x86_64) 50; deb_debian60i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian60i386) 50; From 1832ab71dbb6b24965eb5a873a56a7231da7af4e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jul 2012 17:17:23 -0400 Subject: [PATCH 04/19] Bump version --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index b123147e2..ea710abb9 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1 \ No newline at end of file +1.2 \ No newline at end of file From 6852289c46cdfceb07b459cd1028722ffb124ca6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 16:52:25 -0400 Subject: [PATCH 05/19] Use lutimes() if available to canonicalise the timestamp of symlinks Also use utimes() instead of utime() if lutimes() is not available. --- configure.ac | 5 +++++ src/libstore/local-store.cc | 25 ++++++++++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index 7b814dedc..a2512cfe9 100644 --- a/configure.ac +++ b/configure.ac @@ -115,6 +115,11 @@ AC_CHECK_HEADERS([sys/mount.h], [], [], ]) +# Check for lutimes, optionally used for changing the mtime of +# symlinks. +AC_CHECK_FUNCS([lutimes]) + + # Check for . AC_LANG_PUSH(C++) AC_CHECK_HEADERS([locale]) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 30398a244..e009191b6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -444,7 +445,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse) throw SysError(format("changing owner of `%1%' to %2%") % path % geteuid()); } - + if (!S_ISLNK(st.st_mode)) { /* Mask out all type related bits. */ @@ -458,14 +459,20 @@ void canonicalisePathMetaData(const Path & path, bool recurse) throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); } - if (st.st_mtime != mtimeStore) { - struct utimbuf utimbuf; - utimbuf.actime = st.st_atime; - utimbuf.modtime = mtimeStore; - if (utime(path.c_str(), &utimbuf) == -1) - throw SysError(format("changing modification time of `%1%'") % path); - } - + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError(format("changing modification time of `%1%'") % path); } if (recurse && S_ISDIR(st.st_mode)) { From ed59bf7a181bb382dea7dd72da52bf91f60deb8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 17:11:12 -0400 Subject: [PATCH 06/19] =?UTF-8?q?nix-build:=20Support=20the=20=E2=80=98-?= =?UTF-8?q?=E2=80=99=20argument=20to=20build=20an=20expression=20from=20st?= =?UTF-8?q?din?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/nix-build.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/nix-build.in b/scripts/nix-build.in index 35b186bb7..afe0679a4 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -118,6 +118,10 @@ EOF elsif ($arg eq "--show-trace") { push @instArgs, $arg; } + + elsif ($arg eq "-") { + @exprs = ("-"); + } elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") { push @buildArgs, $arg; From 564fb7d9fa80d06397a88d69f26439727cb922c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 12:08:34 -0400 Subject: [PATCH 07/19] optimiseStore(): Use a content-addressed file store in /nix/store/.links optimiseStore() now creates persistent, content-addressed hard links in /nix/store/.links. For instance, if it encounters a file P with hash H, it will create a hard link P' = /nix/store/.link/ to P if P' doesn't already exist; if P' exist, then P is replaced by a hard link to P'. This is better than the previous in-memory map, because it had the tendency to unnecessarily replace hard links with a hard link to whatever happened to be the first file with a given hash it encountered. It also allows on-the-fly, incremental optimisation. --- src/libstore/local-store.hh | 2 +- src/libstore/optimise-store.cc | 211 +++++++++++++++++---------------- src/nix-store/nix-store.cc | 10 +- 3 files changed, 109 insertions(+), 114 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index eb7705219..1bb47fb3b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -167,7 +167,7 @@ public: /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ - void optimiseStore(bool dryRun, OptimiseStats & stats); + void optimiseStore(OptimiseStats & stats); /* Check the integrity of the Nix store. */ void verifyStore(bool checkContents); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 2ca98f46d..0893db9d3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,7 @@ #include "util.hh" #include "local-store.hh" #include "immutable.hh" +#include "globals.hh" #include #include @@ -12,9 +13,6 @@ namespace nix { -typedef std::map > HashToPath; - - static void makeWritable(const Path & path) { struct stat st; @@ -51,132 +49,135 @@ struct MakeImmutable }; -static void hashAndLink(bool dryRun, HashToPath & hashToPath, - OptimiseStats & stats, const Path & path) +const string linksDir = ".links"; + + +static void hashAndLink(OptimiseStats & stats, const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path `%1%'") % path); + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + hashAndLink(stats, path + "/" + *i); + return; + } + + /* We can hard link regular files and symlinks. */ + if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) return; + /* Sometimes SNAFUs can cause files in the Nix store to be modified, in particular when running programs as root under NixOS (example: $fontconfig/var/cache being modified). Skip - those files. */ + those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); return; } - /* We can hard link regular files and symlinks. */ - if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). - /* Hash the file. Note that hashPath() returns the hash over - the NAR serialisation, which includes the execute bit on - the file. Thus, executable and non-executable files with - the same contents *won't* be linked (which is good because - otherwise the permissions would be screwed up). + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + stats.totalFiles++; + printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); - Also note that if `path' is a symlink, then we're hashing - the contents of the symlink (i.e. the result of - readlink()), not the contents of the target (which may not - even exist). */ - Hash hash = hashPath(htSHA256, path).first; - stats.totalFiles++; - printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); + /* Check if this is a known hash. */ + Path linkPath = nixStore + "/" + linksDir + "/" + printHash32(hash); - std::pair prevPath = hashToPath[hash]; - - if (prevPath.first == "") { - hashToPath[hash] = std::pair(path, st.st_ino); - return; - } - - /* Yes! We've seen a file with the same contents. Replace - the current file with a hard link to that file. */ - stats.sameContents++; - if (prevPath.second == st.st_ino) { - printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first); - return; - } - - if (!dryRun) { - - printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first); + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + makeMutable(path); + MakeImmutable mk1(path); - Path tempLink = (format("%1%.tmp-%2%-%3%") - % path % getpid() % rand()).str(); + if (link(path.c_str(), linkPath.c_str()) == -1) + throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); - /* Make the containing directory writable, but only if - it's not the store itself (we don't want or need to - mess with its permissions). */ - bool mustToggle = !isStorePath(path); - if (mustToggle) makeWritable(dirOf(path)); - - /* When we're done, make the directory read-only again and - reset its timestamp back to 0. */ - MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); - - /* If ‘prevPath’ is immutable, we can't create hard links - to it, so make it mutable first (and make it immutable - again when we're done). We also have to make ‘path’ - mutable, otherwise rename() will fail to delete it. */ - makeMutable(prevPath.first); - MakeImmutable mk1(prevPath.first); - - makeMutable(path); - MakeImmutable mk2(path); - - if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) { - if (errno == EMLINK) { - /* Too many links to the same file (>= 32000 on - most file systems). This is likely to happen - with empty files. Just start over, creating - links to the current file. */ - printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first); - hashToPath[hash] = std::pair(path, st.st_ino); - return; - } - throw SysError(format("cannot link `%1%' to `%2%'") - % tempLink % prevPath.first); - } - - /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { - if (errno == EMLINK) { - /* Some filesystems generate too many links on the - rename, rather than on the original link. - (Probably it temporarily increases the st_nlink - field before decreasing it again.) */ - printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first); - hashToPath[hash] = std::pair(path, st.st_ino); - - /* Unlink the temp link. */ - if (unlink(tempLink.c_str()) == -1) - printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); - return; - } - throw SysError(format("cannot rename `%1%' to `%2%'") - % tempLink % path); - } - } else - printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first); - - stats.filesLinked++; - stats.bytesFreed += st.st_size; - stats.blocksFreed += st.st_blocks; + return; } - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - hashAndLink(dryRun, hashToPath, stats, path + "/" + *i); + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink)) + throw SysError(format("getting attributes of path `%1%'") % linkPath); + + stats.sameContents++; + if (st.st_ino == stLink.st_ino) { + printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); + return; } + + printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath); + + Path tempLink = (format("%1%/.tmp-link-%2%-%3%") + % nixStore % getpid() % rand()).str(); + + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = !isStorePath(path); + if (mustToggle) makeWritable(dirOf(path)); + + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + /* If ‘linkPath’ is immutable, we can't create hard links to it, + so make it mutable first (and make it immutable again when + we're done). We also have to make ‘path’ mutable, otherwise + rename() will fail to delete it. */ + makeMutable(linkPath); + MakeImmutable mk1(linkPath); + + makeMutable(path); + MakeImmutable mk2(path); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + return; + } + throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); + } + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + + /* Unlink the temp link. */ + if (unlink(linkPath.c_str()) == -1) + printMsg(lvlError, format("unable to unlink `%1%'") % linkPath); + return; + } + throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); + } + + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; } -void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) +void LocalStore::optimiseStore(OptimiseStats & stats) { - HashToPath hashToPath; + createDirs(nixStore + "/" + linksDir); PathSet paths = queryValidPaths(); @@ -184,7 +185,7 @@ void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) addTempRoot(*i); if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - hashAndLink(dryRun, hashToPath, stats, *i); + hashAndLink(stats, *i); } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 23863525f..82e08fecf 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -746,18 +746,12 @@ static void showOptimiseStats(OptimiseStats & stats) files with the same contents. */ static void opOptimise(Strings opFlags, Strings opArgs) { - if (!opArgs.empty()) + if (!opArgs.empty() || !opFlags.empty()) throw UsageError("no arguments expected"); - bool dryRun = false; - - foreach (Strings::iterator, i, opFlags) - if (*i == "--dry-run") dryRun = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - OptimiseStats stats; try { - ensureLocalStore().optimiseStore(dryRun, stats); + ensureLocalStore().optimiseStore(stats); } catch (...) { showOptimiseStats(stats); throw; From 619310571002fc74e428824bd603604d1055b61b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 15:02:52 -0400 Subject: [PATCH 08/19] Automatically optimise the Nix store when a new path is added Auto-optimisation is enabled by default. It can be turned off by setting auto-optimise-store to false in nix.conf. --- doc/manual/conf-file.xml | 15 ++++++++++++++- src/libstore/build.cc | 4 ++++ src/libstore/local-store.cc | 7 +++++++ src/libstore/local-store.hh | 7 +++++++ src/libstore/optimise-store.cc | 22 +++++++++++++--------- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 1b19e56b5..c095a001c 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -337,7 +337,20 @@ build-use-chroot = /dev /proc /bin true. - + + + auto-optimise-store + + If set to true (the default), + Nix automatically detects files in the store that have identical + contents, and replaces them with hard links to a single copy. + This saves disk space. If set to false, you + can still run nix-store --optimise to get rid + of duplicate files. + + + + diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 26268f6dd..a3bde3462 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2093,6 +2093,8 @@ void DerivationGoal::computeClosure() if (allowed.find(*i) == allowed.end()) throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); } + + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() } /* Register each output path as valid, and register the sets of @@ -2546,6 +2548,8 @@ void SubstitutionGoal::finished() HashResult hash = hashPath(htSHA256, storePath); + worker.store.optimisePath(storePath); // FIXME: combine with hashPath() + ValidPathInfo info2; info2.path = storePath; info2.hash = hash.first; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e009191b6..05b2b9c6e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -209,6 +209,7 @@ LocalStore::LocalStore(bool reserveSpace) /* Create missing state directories if they don't already exist. */ createDirs(nixStore); + createDirs(linksDir = nixStore + "/.links"); Path profilesDir = nixStateDir + "/profiles"; createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/temproots"); @@ -1116,6 +1117,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, hash.second = dump.size(); } else hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; @@ -1170,6 +1173,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, canonicalisePathMetaData(dstPath); HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); ValidPathInfo info; info.path = dstPath; @@ -1405,6 +1410,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* !!! if we were clever, we could prevent the hashPath() here. */ HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1bb47fb3b..7d30a2d40 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -85,6 +85,8 @@ private: typedef std::map RunningSubstituters; RunningSubstituters runningSubstituters; + + Path linksDir; public: @@ -169,6 +171,9 @@ public: files with the same contents. */ void optimiseStore(OptimiseStats & stats); + /* Optimise a single store path. */ + void optimisePath(const Path & path); + /* Check the integrity of the Nix store. */ void verifyStore(bool checkContents); @@ -267,6 +272,8 @@ private: Path importPath(bool requireSignature, Source & source); void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + + void optimisePath_(OptimiseStats & stats, const Path & path); }; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 0893db9d3..a7aa14fb4 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -49,10 +49,7 @@ struct MakeImmutable }; -const string linksDir = ".links"; - - -static void hashAndLink(OptimiseStats & stats, const Path & path) +void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) @@ -61,7 +58,7 @@ static void hashAndLink(OptimiseStats & stats, const Path & path) if (S_ISDIR(st.st_mode)) { Strings names = readDirectory(path); foreach (Strings::iterator, i, names) - hashAndLink(stats, path + "/" + *i); + optimisePath_(stats, path + "/" + *i); return; } @@ -91,7 +88,7 @@ static void hashAndLink(OptimiseStats & stats, const Path & path) printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); /* Check if this is a known hash. */ - Path linkPath = nixStore + "/" + linksDir + "/" + printHash32(hash); + Path linkPath = linksDir + "/" + printHash32(hash); if (!pathExists(linkPath)) { /* Nope, create a hard link in the links directory. */ @@ -177,15 +174,22 @@ static void hashAndLink(OptimiseStats & stats, const Path & path) void LocalStore::optimiseStore(OptimiseStats & stats) { - createDirs(nixStore + "/" + linksDir); - PathSet paths = queryValidPaths(); foreach (PathSet::iterator, i, paths) { addTempRoot(*i); if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - hashAndLink(stats, *i); + optimisePath_(stats, *i); + } +} + + +void LocalStore::optimisePath(const Path & path) +{ + if (queryBoolSetting("auto-optimise-store", true)) { + OptimiseStats stats; + optimisePath_(stats, path); } } From 680ab6f83def2b636200204542ca352631a46f85 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 15:48:30 -0400 Subject: [PATCH 09/19] Garbage collect unused links in /nix/store/.links Incremental optimisation requires creating links in /nix/store/.links to all files in the store. However, this means that if we delete a store path, no files are actually deleted because links in /nix/store/.links still exists. So we need to check /nix/store/.links for files with a link count of 1 and delete them. --- src/libstore/gc.cc | 37 +++++++++++++++++++++++++++++++++++++ src/libstore/local-store.hh | 2 ++ 2 files changed, 39 insertions(+) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f6ed7dd22..874efe4d3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -436,6 +436,8 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) { checkInterrupt(); + if (path == linksDir) return true; + struct stat st; if (lstat(path.c_str(), &st)) { if (errno == ENOENT) return true; @@ -569,6 +571,37 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) } +/* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ +void LocalStore::removeUnusedLinks() +{ + AutoCloseDir dir = opendir(linksDir.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("statting `%1%'") % path); + + if (st.st_nlink != 1) continue; + + printMsg(lvlTalkative, format("deleting unused link `%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError(format("deleting `%1%'") % path); + } +} + + void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { GCState state(results); @@ -682,6 +715,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) released. */ foreach (PathSet::iterator, i, state.invalidated) deleteGarbage(state, *i); + + /* Clean up the links directory. */ + printMsg(lvlError, format("deleting unused links...")); + removeUnusedLinks(); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7d30a2d40..50910f353 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -264,6 +264,8 @@ private: int openGCLock(LockType lockType); + void removeUnusedLinks(); + void startSubstituter(const Path & substituter, RunningSubstituter & runningSubstituter); From 0f65793f94bd89c973482ac949be1e96e876762b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 17:40:23 -0400 Subject: [PATCH 10/19] Add a test for Nix store optimisation --- tests/Makefile.am | 2 +- tests/optimise-store.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/optimise-store.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index 517c382b1..a562db52b 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -9,7 +9,7 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh negative-caching.sh \ binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ - multiple-outputs.sh import-derivation.sh fetchurl.sh + multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh XFAIL_TESTS = diff --git a/tests/optimise-store.sh b/tests/optimise-store.sh new file mode 100644 index 000000000..4d8077997 --- /dev/null +++ b/tests/optimise-store.sh @@ -0,0 +1,26 @@ +source common.sh + +clearStore + +outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) +outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) + +inode1="$(perl -e "print ((lstat('$outPath1/foo'))[1])")" +inode2="$(perl -e "print ((lstat('$outPath2/foo'))[1])")" +if [ "$inode1" != "$inode2" ]; then + echo "inodes do not match" + exit 1 +fi + +nlink="$(perl -e "print ((lstat('$outPath1/foo'))[3])")" +if [ "$nlink" != 3 ]; then + echo "link count incorrect" + exit 1 +fi + +nix-store --gc + +if [ -n "$(ls $NIX_STORE_DIR/.links)" ]; then + echo ".links directory not empty after GC" + exit 1 +fi From fd63c8bfcd75624e7fbba8899365095400534e01 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 18:06:37 -0400 Subject: [PATCH 11/19] Unlink the right file --- src/libstore/optimise-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index a7aa14fb4..e5bfa332d 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -159,8 +159,8 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); /* Unlink the temp link. */ - if (unlink(linkPath.c_str()) == -1) - printMsg(lvlError, format("unable to unlink `%1%'") % linkPath); + if (unlink(tempLink.c_str()) == -1) + printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); return; } throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); From e98c029717016dfa3e5c618c9fc46da9b2142dcc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 18:42:18 -0400 Subject: [PATCH 12/19] Handle platforms that don't support linking to a symlink E.g. Darwin doesn't allow this. --- configure.ac | 12 ++++++++++++ src/libstore/optimise-store.cc | 9 +++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index a2512cfe9..f920b8c10 100644 --- a/configure.ac +++ b/configure.ac @@ -120,6 +120,18 @@ AC_CHECK_HEADERS([sys/mount.h], [], [], AC_CHECK_FUNCS([lutimes]) +# Check whether the store optimiser can optimise symlinks. +AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) +ln -s bla tmp_link +if ln tmp_link tmp_link2 2> /dev/null; then + AC_MSG_RESULT(yes) + AC_DEFINE(CAN_LINK_SYMLINK, 1, [Whether link() works on symlinks.]) +else + AC_MSG_RESULT(no) +fi +rm -f tmp_link tmp_link2 + + # Check for . AC_LANG_PUSH(C++) AC_CHECK_HEADERS([locale]) diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index e5bfa332d..486538bd2 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -62,8 +62,13 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) return; } - /* We can hard link regular files and symlinks. */ - if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) return; + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) +#if CAN_LINK_SYMLINK + x + && !S_ISLNK(st.st_mode) +#endif + ) return; /* Sometimes SNAFUs can cause files in the Nix store to be modified, in particular when running programs as root under From 566a30c0072690900d4d55679a2981758d6fb888 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 23 Jul 2012 12:51:04 -0400 Subject: [PATCH 13/19] Disable tests temporarily --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 47d296c62..fd9db4793 100644 --- a/release.nix +++ b/release.nix @@ -82,7 +82,7 @@ let enableParallelBuilding = true; - doInstallCheck = true; + doInstallCheck = false; }; binaryTarball = From b1112bbef195bc8397c4e88aa8544537a6d84731 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 23 Jul 2012 13:41:28 -0400 Subject: [PATCH 14/19] import: If the path is a valid .drv file, parse it and generate a derivation attrset. The generated attrset has drvPath and outPath with the right string context, type 'derivation', outputName with the right name, all with a list of outputs, and an attribute for each output. I see three uses for this (though certainly there may be more): * Using derivations generated by something besides nix-instantiate (e.g. guix) * Allowing packages provided by channels to be used in nix expressions. If a channel installed a valid deriver for each package it provides into the store, then those could be imported and used as dependencies or installed in environment.systemPackages, for example. * Enable hydra to be consistent in how it treats inputs that are outputs of another build. Right now, if an input is passed as an argument to the job, it is passed as a derivation, but if it is accessed via NIX_PATH (i.e. through the <> syntax), then it is a path that can be imported. This is problematic because the build being depended upon may have been built with non-obvious arguments passed to its jobset file. With this feature, hydra can just set the name of that input to the path to its drv file in NIX_PATH --- corepkgs/Makefile.am | 3 ++- corepkgs/imported-drv-to-derivation.nix | 21 ++++++++++++++++++++ src/libexpr/primops.cc | 26 ++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 corepkgs/imported-drv-to-derivation.nix diff --git a/corepkgs/Makefile.am b/corepkgs/Makefile.am index 729d15e7b..4b0b8860b 100644 --- a/corepkgs/Makefile.am +++ b/corepkgs/Makefile.am @@ -1,6 +1,7 @@ all-local: config.nix -files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix +files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix \ + imported-drv-to-derivation.nix install-exec-local: $(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs diff --git a/corepkgs/imported-drv-to-derivation.nix b/corepkgs/imported-drv-to-derivation.nix new file mode 100644 index 000000000..bdb601698 --- /dev/null +++ b/corepkgs/imported-drv-to-derivation.nix @@ -0,0 +1,21 @@ +attrs @ { drvPath, outputs, ... }: + +let + + commonAttrs = (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvPath; + type = "derivation"; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName attrs; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + +in (builtins.head outputsList).value diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5d5f0bfb3..2ab3eda53 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -59,7 +59,31 @@ static void prim_import(EvalState & state, Value * * args, Value & v) } } - state.evalFile(path, v); + if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { + Derivation drv = parseDerivation(readFile(path)); + Value w; + state.mkAttrs(w, 1 + drv.outputs.size()); + mkString(*state.allocAttr(w, state.sDrvPath), path, singleton("=" + path)); + state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size()); + unsigned int outputs_index = 0; + + Value * outputsVal = w.attrs->find(state.symbols.create("outputs"))->value; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + mkString(*state.allocAttr(w, state.symbols.create(i->first)), + i->second.path, singleton("!" + i->first + "!" + path)); + mkString(*(outputsVal->list.elems[outputs_index++] = state.allocValue()), + i->first); + } + w.attrs->sort(); + Value fun; + state.mkThunk_(fun, + state.parseExprFromFile(state.findFile("nix/imported-drv-to-derivation.nix"))); + state.forceFunction(fun); + mkApp(v, fun, w); + state.forceAttrs(v); + } else { + state.evalFile(path, v); + } } From 1ef2d5765be35c3d3c13a2aea8748166f576ec8b Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 23 Jul 2012 13:45:51 -0400 Subject: [PATCH 15/19] Turn tests back on --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index fd9db4793..47d296c62 100644 --- a/release.nix +++ b/release.nix @@ -82,7 +82,7 @@ let enableParallelBuilding = true; - doInstallCheck = false; + doInstallCheck = true; }; binaryTarball = From f5954e2d940c3a41a6ed0cad45660e254eb381a3 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Tue, 24 Jul 2012 12:05:27 -0400 Subject: [PATCH 16/19] prim_import: When importing .drvs, allocate the intermediate attrset on the heap just in case it escapes the stack frame. --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2ab3eda53..0d4efc47e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -61,7 +61,7 @@ static void prim_import(EvalState & state, Value * * args, Value & v) if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { Derivation drv = parseDerivation(readFile(path)); - Value w; + Value & w = *state.allocValue(); state.mkAttrs(w, 1 + drv.outputs.size()); mkString(*state.allocAttr(w, state.sDrvPath), path, singleton("=" + path)); state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size()); From 477b0fbeca62bf1957bc0aad26f1a844ebd22231 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Jul 2012 16:56:56 -0400 Subject: [PATCH 17/19] Subscribe to the Nixpkgs rather than NixOS channel --- scripts/nix-profile.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index d343385cc..b18069d94 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -19,9 +19,9 @@ fi export PATH="$HOME/.nix-profile/bin:$PATH" -# Subscribe the root user to the NixOS channel by default. +# Subscribe the root user to the Nixpkgs channel by default. if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then - echo "http://nixos.org/releases/nixos/channels/nixos-unstable nixos" > $HOME/.nix-channels + echo "http://nixos.org/releases/nixos/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels fi # Create the per-user garbage collector roots directory. From 2605f4f4e6a367df67bf8b33b252c350313699c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Jul 2012 17:06:09 -0400 Subject: [PATCH 18/19] nix-profile.sh: Don't set NIX_REMOTE on single user installations Commit 6a214f3e06fa1c5f0a4d40e555f14d87691af297 reused the NixOS environment initialisation for nix-profile.sh, but this is inappropriate on systems that don't have multi-user support enabled. --- scripts/nix-profile.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index b18069d94..bc3dc719e 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -43,7 +43,7 @@ fi # Set up secure multi-user builds: non-root users build through the # Nix daemon. -if test "$USER" != root; then +if [ "$USER" != root -a -e @localstatedir@/nix/daemon-socket/socket ]; then export NIX_REMOTE=daemon else unset NIX_REMOTE From 3a4623afbbc1bff85bde33167d36e8c5a4a3df0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Jul 2012 15:04:40 -0400 Subject: [PATCH 19/19] Set permissions on temporary build directories to 0700 Fixes #39. --- src/libstore/build.cc | 3 ++- src/libutil/util.cc | 4 ++-- src/libutil/util.hh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index a3bde3462..290635695 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1527,7 +1527,7 @@ void DerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false); + tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false, 0700); /* For convenience, set an environment pointing to the top build directory. */ @@ -2178,6 +2178,7 @@ void DerivationGoal::deleteTmpDir(bool force) % drvPath % tmpDir); if (buildUser.enabled() && !amPrivileged()) getOwnership(tmpDir); + chmod(tmpDir.c_str(), 0755); } else deletePathWrapped(tmpDir); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index b188a9fc0..689fc543a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -380,7 +380,7 @@ static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter) + bool includePid, bool useGlobalCounter, mode_t mode) { static int globalCounter = 0; int localCounter = 0; @@ -389,7 +389,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, while (1) { checkInterrupt(); Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), 0777) == 0) { + if (mkdir(tmpDir.c_str(), mode) == 0) { /* Explicitly set the group of the directory. This is to work around around problems caused by BSD's group ownership semantics (directories inherit the group of diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 362d0f65e..9b8656f70 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -88,7 +88,7 @@ void makePathReadOnly(const Path & path); /* Create a temporary directory. */ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true); + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */