From 087be7281a02a6d0e548ae56b910771837023d21 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sun, 13 Jan 2019 10:33:15 -0500 Subject: [PATCH 01/45] Treat plain derivation paths in context as normal paths. Previously, plain derivation paths in the string context (e.g. those that arose from builtins.storePath on a drv file, not those that arose from accessing .drvPath of a derivation) were treated somewhat like derivaiton paths derived from .drvPath, except their dependencies weren't recursively added to the input set. With this change, such plain derivation paths are simply treated as paths and added to the source inputs set accordingly, simplifying context handling code and removing the inconsistency. If drvPath-like behavior is desired, the .drv file can be imported and then .drvPath can be accessed. This is a backwards-incompatibility, but storePath is never used on drv files within nixpkgs and almost never used elsewhere. --- .gitignore | 3 +++ src/libexpr/primops.cc | 20 +++----------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 0f2f3ddee..b75c5d489 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,9 @@ perl/Makefile.config /tests/common.sh /tests/dummy /tests/result* +/tests/restricted-innocent +/tests/shell +/tests/shell.drv # /tests/lang/ /tests/lang/*.out diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 60698f740..113ceaf29 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -687,21 +687,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } - /* See prim_unsafeDiscardOutputDependency. */ - else if (path.at(0) == '~') - drv.inputSrcs.insert(string(path, 1)); - /* Handle derivation outputs of the form ‘!!’. */ else if (path.at(0) == '!') { std::pair ctx = decodeContext(path); drv.inputDrvs[ctx.first].insert(ctx.second); } - /* Handle derivation contexts returned by - ‘builtins.storePath’. */ - else if (isDerivation(path)) - drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path); - /* Otherwise it's a source file. */ else drv.inputSrcs.insert(path); @@ -1004,13 +995,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu PathSet refs; for (auto path : context) { - if (path.at(0) == '=') path = string(path, 1); - if (isDerivation(path)) { - /* See prim_unsafeDiscardOutputDependency. */ - if (path.at(0) != '~') - throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); - path = string(path, 1); - } + if (path.at(0) != '/') + throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); refs.insert(path); } @@ -1823,7 +1809,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po PathSet context2; for (auto & p : context) - context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p); + context2.insert(p.at(0) == '=' ? string(p, 1) : p); mkString(v, s, context2); } From 1d757292d0cb78beec32fcdfe15c2944a4bc4a95 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sun, 13 Jan 2019 12:43:46 -0500 Subject: [PATCH 02/45] Add builtins.getContext. This can be very helpful when debugging, as well as enabling complex black magic like surgically removing a single dependency from a string's context. --- src/libexpr/eval.hh | 3 + src/libexpr/primops.cc | 38 ----- src/libexpr/primops/context.cc | 132 ++++++++++++++++++ .../lang/eval-okay-context-introspection.exp | 1 + .../lang/eval-okay-context-introspection.nix | 22 +++ 5 files changed, 158 insertions(+), 38 deletions(-) create mode 100644 src/libexpr/primops/context.cc create mode 100644 tests/lang/eval-okay-context-introspection.exp create mode 100644 tests/lang/eval-okay-context-introspection.nix diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d0f298e16..9fe387891 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -316,6 +316,9 @@ private: /* Return a string representing the type of the value `v'. */ string showType(const Value & v); +/* Decode a context string ‘!!’ into a pair . */ +std::pair decodeContext(const string & s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 113ceaf29..0da9f702f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1780,41 +1780,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args } -static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkString(v, s, PathSet()); -} - - -static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - state.forceString(*args[0], context, pos); - mkBool(v, !context.empty()); -} - - -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ -static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - - PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? string(p, 1) : p); - - mkString(v, s, context2); -} - - /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2285,9 +2250,6 @@ void EvalState::createBaseEnv() addPrimOp("toString", 1, prim_toString); addPrimOp("__substring", 3, prim_substring); addPrimOp("__stringLength", 1, prim_stringLength); - addPrimOp("__hasContext", 1, prim_hasContext); - addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); - addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); addPrimOp("__hashString", 2, prim_hashString); addPrimOp("__match", 2, prim_match); addPrimOp("__split", 2, prim_split); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc new file mode 100644 index 000000000..6849aa261 --- /dev/null +++ b/src/libexpr/primops/context.cc @@ -0,0 +1,132 @@ +#include "primops.hh" +#include "eval-inline.hh" + +namespace nix { + +static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + mkString(v, s, PathSet()); +} + +static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); + + +static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); +} + +static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); + + +/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a + builder without causing the derivation to be built (for instance, + in the derivation that builds NARs in nix-push, when doing + source-only deployment). This primop marks the string context so + that builtins.derivation adds the path to drv.inputSrcs rather than + drv.inputDrvs. */ +static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + + PathSet context2; + for (auto & p : context) + context2.insert(p.at(0) == '=' ? string(p, 1) : p); + + mkString(v, s, context2); +} + +static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); + + +/* Extract the context of a string as a structured Nix value. + + The context is represented as an attribute set whose keys are the + paths in the context set and whose values are attribute sets with + the following keys: + path: True if the relevant path is in the context as a plain store + path (i.e. the kind of context you get when interpolating + a Nix path (e.g. ./.) into a string). False if missing. + allOutputs: True if the relevant path is a derivation and it is + in the context as a drv file with all of its outputs + (i.e. the kind of context you get when referencing + .drvPath of some derivation). False if missing. + outputs: If a non-empty list, the relevant path is a derivation + and the provided outputs are referenced in the context + (i.e. the kind of context you get when referencing + .outPath of some derivation). Empty list if missing. + Note that for a given path any combination of the above attributes + may be present, but at least one must be set to something other + than the default. +*/ +static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + struct ContextInfo { + bool path = false; + bool allOutputs = false; + Strings outputs; + }; + PathSet context; + state.forceString(*args[0], context, pos); + auto contextInfos = std::map(); + for (const auto & p : context) { + Path drv; + string output; + const Path * path = &p; + if (p.at(0) == '=') { + drv = string(p, 1); + path = &drv; + } else if (p.at(0) == '!') { + std::pair ctx = decodeContext(p); + drv = ctx.first; + output = ctx.second; + path = &drv; + } + auto isPath = drv.empty(); + auto isAllOutputs = (!drv.empty()) && output.empty(); + + auto iter = contextInfos.find(*path); + if (iter == contextInfos.end()) { + contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); + } else { + if (isPath) + iter->second.path = true; + else if (isAllOutputs) + iter->second.allOutputs = true; + else + iter->second.outputs.emplace_back(std::move(output)); + } + } + + state.mkAttrs(v, contextInfos.size()); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (const auto & info : contextInfos) { + auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); + state.mkAttrs(infoVal, 3); + if (info.second.path) + mkBool(*state.allocAttr(infoVal, sPath), true); + if (info.second.allOutputs) + mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + if (!info.second.outputs.empty()) { + auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); + state.mkList(outputsVal, info.second.outputs.size()); + size_t i = 0; + for (const auto & output : info.second.outputs) { + mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); + } + } + infoVal.attrs->sort(); + } + v.attrs->sort(); +} + +static RegisterPrimOp r4("__getContext", 1, prim_getContext); + +} diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/lang/eval-okay-context-introspection.exp new file mode 100644 index 000000000..27ba77dda --- /dev/null +++ b/tests/lang/eval-okay-context-introspection.exp @@ -0,0 +1 @@ +true diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/lang/eval-okay-context-introspection.nix new file mode 100644 index 000000000..d9b2ea354 --- /dev/null +++ b/tests/lang/eval-okay-context-introspection.nix @@ -0,0 +1,22 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + path = "${./eval-okay-context-introspection.nix}"; + + desired-context = { + "${builtins.unsafeDiscardStringContext path}" = { + path = true; + }; + "${builtins.unsafeDiscardStringContext drv.drvPath}" = { + outputs = [ "foo" "out" ]; + allOutputs = true; + }; + }; + + legit-context = "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; +in builtins.getContext legit-context == desired-context From 5f1891b795c4c4daac5336d1ec60c94f521ede1d Mon Sep 17 00:00:00 2001 From: Spencer Baugh Date: Wed, 12 Dec 2018 14:19:54 -0500 Subject: [PATCH 03/45] nix-store: make --dump-db take a list of paths to dump Inside a derivation, exportReferencesGraph already provides a way to dump the Nix database for a specific closure. On the command line, --dump-db gave us the same information, but only for the entire Nix database at once. With this change, one can now pass a list of paths to --dump-db to get the Nix database dumped for just those paths. (The user is responsible for ensuring this is a closure, like for --export). Among other things, this is useful for deploying a closure to a new host without using --import/--export; one can use tar to transfer the store paths, and --dump-db/--load-db to transfer the validity information. This is useful if the new host doesn't actually have Nix yet, and the closure that is being deployed itself contains Nix. --- doc/manual/command-ref/nix-store.xml | 8 ++++++++ src/nix-store/nix-store.cc | 15 ++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 41a04f265..d73cb92ee 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -1282,6 +1282,7 @@ ktorrent-2.2.1/NEWS nix-store + paths @@ -1292,6 +1293,13 @@ Nix database to standard output. It can be loaded into an empty Nix store using . This is useful for making backups and when migrating to different database schemas. +By default, will dump the entire Nix +database. When one or more store paths is passed, only the subset of +the Nix database for those store paths is dumped. As with +, the user is responsible for passing all the +store paths for a closure. See for an +example. + diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index a9ad14762..33138baff 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -485,11 +485,16 @@ static void opReadLog(Strings opFlags, Strings opArgs) static void opDumpDB(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - PathSet validPaths = store->queryAllValidPaths(); - for (auto & i : validPaths) - cout << store->makeValidityRegistration({i}, true, true); + if (!opArgs.empty()) { + for (auto & i : opArgs) + i = store->followLinksToStorePath(i); + for (auto & i : opArgs) + cout << store->makeValidityRegistration({i}, true, true); + } else { + PathSet validPaths = store->queryAllValidPaths(); + for (auto & i : validPaths) + cout << store->makeValidityRegistration({i}, true, true); + } } From b30be6b450f872f8be6dc8afa28f4b030fa8d1d1 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 14 Jan 2019 11:34:54 -0500 Subject: [PATCH 04/45] Add builtins.appendContext. A partner of builtins.getContext, useful for the same reasons. --- src/libexpr/primops/context.cc | 59 ++++++++++++++++++- .../lang/eval-okay-context-introspection.nix | 6 +- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 6849aa261..2d79739ea 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "derivations.hh" namespace nix { @@ -61,8 +62,7 @@ static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscar (i.e. the kind of context you get when referencing .outPath of some derivation). Empty list if missing. Note that for a given path any combination of the above attributes - may be present, but at least one must be set to something other - than the default. + may be present. */ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -129,4 +129,59 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, static RegisterPrimOp r4("__getContext", 1, prim_getContext); + +/* Append the given context to a given string. + + See the commentary above unsafeGetContext for details of the + context representation. +*/ +static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + auto orig = state.forceString(*args[0], context, pos); + + state.forceAttrs(*args[1], pos); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (auto & i : *args[1]->attrs) { + if (!state.store->isStorePath(i.name)) + throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); + if (!settings.readOnlyMode) + state.store->ensurePath(i.name); + state.forceAttrs(*i.value, *i.pos); + auto iter = i.value->attrs->find(sPath); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) + context.insert(i.name); + } + + iter = i.value->attrs->find(sAllOutputs); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) { + if (!isDerivation(i.name)) { + throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + context.insert("=" + string(i.name)); + } + } + + iter = i.value->attrs->find(state.sOutputs); + if (iter != i.value->attrs->end()) { + state.forceList(*iter->value, *iter->pos); + if (iter->value->listSize() && !isDerivation(i.name)) { + throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + for (unsigned int n = 0; n < iter->value->listSize(); ++n) { + auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + context.insert("!" + name + "!" + string(i.name)); + } + } + } + + mkString(v, orig, context); +} + +static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); + } diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/lang/eval-okay-context-introspection.nix index d9b2ea354..43178bd2e 100644 --- a/tests/lang/eval-okay-context-introspection.nix +++ b/tests/lang/eval-okay-context-introspection.nix @@ -18,5 +18,7 @@ let }; }; - legit-context = "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; -in builtins.getContext legit-context == desired-context + legit-context = builtins.getContext "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; + + constructed-context = builtins.getContext (builtins.appendContext "" desired-context); +in legit-context == constructed-context From 8ac1130cc2f0d717fa49adf18465ecf5613d295c Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 9 Feb 2019 20:32:12 +0100 Subject: [PATCH 05/45] nix-daemon: add variable to disable fork safety Since macOS 10.14 this has become an error, causing problems if the nix-daemon loads nix during substitution (this is a forked process). Workaround for #2523. --- misc/launchd/org.nixos.nix-daemon.plist.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 549619a57..92ed12fa5 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -2,6 +2,11 @@ + EnvironmentVariables + + OBJC_DISABLE_INITIALIZE_FORK_SAFETY + YES + Label org.nixos.nix-daemon KeepAlive From 7ce1fae59f6cb4341f61a48a5216cb252e4eca54 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 13 Feb 2019 00:03:10 -0500 Subject: [PATCH 06/45] Support --disable-shared flag. This tells Nix to not build the shared libraries. --- Makefile.config.in | 2 ++ configure.ac | 10 ++++++++++ mk/libraries.mk | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Makefile.config.in b/Makefile.config.in index b01a4afbf..59730b646 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,4 +1,6 @@ +AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ +BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ diff --git a/configure.ac b/configure.ac index 5a2526672..410b20972 100644 --- a/configure.ac +++ b/configure.ac @@ -64,6 +64,7 @@ AC_PROG_CXX AC_PROG_CPP AX_CXX_COMPILE_STDCXX_14 +AC_CHECK_TOOL([AR], [ar]) # Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE @@ -267,6 +268,15 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], sandbox_shell=$withval) AC_SUBST(sandbox_shell) +AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], + [Build shared libraries for Nix [default=yes]]), + shared=$enableval, shared=yes) +if test "$shared" = yes; then + AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) +else + AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) +fi + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix diff --git a/mk/libraries.mk b/mk/libraries.mk index 14c95fa91..3953446cb 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -125,7 +125,7 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - $(trace-ar) ar crs $$@ $$? + $(trace-ar) $(AR) crs $$@ $$? $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) From e58a71442ad4a538b48fc7a9938c3690628c4741 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 14 Feb 2019 13:24:16 -0500 Subject: [PATCH 07/45] nix.sh: Be set -u compliant. --- scripts/nix-profile.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index a5f52274f..db03e16ba 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -75,7 +75,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt" fi - if [ -n "${MANPATH}" ]; then + if [ -n "${MANPATH-}" ]; then export MANPATH="$NIX_LINK/share/man:$MANPATH" fi From b402148d8f584144b80fe159e76e553a1792ecb7 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 22 Feb 2019 21:07:48 +0100 Subject: [PATCH 08/45] remove noop uses of nix-store --init the nix-store --init command is a noop apparently --- release.nix | 1 - scripts/install-multi-user.sh | 3 --- scripts/install-nix-from-closure.sh | 6 ------ tests/common.sh.in | 1 - tests/init.sh | 3 --- 5 files changed, 14 deletions(-) diff --git a/release.nix b/release.nix index 271645067..5c0027301 100644 --- a/release.nix +++ b/release.nix @@ -278,7 +278,6 @@ let pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; } '' export NIX_STATE_DIR=$TMPDIR - nix-store --init nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \ --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }' diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 6ee8dd485..f93be9d3a 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -674,9 +674,6 @@ $NIX_INSTALLED_NIX. EOF fi - _sudo "to initialize the Nix Database" \ - $NIX_INSTALLED_NIX/bin/nix-store --init - cat ./.reginfo \ | _sudo "to load data for the first time in to the Nix Database" \ "$NIX_INSTALLED_NIX/bin/nix-store" --load-db diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index ab20774bb..fc633fa23 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -109,12 +109,6 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do done echo "" >&2 -echo "initialising Nix database..." >&2 -if ! $nix/bin/nix-store --init; then - echo "$0: failed to initialize the Nix database" >&2 - exit 1 -fi - if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then echo "$0: unable to register valid paths" >&2 exit 1 diff --git a/tests/common.sh.in b/tests/common.sh.in index 6a523ca9d..28d0e8bfd 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -53,7 +53,6 @@ clearStore() { mkdir "$NIX_STORE_DIR" rm -rf "$NIX_STATE_DIR" mkdir "$NIX_STATE_DIR" - nix-store --init clearProfiles } diff --git a/tests/init.sh b/tests/init.sh index 19a12c1e2..6fa4f815b 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -25,9 +25,6 @@ fsync-metadata = false !include nix.conf.extra.not-there EOF -# Initialise the database. -nix-store --init - # Did anything happen? test -e "$NIX_STATE_DIR"/db/db.sqlite From d7a7a029ff9c1fc6c5a22b91718e28c4c1062d9f Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Fri, 22 Feb 2019 13:07:35 -0600 Subject: [PATCH 09/45] scripts: remove default 'sandbox = false' from multi-user installer Sandboxing is now enabled by default on Linux, but is still disabled on macOS. However, the installer always turned it off to ensure consistent behavior. Remove this default configuration, so we fall back to the default platform-specific value. Signed-off-by: Austin Seipp --- scripts/install-multi-user.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 6ee8dd485..4c793247e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -747,7 +747,6 @@ build-users-group = $NIX_BUILD_GROUP_NAME max-jobs = $NIX_USER_COUNT cores = 1 -sandbox = false EOF _sudo "to place the default nix daemon configuration (part 2)" \ install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf From 07f992a74b64f4376d5b415d0042babc924772f3 Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Thu, 21 Feb 2019 11:44:25 +0100 Subject: [PATCH 10/45] Extract and expose splitUriAndParams function which splits a URL into localtor and parameter parts --- src/libstore/store-api.cc | 16 ++++++++++++---- src/libstore/store-api.hh | 4 ++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 913a11121..c13ff1156 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -842,12 +842,11 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - -ref openStore(const std::string & uri_, - const Store::Params & extraParams) +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); - Store::Params params(extraParams); + Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString(uri.substr(q + 1), "&")) { @@ -873,6 +872,15 @@ ref openStore(const std::string & uri_, } uri = uri_.substr(0, q); } + return {uri, params}; +} + +ref openStore(const std::string & uri_, + const Store::Params & extraParams) +{ + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f504735e0..ad0f8df11 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -798,4 +798,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri); + } From 56c18c67d98078dbed1d05ac68663cc52d2cb543 Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Mon, 25 Feb 2019 17:56:52 +0100 Subject: [PATCH 11/45] Extend S3 URL parsing with parameters extraction --- src/libstore/download.cc | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 467f570bb..8bc496515 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader writeFull(wakeupPipe.writeSide.get(), " "); } +#ifdef ENABLE_S3 + std::tuple parseS3Uri(std::string uri) + { + auto [path, params] = splitUriAndParams(uri); + + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); + + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); + + return {bucketName, key, params}; + } +#endif + void enqueueDownload(const DownloadRequest & request, Callback callback) override { @@ -622,12 +638,8 @@ struct CurlDownloader : public Downloader // FIXME: do this on a worker thread try { #ifdef ENABLE_S3 + auto [bucketName, key, params] = parseS3Uri(request.uri); S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable - auto slash = request.uri.find('/', 5); - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", request.uri); - std::string bucketName(request.uri, 5, slash - 5); - std::string key(request.uri, slash + 1); // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); DownloadResult res; From ac200c3678c0f569cd962d8bbd22bb74b223d748 Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Thu, 21 Feb 2019 14:26:50 +0100 Subject: [PATCH 12/45] Apply param values on S3Helper initialization --- src/libstore/download.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 8bc496515..f1666d293 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -639,7 +639,14 @@ struct CurlDownloader : public Downloader try { #ifdef ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); - S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable + + std::string profile = get(params, "profile", ""); + std::string region = get(params, "region", Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme", ""); + std::string endpoint = get(params, "endpoint", ""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); DownloadResult res; From 06d633598727763c54b4b049dbc213106474d10c Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Tue, 26 Feb 2019 11:04:18 +0100 Subject: [PATCH 13/45] fix indentation --- src/libstore/download.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index f1666d293..22382ab1d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -620,8 +620,8 @@ struct CurlDownloader : public Downloader auto [path, params] = splitUriAndParams(uri); auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", path); + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); std::string bucketName(path, 5, slash - 5); std::string key(path, slash + 1); From 0963479741d476393260b831a242c74c918ed6e6 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Wed, 4 Apr 2018 13:48:50 -0500 Subject: [PATCH 14/45] archive.cc: ignore more posix_fallocate "not supported" error codes Fixes w/musl. --- src/libutil/archive.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index bb68e8288..3aa120270 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -331,7 +331,7 @@ struct RestoreSink : ParseSink filesystem doesn't support preallocation (e.g. on OpenSolaris). Since preallocation is just an optimisation, ignore it. */ - if (errno && errno != EINVAL) + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) throw SysError(format("preallocating file of %1% bytes") % len); } #endif From 4cfc131ec43ccbfa2f87efc3e96bdbe127ab3b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= Date: Fri, 1 Mar 2019 14:30:30 +0100 Subject: [PATCH 15/45] manual nitpick: document sha512 support in hashString --- doc/manual/expressions/builtins.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 412622714..22d998bd0 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -693,8 +693,8 @@ builtins.genList (x: x * x) 5 Return a base-16 representation of the cryptographic hash of string s. The hash algorithm specified by type must - be one of "md5", "sha1" or - "sha256". + be one of "md5", "sha1", + "sha256" or "sha512". From b71e1fb342b2344b738733698b20c5ec03dabe5a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2019 18:27:45 +0800 Subject: [PATCH 16/45] Restore --init calls in tests --- tests/common.sh.in | 1 + tests/init.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tests/common.sh.in b/tests/common.sh.in index 28d0e8bfd..6a523ca9d 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -53,6 +53,7 @@ clearStore() { mkdir "$NIX_STORE_DIR" rm -rf "$NIX_STATE_DIR" mkdir "$NIX_STATE_DIR" + nix-store --init clearProfiles } diff --git a/tests/init.sh b/tests/init.sh index 6fa4f815b..19a12c1e2 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -25,6 +25,9 @@ fsync-metadata = false !include nix.conf.extra.not-there EOF +# Initialise the database. +nix-store --init + # Did anything happen? test -e "$NIX_STATE_DIR"/db/db.sqlite From fff8db205c393c53e42326f274b322eb70259250 Mon Sep 17 00:00:00 2001 From: volth Date: Wed, 6 Mar 2019 05:03:25 +0000 Subject: [PATCH 17/45] canBuildLocally: check for features It could happen that the local builder match the system but lacks some features. Now it results a failure. The fix gracefully excludes the local builder from the set of available builders for derivation which requires the feature, so the derivation is built on remote builders only (as though it has incompatible system, like ```aarch64-linux``` when local is x86) --- src/build-remote/build-remote.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index abf3669b5..279ae62f6 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -38,6 +38,12 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } +static bool allSupportedLocally(const std::set& requiredFeatures) { + for (auto & feature : requiredFeatures) + if (!settings.systemFeatures.get().count(feature)) return false; + return true; +} + static int _main(int argc, char * * argv) { { @@ -97,9 +103,10 @@ static int _main(int argc, char * * argv) source >> drvPath; auto requiredFeatures = readStrings>(source); - auto canBuildLocally = amWilling - && ( neededSystem == settings.thisSystem - || settings.extraPlatforms.get().count(neededSystem) > 0); + auto canBuildLocally = amWilling + && ( neededSystem == settings.thisSystem + || settings.extraPlatforms.get().count(neededSystem) > 0) + && allSupportedLocally(requiredFeatures); /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); From 43331d634498154f6a1835c8c417dd07eaefd45d Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Wed, 27 Feb 2019 23:32:12 +0100 Subject: [PATCH 18/45] detail memory roots --- src/libstore/gc.cc | 61 +++++++++++++++++++------------------ src/libstore/local-store.hh | 2 +- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index b415d5421..ecfa5e1ed 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -333,9 +333,7 @@ Roots LocalStore::findRootsNoTemp() NIX_ROOT_FINDER environment variable. This is typically used to add running programs to the set of roots (to prevent them from being garbage collected). */ - size_t n = 0; - for (auto & root : findRuntimeRoots()) - roots[fmt("{memory:%d}", n++)] = root; + findRuntimeRoots(roots); return roots; } @@ -357,8 +355,7 @@ Roots LocalStore::findRoots() return roots; } - -static void readProcLink(const string & file, StringSet & paths) +static void readProcLink(const string & file, Roots & roots) { /* 64 is the starting buffer size gnu readlink uses... */ auto bufsiz = ssize_t{64}; @@ -377,7 +374,7 @@ try_again: goto try_again; } if (res > 0 && buf[0] == '/') - paths.emplace(static_cast(buf), res); + roots.emplace(file, std::string(static_cast(buf), res)); return; } @@ -387,20 +384,20 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } -static void readFileRoots(const char * path, StringSet & paths) +static void readFileRoots(const char * path, Roots & roots) { try { - paths.emplace(readFile(path)); + roots.emplace(path, readFile(path)); } catch (SysError & e) { if (e.errNo != ENOENT && e.errNo != EACCES) throw; } } -PathSet LocalStore::findRuntimeRoots() +void LocalStore::findRuntimeRoots(Roots & roots) { - PathSet roots; - StringSet paths; + Roots unchecked; + auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { struct dirent * ent; @@ -410,8 +407,8 @@ PathSet LocalStore::findRuntimeRoots() while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths); - readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths); + readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), unchecked); + readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), unchecked); auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); @@ -423,7 +420,7 @@ PathSet LocalStore::findRuntimeRoots() struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { if (fd_ent->d_name[0] != '.') { - readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths); + readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), unchecked); } } if (errno) { @@ -434,18 +431,22 @@ PathSet LocalStore::findRuntimeRoots() fdDir.reset(); try { - auto mapLines = - tokenizeString>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n"); + auto mapFile = (format("/proc/%1%/maps") % ent->d_name).str(); + auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); + int n = 0; for (const auto& line : mapLines) { + n++; auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - paths.emplace(match[1]); + unchecked.emplace((format("{%1%:%2%}") % mapFile % n).str(), match[1]); } - auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true); + auto envFile = (format("/proc/%1%/environ") % ent->d_name).str(); + auto envString = readFile(envFile, true); auto env_end = std::sregex_iterator{}; + n = 0; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - paths.emplace(i->str()); + unchecked.emplace((format("{%1%:%2%}") % envFile % envString).str(), i->str()); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -462,10 +463,11 @@ PathSet LocalStore::findRuntimeRoots() std::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + int n = 0; for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - paths.emplace(match[1]); + unchecked.emplace((format("{%1%:%2%}" % LSOF % n++).str(), match[1]); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ @@ -473,21 +475,20 @@ PathSet LocalStore::findRuntimeRoots() #endif #if defined(__linux__) - readFileRoots("/proc/sys/kernel/modprobe", paths); - readFileRoots("/proc/sys/kernel/fbsplash", paths); - readFileRoots("/proc/sys/kernel/poweroff_cmd", paths); + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); #endif - for (auto & i : paths) - if (isInStore(i)) { - Path path = toStorePath(i); - if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) { + for (auto & root : unchecked) { + if (isInStore(root.second)) { + Path path = toStorePath(root.second); + if (isStorePath(path) && isValidPath(path)) { debug(format("got additional root '%1%'") % path); - roots.insert(path); + roots.emplace(root.first, path); } } - - return roots; + } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fce963433..730d4c917 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -269,7 +269,7 @@ private: Roots findRootsNoTemp(); - PathSet findRuntimeRoots(); + void findRuntimeRoots(Roots & roots); void removeUnusedLinks(const GCState & state); From a17f86ce3a67dd2dab2329d7262bc4ad4e7c37ff Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Thu, 28 Feb 2019 23:26:07 +0100 Subject: [PATCH 19/45] Obfuscate memory roots for non-root users --- src/libstore/gc.cc | 14 +++++++------- src/nix-daemon/nix-daemon.cc | 11 +++++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ecfa5e1ed..73630f36d 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -374,7 +374,8 @@ try_again: goto try_again; } if (res > 0 && buf[0] == '/') - roots.emplace(file, std::string(static_cast(buf), res)); + roots.emplace((format("{memory:%1%") % file).str(), + std::string(static_cast(buf), res)); return; } @@ -407,8 +408,8 @@ void LocalStore::findRuntimeRoots(Roots & roots) while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), unchecked); - readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), unchecked); + readProcLink((format("{memory:/proc/%1%/exe}") % ent->d_name).str(), unchecked); + readProcLink((format("{memory:/proc/%1%/cwd}") % ent->d_name).str(), unchecked); auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); @@ -435,10 +436,9 @@ void LocalStore::findRuntimeRoots(Roots & roots) auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); int n = 0; for (const auto& line : mapLines) { - n++; auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - unchecked.emplace((format("{%1%:%2%}") % mapFile % n).str(), match[1]); + unchecked.emplace((format("{memory:%1%:%2%}") % mapFile % n++).str(), match[1]); } auto envFile = (format("/proc/%1%/environ") % ent->d_name).str(); @@ -446,7 +446,7 @@ void LocalStore::findRuntimeRoots(Roots & roots) auto env_end = std::sregex_iterator{}; n = 0; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - unchecked.emplace((format("{%1%:%2%}") % envFile % envString).str(), i->str()); + unchecked.emplace((format("{memory:%1%:%2%}") % envFile % n++).str(), i->str()); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -467,7 +467,7 @@ void LocalStore::findRuntimeRoots(Roots & roots) for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - unchecked.emplace((format("{%1%:%2%}" % LSOF % n++).str(), match[1]); + unchecked.emplace((format("{memory:%1%:%2%}" % LSOF % n++).str(), match[1]); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8368c3266..faa23b268 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -478,8 +478,15 @@ static void performOp(TunnelLogger * logger, ref store, Roots roots = store->findRoots(); logger->stopWork(); to << roots.size(); - for (auto & i : roots) - to << i.first << i.second; + int n = 0; + for (auto & i : roots) { + // Obfuscate 'memory' roots as they exposes information about other users, + if (i.first.rfind("{memory:", 0) == 0) { + to << fmt("{memory:%d}", n++) << i.second; + } else { + to << i.first << i.second; + } + } break; } From ebc86550f92ec76cda0961ecc625944ec402d2cd Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 1 Mar 2019 00:54:52 +0100 Subject: [PATCH 20/45] Make roots a map of store paths to pinning links This new structure makes more sense as there may be many sources rooting the same store path. Many profiles can reference the same path but this is even more true with /proc//maps where distinct pids can and often do map the same store path. This implementation is also more efficient as the `Roots` map contains only one entry per rooted store path. --- src/libstore/gc.cc | 39 +++++++++++++++++------------------- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.hh | 2 +- src/nix-daemon/nix-daemon.cc | 19 +++++++++++------- src/nix-store/nix-store.cc | 12 ++++++----- 5 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 73630f36d..da6799f6f 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -130,7 +130,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, gcroots directory. */ if (settings.checkRootReachability) { Roots roots = findRoots(); - if (roots.find(gcRoot) == roots.end()) + if (roots[storePath].count(gcRoot) == 0) printError( format( "warning: '%1%' is not in a directory where the garbage collector looks for roots; " @@ -266,7 +266,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) auto foundRoot = [&](const Path & path, const Path & target) { Path storePath = toStorePath(target); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); else printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); }; @@ -306,7 +306,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) else if (type == DT_REG) { Path storePath = storeDir + "/" + baseNameOf(path); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); } } @@ -346,10 +346,10 @@ Roots LocalStore::findRoots() FDs fds; pid_t prev = -1; size_t n = 0; - for (auto & root : readTempRoots(fds)) { - if (prev != root.first) n = 0; - prev = root.first; - roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second; + for (auto & [pid, root] : readTempRoots(fds)) { + if (prev != pid) n = 0; + prev = pid; + roots[root].emplace(fmt("{temp:%d:%d}", pid, n++)); } return roots; @@ -374,8 +374,8 @@ try_again: goto try_again; } if (res > 0 && buf[0] == '/') - roots.emplace((format("{memory:%1%") % file).str(), - std::string(static_cast(buf), res)); + roots[std::string(static_cast(buf), res)] + .emplace((format("{memory:%1%") % file).str()); return; } @@ -388,7 +388,7 @@ static string quoteRegexChars(const string & raw) static void readFileRoots(const char * path, Roots & roots) { try { - roots.emplace(path, readFile(path)); + roots[readFile(path)].emplace(path); } catch (SysError & e) { if (e.errNo != ENOENT && e.errNo != EACCES) throw; @@ -434,19 +434,17 @@ void LocalStore::findRuntimeRoots(Roots & roots) try { auto mapFile = (format("/proc/%1%/maps") % ent->d_name).str(); auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); - int n = 0; for (const auto& line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - unchecked.emplace((format("{memory:%1%:%2%}") % mapFile % n++).str(), match[1]); + unchecked[match[1]].emplace((format("{memory:%1%}") % mapFile).str()); } auto envFile = (format("/proc/%1%/environ") % ent->d_name).str(); auto envString = readFile(envFile, true); auto env_end = std::sregex_iterator{}; - n = 0; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - unchecked.emplace((format("{memory:%1%:%2%}") % envFile % n++).str(), i->str()); + unchecked[i->str()].emplace((format("{memory:%1%}") % envFile).str()); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -463,11 +461,10 @@ void LocalStore::findRuntimeRoots(Roots & roots) std::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); - int n = 0; for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - unchecked.emplace((format("{memory:%1%:%2%}" % LSOF % n++).str(), match[1]); + unchecked[match[1]].emplace((format("{memory:%1%}" % LSOF).str()); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ @@ -480,12 +477,12 @@ void LocalStore::findRuntimeRoots(Roots & roots) readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); #endif - for (auto & root : unchecked) { - if (isInStore(root.second)) { - Path path = toStorePath(root.second); + for (auto & [target, links] : unchecked) { + if (isInStore(target)) { + Path path = toStorePath(target); if (isStorePath(path) && isValidPath(path)) { debug(format("got additional root '%1%'") % path); - roots.emplace(root.first, path); + roots[path].insert(links.begin(), links.end()); } } } @@ -757,7 +754,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printError(format("finding garbage collector roots...")); Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp(); - for (auto & i : rootMap) state.roots.insert(i.second); + for (auto & i : rootMap) state.roots.insert(i.first); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index def140cfb..a4dd28af0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -606,7 +606,7 @@ Roots RemoteStore::findRoots() while (count--) { Path link = readString(conn->from); Path target = readStorePath(*this, conn->from); - result[link] = target; + result[target].emplace(link); } return result; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ad0f8df11..b4e5f5511 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -47,7 +47,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const uint32_t exportMagic = 0x4558494e; -typedef std::map Roots; +typedef std::map> Roots; struct GCOptions diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index faa23b268..014378d27 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -477,14 +477,19 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); Roots roots = store->findRoots(); logger->stopWork(); - to << roots.size(); + size_t total_length = 0; + for (auto & root : roots) + total_length += root.second.size(); + to << total_length; int n = 0; - for (auto & i : roots) { - // Obfuscate 'memory' roots as they exposes information about other users, - if (i.first.rfind("{memory:", 0) == 0) { - to << fmt("{memory:%d}", n++) << i.second; - } else { - to << i.first << i.second; + for (auto & [target, links] : roots) { + for (auto & link : links) { + // Obfuscate 'memory' roots as they expose information about other users, + if (link.rfind("{memory:", 0) == 0) { + to << fmt("{memory:%d}", n++) << target; + } else { + to << link << target; + } } } break; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 33138baff..b281ea2dd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -428,9 +428,10 @@ static void opQuery(Strings opFlags, Strings opArgs) referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } Roots roots = store->findRoots(); - for (auto & i : roots) - if (referrers.find(i.second) != referrers.end()) - cout << format("%1%\n") % i.first; + for (auto & [path, roots] : roots) + if (referrers.find(path) != referrers.end()) + for (auto & root : roots) + cout << format("%1% -> %2%\n") % root % path; break; } @@ -591,8 +592,9 @@ static void opGC(Strings opFlags, Strings opArgs) if (printRoots) { Roots roots = store->findRoots(); - for (auto & i : roots) - cout << i.first << " -> " << i.second << std::endl; + for (auto & [path, roots] : roots) + for (auto & root : roots) + cout << root << " -> " << path << std::endl; } else { From 8574b703424b73f6f75fc549b1738a6534d6a01e Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 1 Mar 2019 01:15:10 +0100 Subject: [PATCH 21/45] fixup! Make roots a map of store paths to pinning links --- src/nix-store/nix-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b281ea2dd..89831eb3d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -428,10 +428,10 @@ static void opQuery(Strings opFlags, Strings opArgs) referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } Roots roots = store->findRoots(); - for (auto & [path, roots] : roots) + for (auto & [path, links] : roots) if (referrers.find(path) != referrers.end()) - for (auto & root : roots) - cout << format("%1% -> %2%\n") % root % path; + for (auto & link : links) + cout << format("%1%\n") % link; break; } From fc02b1b3eee7632049040a96b056d769a088b2ea Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 1 Mar 2019 01:16:25 +0100 Subject: [PATCH 22/45] Also print rooted path in `nix-store -q --roots` --- src/nix-store/nix-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 89831eb3d..11d0d573f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -431,7 +431,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & [path, links] : roots) if (referrers.find(path) != referrers.end()) for (auto & link : links) - cout << format("%1%\n") % link; + cout << format("%1% -> %2%\n") % link % path; break; } From 5c56570726bc3c546b5cd24da96ceed23b9fdc90 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 1 Mar 2019 01:29:37 +0100 Subject: [PATCH 23/45] Also obfuscate the number of memory roots --- src/nix-daemon/nix-daemon.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 014378d27..63b5eb87d 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -478,15 +478,29 @@ static void performOp(TunnelLogger * logger, ref store, Roots roots = store->findRoots(); logger->stopWork(); size_t total_length = 0; - for (auto & root : roots) - total_length += root.second.size(); + bool hasMemoryLink; + for (auto & [target, links] : roots) { + hasMemoryLink = false; + for (auto & link : links) { + if (link.rfind("{memory:", 0) == 0) { + if (hasMemoryLink) continue; + ++total_length; + hasMemoryLink = true; + } else { + ++total_length; + } + } + } to << total_length; int n = 0; for (auto & [target, links] : roots) { + bool hasMemoryLink = false; for (auto & link : links) { // Obfuscate 'memory' roots as they expose information about other users, if (link.rfind("{memory:", 0) == 0) { + if (hasMemoryLink) continue; to << fmt("{memory:%d}", n++) << target; + hasMemoryLink = true; } else { to << link << target; } From 9d87e3fbd2c690f03eadbcada7ff8b880033155c Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sat, 9 Mar 2019 23:55:15 +0100 Subject: [PATCH 24/45] require c++ 17 This fixes warnings about "structured binding declaration" requiring "-std=c++1z". --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 410b20972..4eea008b0 100644 --- a/configure.ac +++ b/configure.ac @@ -62,7 +62,7 @@ CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_14 +AX_CXX_COMPILE_STDCXX_17 AC_CHECK_TOOL([AR], [ar]) From 9d7221183a6825d798267288ebe7d3e2adcd8cde Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sun, 10 Mar 2019 00:04:37 +0100 Subject: [PATCH 25/45] unify naming of roots as links & targets --- src/nix-store/nix-store.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 11d0d573f..b5530b39e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -428,10 +428,10 @@ static void opQuery(Strings opFlags, Strings opArgs) referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } Roots roots = store->findRoots(); - for (auto & [path, links] : roots) - if (referrers.find(path) != referrers.end()) + for (auto & [target, links] : roots) + if (referrers.find(target) != referrers.end()) for (auto & link : links) - cout << format("%1% -> %2%\n") % link % path; + cout << format("%1% -> %2%\n") % link % target; break; } @@ -592,9 +592,9 @@ static void opGC(Strings opFlags, Strings opArgs) if (printRoots) { Roots roots = store->findRoots(); - for (auto & [path, roots] : roots) - for (auto & root : roots) - cout << root << " -> " << path << std::endl; + for (auto & [target, links] : roots) + for (auto & link : links) + cout << link << " -> " << target << std::endl; } else { From 38ee16ae9c277b0512559282c950c48c71e27697 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sun, 10 Mar 2019 00:37:52 +0100 Subject: [PATCH 26/45] Unify internal findRootsXxx() api --- src/libstore/gc.cc | 35 +++++++++++++---------------------- src/libstore/local-store.hh | 4 ++-- 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index da6799f6f..2a3140354 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -197,10 +197,8 @@ void LocalStore::addTempRoot(const Path & path) } -std::set> LocalStore::readTempRoots(FDs & fds) +void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots) { - std::set> tempRoots; - /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : readDirectory(tempRootsDir)) { @@ -250,14 +248,12 @@ std::set> LocalStore::readTempRoots(FDs & fds) Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); assertStorePath(root); - tempRoots.emplace(pid, root); + tempRoots[root].emplace(fmt("{temp:%d}", pid)); pos = end + 1; } fds.push_back(fd); /* keep open */ } - - return tempRoots; } @@ -321,10 +317,8 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } -Roots LocalStore::findRootsNoTemp() +void LocalStore::findRootsNoTemp(Roots & roots) { - Roots roots; - /* Process direct roots in {gcroots,profiles}. */ findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); @@ -334,23 +328,16 @@ Roots LocalStore::findRootsNoTemp() to add running programs to the set of roots (to prevent them from being garbage collected). */ findRuntimeRoots(roots); - - return roots; } Roots LocalStore::findRoots() { - Roots roots = findRootsNoTemp(); + Roots roots; + findRootsNoTemp(roots); FDs fds; - pid_t prev = -1; - size_t n = 0; - for (auto & [pid, root] : readTempRoots(fds)) { - if (prev != pid) n = 0; - prev = pid; - roots[root].emplace(fmt("{temp:%d:%d}", pid, n++)); - } + findTempRoots(fds, roots); return roots; } @@ -752,7 +739,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printError(format("finding garbage collector roots...")); - Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp(); + Roots rootMap; + if (!options.ignoreLiveness) + findRootsNoTemp(rootMap); for (auto & i : rootMap) state.roots.insert(i.first); @@ -760,8 +749,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) per-process temporary root files. So after this point no paths can be added to the set of temporary roots. */ FDs fds; - for (auto & root : readTempRoots(fds)) - state.tempRoots.insert(root.second); + Roots tempRoots; + findTempRoots(fds, tempRoots); + for (auto & root : tempRoots) + state.tempRoots.insert(root.first); state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); /* After this point the set of roots or temporary roots cannot diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 730d4c917..39a34597e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -180,7 +180,7 @@ private: typedef std::shared_ptr FDPtr; typedef list FDs; - std::set> readTempRoots(FDs & fds); + void findTempRoots(FDs & fds, Roots & roots); public: @@ -267,7 +267,7 @@ private: void findRoots(const Path & path, unsigned char type, Roots & roots); - Roots findRootsNoTemp(); + void findRootsNoTemp(Roots & roots); void findRuntimeRoots(Roots & roots); From 4f4391193cca2f7a28d351947e12f954d9432693 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sun, 10 Mar 2019 00:45:55 +0100 Subject: [PATCH 27/45] Fix warning about unused variable --- src/nix-daemon/nix-daemon.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 63b5eb87d..0feafb013 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -477,11 +477,13 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); Roots roots = store->findRoots(); logger->stopWork(); + + // Pre-compute roots length using same algo as below. size_t total_length = 0; bool hasMemoryLink; - for (auto & [target, links] : roots) { + for (auto & root : roots) { hasMemoryLink = false; - for (auto & link : links) { + for (auto & link : root.second) { if (link.rfind("{memory:", 0) == 0) { if (hasMemoryLink) continue; ++total_length; @@ -491,6 +493,7 @@ static void performOp(TunnelLogger * logger, ref store, } } } + to << total_length; int n = 0; for (auto & [target, links] : roots) { From 115e2c8c67a949d3fa78fda450186803cd713958 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sun, 10 Mar 2019 01:50:33 +0100 Subject: [PATCH 28/45] Update tests to the new --roots format --- tests/gc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gc.sh b/tests/gc.sh index 0adb05bf1..8b4f8d282 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -7,7 +7,7 @@ outPath=$(nix-store -rvv "$drvPath") rm -f "$NIX_STATE_DIR"/gcroots/foo ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo -[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR"/gcroots/foo ] +[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] nix-store --gc --print-roots | grep $outPath nix-store --gc --print-live | grep $outPath From a3f37d87eabcfb5dc581abcfa46e5e7d387dfa8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Mar 2019 13:27:16 +0100 Subject: [PATCH 29/45] findRuntimeRoots: Simplify/fix handling of /proc files Scanning of /proc//{exe,cwd} was broken because '{memory:' was prepended twice. Also, get rid of the whole '{memory:...}' thing because it's unnecessary, we can just list the file in /proc directly. --- src/libstore/gc.cc | 26 ++++++++++++-------------- src/libstore/store-api.hh | 3 ++- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 2a3140354..8c3607f66 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -362,8 +362,7 @@ try_again: } if (res > 0 && buf[0] == '/') roots[std::string(static_cast(buf), res)] - .emplace((format("{memory:%1%") % file).str()); - return; + .emplace(file); } static string quoteRegexChars(const string & raw) @@ -395,10 +394,10 @@ void LocalStore::findRuntimeRoots(Roots & roots) while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("{memory:/proc/%1%/exe}") % ent->d_name).str(), unchecked); - readProcLink((format("{memory:/proc/%1%/cwd}") % ent->d_name).str(), unchecked); + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); + auto fdStr = fmt("/proc/%s/fd", ent->d_name); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); if (!fdDir) { if (errno == ENOENT || errno == EACCES) @@ -407,9 +406,8 @@ void LocalStore::findRuntimeRoots(Roots & roots) } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') { - readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), unchecked); - } + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); } if (errno) { if (errno == ESRCH) @@ -419,19 +417,19 @@ void LocalStore::findRuntimeRoots(Roots & roots) fdDir.reset(); try { - auto mapFile = (format("/proc/%1%/maps") % ent->d_name).str(); + auto mapFile = fmt("/proc/%s/maps", ent->d_name); auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); - for (const auto& line : mapLines) { + for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - unchecked[match[1]].emplace((format("{memory:%1%}") % mapFile).str()); + unchecked[match[1]].emplace(mapFile); } - auto envFile = (format("/proc/%1%/environ") % ent->d_name).str(); + auto envFile = fmt("/proc/%s/environ", ent->d_name); auto envString = readFile(envFile, true); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - unchecked[i->str()].emplace((format("{memory:%1%}") % envFile).str()); + unchecked[i->str()].emplace(envFile); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -451,7 +449,7 @@ void LocalStore::findRuntimeRoots(Roots & roots) for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace((format("{memory:%1%}" % LSOF).str()); + unchecked[match[1]].emplace("{lsof}"); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b4e5f5511..bb1665976 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -47,7 +48,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const uint32_t exportMagic = 0x4558494e; -typedef std::map> Roots; +typedef std::unordered_map> Roots; struct GCOptions From 53522cb6ac19bd1da35a657988231cce9387be9c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Mar 2019 13:50:07 +0100 Subject: [PATCH 30/45] findRoots(): Add 'censor' parameter This is less brittle than filtering paths after the fact in nix-daemon. --- src/libstore/gc.cc | 30 +++++++++++++++----------- src/libstore/local-store.hh | 8 +++---- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 6 ++++-- src/nix-daemon/nix-daemon.cc | 41 +++++++++--------------------------- src/nix-store/nix-store.cc | 4 ++-- 7 files changed, 40 insertions(+), 53 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8c3607f66..d8a5da0d4 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -129,7 +129,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = findRoots(); + Roots roots = findRoots(false); if (roots[storePath].count(gcRoot) == 0) printError( format( @@ -197,7 +197,10 @@ void LocalStore::addTempRoot(const Path & path) } -void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots) +static std::string censored = "{censored}"; + + +void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ @@ -248,7 +251,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots) Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); assertStorePath(root); - tempRoots[root].emplace(fmt("{temp:%d}", pid)); + tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } @@ -317,7 +320,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } -void LocalStore::findRootsNoTemp(Roots & roots) +void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { /* Process direct roots in {gcroots,profiles}. */ findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); @@ -327,17 +330,17 @@ void LocalStore::findRootsNoTemp(Roots & roots) NIX_ROOT_FINDER environment variable. This is typically used to add running programs to the set of roots (to prevent them from being garbage collected). */ - findRuntimeRoots(roots); + findRuntimeRoots(roots, censor); } -Roots LocalStore::findRoots() +Roots LocalStore::findRoots(bool censor) { Roots roots; - findRootsNoTemp(roots); + findRootsNoTemp(roots, censor); FDs fds; - findTempRoots(fds, roots); + findTempRoots(fds, roots, censor); return roots; } @@ -381,7 +384,7 @@ static void readFileRoots(const char * path, Roots & roots) } } -void LocalStore::findRuntimeRoots(Roots & roots) +void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { Roots unchecked; @@ -467,7 +470,10 @@ void LocalStore::findRuntimeRoots(Roots & roots) Path path = toStorePath(target); if (isStorePath(path) && isValidPath(path)) { debug(format("got additional root '%1%'") % path); - roots[path].insert(links.begin(), links.end()); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); } } } @@ -739,7 +745,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printError(format("finding garbage collector roots...")); Roots rootMap; if (!options.ignoreLiveness) - findRootsNoTemp(rootMap); + findRootsNoTemp(rootMap, true); for (auto & i : rootMap) state.roots.insert(i.first); @@ -748,7 +754,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) can be added to the set of temporary roots. */ FDs fds; Roots tempRoots; - findTempRoots(fds, tempRoots); + findTempRoots(fds, tempRoots, true); for (auto & root : tempRoots) state.tempRoots.insert(root.first); state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 39a34597e..6b655647b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -180,11 +180,11 @@ private: typedef std::shared_ptr FDPtr; typedef list FDs; - void findTempRoots(FDs & fds, Roots & roots); + void findTempRoots(FDs & fds, Roots & roots, bool censor); public: - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -267,9 +267,9 @@ private: void findRoots(const Path & path, unsigned char type, Roots & roots); - void findRootsNoTemp(Roots & roots); + void findRootsNoTemp(Roots & roots, bool censor); - void findRuntimeRoots(Roots & roots); + void findRuntimeRoots(Roots & roots, bool censor); void removeUnusedLinks(const GCState & state); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a4dd28af0..15faf78a5 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -596,7 +596,7 @@ void RemoteStore::syncWithGC() } -Roots RemoteStore::findRoots() +Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); conn->to << wopFindRoots; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4f554b598..e5ab79e59 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -82,7 +82,7 @@ public: void syncWithGC() override; - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index bb1665976..40887b6aa 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -484,8 +484,10 @@ public: /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. */ - virtual Roots findRoots() + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) { unsupported("findRoots"); } /* Perform a garbage collection. */ diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 0feafb013..dd8b9bd1e 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -475,40 +475,19 @@ static void performOp(TunnelLogger * logger, ref store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(); + Roots roots = store->findRoots(true); logger->stopWork(); - // Pre-compute roots length using same algo as below. - size_t total_length = 0; - bool hasMemoryLink; - for (auto & root : roots) { - hasMemoryLink = false; - for (auto & link : root.second) { - if (link.rfind("{memory:", 0) == 0) { - if (hasMemoryLink) continue; - ++total_length; - hasMemoryLink = true; - } else { - ++total_length; - } - } - } + size_t size = 0; + for (auto & i : roots) + size += i.second.size(); + + to << size; + + for (auto & [target, links] : roots) + for (auto & link : links) + to << link << target; - to << total_length; - int n = 0; - for (auto & [target, links] : roots) { - bool hasMemoryLink = false; - for (auto & link : links) { - // Obfuscate 'memory' roots as they expose information about other users, - if (link.rfind("{memory:", 0) == 0) { - if (hasMemoryLink) continue; - to << fmt("{memory:%d}", n++) << target; - hasMemoryLink = true; - } else { - to << link << target; - } - } - } break; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b5530b39e..6f315e509 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -427,7 +427,7 @@ static void opQuery(Strings opFlags, Strings opArgs) maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } - Roots roots = store->findRoots(); + Roots roots = store->findRoots(false); for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) for (auto & link : links) @@ -591,7 +591,7 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); if (printRoots) { - Roots roots = store->findRoots(); + Roots roots = store->findRoots(false); for (auto & [target, links] : roots) for (auto & link : links) cout << link << " -> " << target << std::endl; From 578ed7a259ddf7a85c85fe4e0200179ae908171e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Mar 2019 13:52:13 +0100 Subject: [PATCH 31/45] findRoots(): Don't censor for trusted users They're pretty much root anyway. --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index dd8b9bd1e..8d63b8f36 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -475,7 +475,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(true); + Roots roots = store->findRoots(!trusted); logger->stopWork(); size_t size = 0; From 86f3b94c8c28defd7402245d60af15ce429986ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Mar 2019 13:59:10 +0100 Subject: [PATCH 32/45] nix-store --gc --print-roots: Sort output --- src/nix-store/nix-store.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6f315e509..f324056bb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -592,9 +592,13 @@ static void opGC(Strings opFlags, Strings opArgs) if (printRoots) { Roots roots = store->findRoots(false); + std::set> roots2; + // Transpose and sort the roots. for (auto & [target, links] : roots) for (auto & link : links) - cout << link << " -> " << target << std::endl; + roots2.emplace(link, target); + for (auto & [link, target] : roots2) + std::cout << link << " -> " << target << "\n"; } else { From ef52ccf035fe0bcd78226891022171a28dd6fcbe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 13:43:32 +0100 Subject: [PATCH 33/45] experimental/optional -> optional --- src/libexpr/eval.hh | 2 +- src/libexpr/get-drvs.cc | 2 +- src/libexpr/get-drvs.hh | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/fetchGit.cc | 4 ++-- src/libstore/build.cc | 10 +++++----- src/libstore/parsed-derivations.cc | 4 ++-- src/libstore/parsed-derivations.hh | 8 ++++---- src/libstore/remote-store.hh | 2 +- src/libutil/lru-cache.hh | 4 ++-- src/libutil/serialise.cc | 2 +- src/libutil/util.cc | 2 +- src/libutil/util.hh | 6 +++--- src/nix/add-to-store.cc | 2 +- 14 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9fe387891..a314e01e0 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -81,7 +81,7 @@ public: /* The allowed filesystem paths in restricted or pure evaluation mode. */ - std::experimental::optional allowedPaths; + std::optional allowedPaths; Value vEmptySet; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index d38ed2df3..21a4d7917 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v, } -std::experimental::optional getDerivation(EvalState & state, Value & v, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { Done done; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index daaa635fe..d7860fc6a 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -78,7 +78,7 @@ typedef list DrvInfos; /* If value `v' denotes a derivation, return a DrvInfo object describing it. Otherwise return nothing. */ -std::experimental::optional getDerivation(EvalState & state, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); void getDerivations(EvalState & state, Value & v, const string & pathPrefix, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0da9f702f..6b0c55e72 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -555,7 +555,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; - std::experimental::optional outputHash; + std::optional outputHash; std::string outputHashAlgo; bool outputHashRecursive = false; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b46d2f258..aaf02c856 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -26,7 +26,7 @@ struct GitInfo std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref store, const std::string & uri, - std::experimental::optional ref, std::string rev, + std::optional ref, std::string rev, const std::string & name) { if (evalSettings.pureEval && rev == "") @@ -190,7 +190,7 @@ GitInfo exportGit(ref store, const std::string & uri, static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::experimental::optional ref; + std::optional ref; std::string rev; std::string name = "source"; PathSet context; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 47ee8b48f..6b88b1307 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2413,7 +2413,7 @@ void DerivationGoal::writeStructuredAttrs() objects consisting entirely of those values. (So nested arrays or objects are not supported.) */ - auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional { + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { if (value.is_string()) return shellEscape(value); @@ -3311,8 +3311,8 @@ void DerivationGoal::checkOutputs(const std::map & outputs) struct Checks { bool ignoreSelfRefs = false; - std::experimental::optional maxSize, maxClosureSize; - std::experimental::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + std::optional maxSize, maxClosureSize; + std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; }; /* Compute the closure and closure size of some output. This @@ -3359,7 +3359,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) info.path, closureSize, *checks.maxClosureSize); } - auto checkRefs = [&](const std::experimental::optional & value, bool allowed, bool recursive) + auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) { if (!value) return; @@ -3413,7 +3413,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) if (maxClosureSize != output->end()) checks.maxClosureSize = maxClosureSize->get(); - auto get = [&](const std::string & name) -> std::experimental::optional { + auto get = [&](const std::string & name) -> std::optional { auto i = output->find(name); if (i != output->end()) { Strings res; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index dc3286482..17fde00a0 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -16,7 +16,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) } } -std::experimental::optional ParsedDerivation::getStringAttr(const std::string & name) const +std::optional ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); @@ -56,7 +56,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const } } -std::experimental::optional ParsedDerivation::getStringsAttr(const std::string & name) const +std::optional ParsedDerivation::getStringsAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 0a82c1461..ed07dc652 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -8,22 +8,22 @@ class ParsedDerivation { Path drvPath; BasicDerivation & drv; - std::experimental::optional structuredAttrs; + std::optional structuredAttrs; public: ParsedDerivation(const Path & drvPath, BasicDerivation & drv); - const std::experimental::optional & getStructuredAttrs() const + const std::optional & getStructuredAttrs() const { return structuredAttrs; } - std::experimental::optional getStringAttr(const std::string & name) const; + std::optional getStringAttr(const std::string & name) const; bool getBoolAttr(const std::string & name, bool def = false) const; - std::experimental::optional getStringsAttr(const std::string & name) const; + std::optional getStringsAttr(const std::string & name) const; StringSet getRequiredSystemFeatures() const; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index e5ab79e59..80f18ab71 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -149,7 +149,7 @@ public: private: ref openConnection() override; - std::experimental::optional path; + std::optional path; }; diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 9b8290e63..8b83f842c 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -2,7 +2,7 @@ #include #include -#include +#include namespace nix { @@ -64,7 +64,7 @@ public: /* Look up an item in the cache. If it exists, it becomes the most recently used item. */ - std::experimental::optional get(const Key & key) + std::optional get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0e75eeec2..8201549fd 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -171,7 +171,7 @@ std::unique_ptr sinkToSource( std::function fun; std::function eof; - std::experimental::optional coro; + std::optional coro; bool started = false; SinkToSource(std::function fun, std::function eof) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 7eca35577..e3dcd246c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -965,7 +965,7 @@ std::vector stringsToCharPtrs(const Strings & ss) string runProgram(Path program, bool searchPath, const Strings & args, - const std::experimental::optional & input) + const std::optional & input) { RunOptions opts(program, args); opts.searchPath = searchPath; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bda87bee4..9f239bff3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #ifndef HAVE_STRUCT_DIRENT_D_TYPE @@ -259,14 +259,14 @@ pid_t startProcess(std::function fun, const ProcessOptions & options = P shell backtick operator). */ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), - const std::experimental::optional & input = {}); + const std::optional & input = {}); struct RunOptions { Path program; bool searchPath = true; Strings args; - std::experimental::optional input; + std::optional input; Source * standardIn = nullptr; Sink * standardOut = nullptr; bool _killStderr = false; diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index d0003790d..e86b96e3f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -8,7 +8,7 @@ using namespace nix; struct CmdAddToStore : MixDryRun, StoreCommand { Path path; - std::experimental::optional namePart; + std::optional namePart; CmdAddToStore() { From 1f64f4c7c83d8efd67051a399c738c71815e0594 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 13:44:20 +0100 Subject: [PATCH 34/45] pkg-config files: Use c++17 --- src/libexpr/nix-expr.pc.in | 2 +- src/libmain/nix-main.pc.in | 2 +- src/libstore/nix-store.pc.in | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 79f3e2f45..80f7a492b 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 38bc85c48..37b03dcd4 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 5cf22faad..6d67b1e03 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 From 34fade478a84d822646a8daa334014134c0ec59e Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Thu, 14 Mar 2019 23:51:18 +0100 Subject: [PATCH 35/45] docs: fix typo in AllowDirectReads It was just missing an `l` but should be fixed anyway. --- doc/manual/packages/s3-substituter.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index 2ec9687a0..e7589ffdb 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -89,7 +89,7 @@ the S3 URL: "Version": "2012-10-17", "Statement": [ { - "Sid": "AlowDirectReads", + "Sid": "AllowDirectReads", "Action": [ "s3:GetObject", "s3:GetBucketLocation" From 81a23fa7e2eb4d58d5edf2a145af5c8bbe4c331e Mon Sep 17 00:00:00 2001 From: Chaker Benhamed Date: Fri, 15 Mar 2019 13:16:20 +0100 Subject: [PATCH 36/45] Add a 5 seconds of timeouts to connect to S3. The default is 1000ms, but we can hit it a lot of we don't have direct link to AWS (e.g. using VPN). --- src/libstore/s3-binary-cache-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 51de89e0d..cd547a964 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -126,6 +126,7 @@ ref S3Helper::makeConfig(const string & region res->endpointOverride = endpoint; } res->requestTimeoutMs = 600 * 1000; + res->connectTimeoutMs = 5 * 1000; res->retryStrategy = std::make_shared(); res->caFile = settings.caFile; return res; From 684c7fff80b575da8055015c6b87693fd1d858ca Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Sat, 16 Mar 2019 19:03:03 +0100 Subject: [PATCH 37/45] Link to `builders-use-substitutes` in chapter on distributed builds fixes #2730. --- doc/manual/advanced-topics/distributed-builds.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index bbb573e35..ce2e077ed 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -180,4 +180,8 @@ builders = @/etc/nix/machines causes the list of machines in /etc/nix/machines to be included. (This is the default.) +If you want the builders to use caches, you likely want to set +the option builders-use-substitutes +in your local nix.conf. + From 6f093073b6ff38fb2c4e22331e73930baa1b30e7 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 21 Mar 2019 09:35:43 +0100 Subject: [PATCH 38/45] manual: include builtins.* for globally available builtins This improves searchability. --- doc/manual/expressions/builtins.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 22d998bd0..cefc8b78e 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -23,6 +23,7 @@ available as builtins.derivation. abort s + builtins.abort s Abort Nix expression evaluation, print error message s. @@ -251,6 +252,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" derivation attrs + builtins.derivation + attrs derivation is described in . @@ -260,6 +263,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" dirOf s + builtins.dirOf s Return the directory part of the string s, that is, everything before the final @@ -318,6 +322,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" fetchTarball url + builtins.fetchTarball + url Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive @@ -714,6 +720,8 @@ builtins.genList (x: x * x) 5 import path + builtins.import + path Load, parse and return the Nix expression in the file path. If path @@ -857,6 +865,8 @@ x: x + 456 isNull e + builtins.isNull + e Return true if e evaluates to null, @@ -925,6 +935,8 @@ builtins.listToAttrs map f list + builtins.map + f list Apply the function f to each element in the list list. For @@ -1119,6 +1131,8 @@ Evaluates to [ "foo" ]. removeAttrs set list + builtins.removeAttrs + set list Remove the attributes listed in list from @@ -1287,6 +1301,8 @@ builtins.substring 0 3 "nixos" throw s + builtins.throw + s Throw an error message s. This usually aborts Nix expression @@ -1405,6 +1421,7 @@ in foo toString e + builtins.toString e Convert the expression e to a string. From 2aa89daab3a8351a03f6d57d3e1f737cccd59850 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 21 Mar 2019 14:32:20 +0100 Subject: [PATCH 39/45] eval: improve type description for primops and applied primops This can make type errors a little easier to understand. --- src/libexpr/eval.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2a194d0e0..211f7a55f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -130,6 +130,16 @@ std::ostream & operator << (std::ostream & str, const Value & v) } +const Value *getPrimOp(const Value &v) { + const Value * primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; +} + + string showType(const Value & v) { switch (v.type) { @@ -144,8 +154,10 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; + case tPrimOp: + return fmt("the built-in function '%s'", string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); case tFloat: return "a float"; } From 514b3c7f8345cfcbbe166981214497ed9d93ae18 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 29 Jan 2018 12:36:59 +0000 Subject: [PATCH 40/45] Add isPath primop this is added for completeness' sake since all the other possible `builtins.typeOf` results have a corresponding `builtins.is` --- doc/manual/expressions/builtins.xml | 8 ++++++++ src/libexpr/primops.cc | 7 +++++++ tests/lang/eval-okay-types.nix | 2 ++ 3 files changed, 17 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index cefc8b78e..0fb5261b3 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -861,6 +861,14 @@ x: x + 456 + builtins.isPath + e + + Return true if + e evaluates to a path, and + false otherwise. + + isNull diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 6b0c55e72..39073725e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -315,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu mkBool(v, args[0]->type == tBool); } +/* Determine whether the argument is a path. */ +static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tPath); +} struct CompareValues { @@ -2169,6 +2175,7 @@ void EvalState::createBaseEnv() addPrimOp("__isInt", 1, prim_isInt); addPrimOp("__isFloat", 1, prim_isFloat); addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); addPrimOp("__genericClosure", 1, prim_genericClosure); addPrimOp("abort", 1, prim_abort); addPrimOp("__addErrorContext", 2, prim_addErrorContext); diff --git a/tests/lang/eval-okay-types.nix b/tests/lang/eval-okay-types.nix index a34775f5e..9b58be5d1 100644 --- a/tests/lang/eval-okay-types.nix +++ b/tests/lang/eval-okay-types.nix @@ -20,6 +20,8 @@ with builtins; (isFloat (1 - 2.0)) (isBool (true && false)) (isBool null) + (isPath /nix/store) + (isPath ./.) (isAttrs { x = 123; }) (isAttrs null) (typeOf (3 * 4)) From 0bebca402afa5f334c6d268d937097efbe1762d5 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Thu, 14 Mar 2019 09:55:30 -0500 Subject: [PATCH 41/45] version -> .version, avoid conflict with C++20 --- version => .version | 0 release.nix | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename version => .version (100%) diff --git a/version b/.version similarity index 100% rename from version rename to .version diff --git a/release.nix b/release.nix index 5c0027301..55808407e 100644 --- a/release.nix +++ b/release.nix @@ -18,7 +18,7 @@ let releaseTools.sourceTarball { name = "nix-tarball"; - version = builtins.readFile ./version; + version = builtins.readFile ./.version; versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"; src = nix; inherit officialRelease; From 42e2d5e7b7214772a0afa57e87de08715994dc29 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Thu, 14 Mar 2019 10:00:00 -0500 Subject: [PATCH 42/45] store-api.hh: add missing include for unordered_map --- src/libstore/store-api.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 40887b6aa..7a1b31d0f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include From 1f9c8cd68b9cd06a7c604ee7a86a83eef676273c Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Thu, 14 Mar 2019 10:21:22 -0500 Subject: [PATCH 43/45] configure.ac: update for new version file location --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 4eea008b0..f5b1614f1 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./version)$VERSION_SUFFIX"])) +AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) From 63e7fc509628874bf60d9e436af387bfb1bc00a5 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Thu, 14 Mar 2019 10:28:10 -0500 Subject: [PATCH 44/45] perl/configure.ac: fix for new version location too --- perl/configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/configure.ac b/perl/configure.ac index 9f49db4d2..966700695 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../version)$VERSION_SUFFIX"])) +AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) AC_CONFIG_SRCDIR(MANIFEST) AC_CONFIG_AUX_DIR(../config) From 75ec68f93ac98956552c06a7fdf4dc8da6841a65 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Mon, 25 Mar 2019 20:55:28 -0400 Subject: [PATCH 45/45] manual: mention the "channel:" shorthand for NIX_PATH Bumped to 15.09 because older channels, when downloaded from nixos.org, require firefox to be accessed via `pkgs.firefox` --- doc/manual/command-ref/env-common.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index 361d3e2b0..c532ffdde 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -52,10 +52,15 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos NIX_PATH to -nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz +nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz tells Nix to download the latest revision in the Nixpkgs/NixOS - 14.12 channel. + 15.09 channel. + + A following shorthand can be used to refer to the official channels: + + nixpkgs=channel:nixos-15.09 + The search path can be extended using the option, which takes precedence over