Merge remote-tracking branch 'upstream/master' into trustless-remote-builder-simple

Makefile

@@ -10,7 +10,6 @@ makefiles = \
   src/nix/local.mk \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
-  corepkgs/local.mk \
   misc/bash/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \

Makefile.config.in

@@ -10,7 +10,6 @@ EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_S3 = @ENABLE_S3@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_SECCOMP = @HAVE_SECCOMP@
-HAVE_SODIUM = @HAVE_SODIUM@
 LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@

README.md

@@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download
 
 ## Building And Developing
 
-See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/hacking.html) in our manual for instruction on how to
+See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to
 build nix from source with nix-build or how to get a development environment.
 
 ## Additional Resources

configure.ac

@@ -174,9 +174,9 @@ PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]
 
 # Look for libbz2, a required dependency.
 AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
 AC_CHECK_HEADERS([bzlib.h], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
+  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
 # Checks for libarchive
 PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed
@@ -203,11 +203,7 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
 ])
 
 # Look for libsodium, an optional dependency.
-PKG_CHECK_MODULES([SODIUM], [libsodium],
+PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
-  [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.])
-   CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"
-   have_sodium=1], [have_sodium=])
-AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
 # Look for liblzma, a required dependency.
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])

corepkgs/local.mk

@@ -1,4 +0,0 @@
-corepkgs_FILES = \
-  fetchurl.nix
-
-$(foreach file,$(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))

doc/manual/generate-manpage.nix

@@ -20,11 +20,6 @@ let
            (attrNames def.commands))
          + "\n"
        else "")
-    + (if def.examples or [] != []
-       then
-         "# Examples\n\n"
-         + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples)
-       else "")
     + (if def ? doc
        then def.doc + "\n\n"
        else "")
@@ -43,7 +38,7 @@ let
         if flag.category or "" != "config"
         then
           "  - `--${longName}`"
-          + (if flag ? shortName then " / `${flag.shortName}`" else "")
+          + (if flag ? shortName then " / `-${flag.shortName}`" else "")
           + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "")
           + "  \n"
           + "    " + flag.description + "\n\n"

doc/manual/src/advanced-topics/post-build-hook.md

@@ -53,7 +53,7 @@ set -f # disable globbing
 export IFS=' '
 
 echo "Signing paths" $OUT_PATHS
-nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
+nix store sign --key-file /etc/nix/key.private $OUT_PATHS
 echo "Uploading paths" $OUT_PATHS
 exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
 ```
@@ -63,7 +63,7 @@ exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
 > The `$OUT_PATHS` variable is a space-separated list of Nix store
 > paths. In this case, we expect and want the shell to perform word
 > splitting to make each output path its own argument to `nix
-> sign-paths`. Nix guarantees the paths will not contain any spaces,
+> store sign`. Nix guarantees the paths will not contain any spaces,
 > however a store path might contain glob characters. The `set -f`
 > disables globbing in the shell.
 

doc/manual/src/command-ref/nix-store.md

@@ -226,7 +226,7 @@ control what gets deleted and in what order:
     or TiB units.
 
 The behaviour of the collector is also influenced by the
-`keep-outputs` and `keep-derivations` variables in the Nix
+`keep-outputs` and `keep-derivations` settings in the Nix
 configuration file.
 
 By default, the collector prints the total number of freed bytes when it

doc/manual/src/installation/prerequisites-source.md

@@ -30,7 +30,7 @@
     have bzip2 installed, including development headers and libraries.
     If your distribution does not provide these, you can obtain bzip2
     from
-    <https://web.archive.org/web/20180624184756/http://www.bzip.org/>.
+    <https://sourceware.org/bzip2/>.
 
   - `liblzma`, which is provided by XZ Utils. If your distribution does
     not provide this, you can get it from <https://tukaani.org/xz/>.

doc/manual/src/introduction.md

@@ -165,10 +165,10 @@ You’re then dropped into a shell where you can edit, build and test
 the package:
 
 ```console
-[nix-shell]$ tar xf $src
+[nix-shell]$ unpackPhase
 [nix-shell]$ cd pan-*
-[nix-shell]$ ./configure
+[nix-shell]$ configurePhase
-[nix-shell]$ make
+[nix-shell]$ buildPhase
 [nix-shell]$ ./pan/gui/pan
 ```
 

perl/Makefile.config.in

@@ -2,7 +2,6 @@ CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
-HAVE_SODIUM = @HAVE_SODIUM@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SODIUM_LIBS = @SODIUM_LIBS@

perl/configure.ac

@@ -40,11 +40,7 @@ AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
 AC_MSG_RESULT($perllibdir)
 
 # Look for libsodium, an optional dependency.
-PKG_CHECK_MODULES([SODIUM], [libsodium],
+PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
-  [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.])
-   CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"
-   have_sodium=1], [have_sodium=])
-AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
 # Check for the required Perl dependencies (DBI and DBD::SQLite).
 perlFlags="-I$perllibdir"

perl/lib/Nix/Store.xs

@@ -14,9 +14,7 @@
 #include "util.hh"
 #include "crypto.hh"
 
-#if HAVE_SODIUM
 #include <sodium.h>
-#endif
 
 
 using namespace nix;
@@ -239,12 +237,8 @@ SV * convertHash(char * algo, char * s, int toBase32)
 SV * signString(char * secretKey_, char * msg)
     PPCODE:
         try {
-#if HAVE_SODIUM
             auto sig = SecretKey(secretKey_).signDetached(msg);
             XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size())));
-#else
-            throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
-#endif
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -253,7 +247,6 @@ SV * signString(char * secretKey_, char * msg)
 int checkSignature(SV * publicKey_, SV * sig_, char * msg)
     CODE:
         try {
-#if HAVE_SODIUM
             STRLEN publicKeyLen;
             unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen);
             if (publicKeyLen != crypto_sign_PUBLICKEYBYTES)
@@ -265,9 +258,6 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
                 throw Error("signature is not valid");
 
             RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0;
-#else
-            throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
-#endif
         } catch (Error & e) {
             croak("%s", e.what());
         }

src/libexpr/attr-path.cc

@@ -52,9 +52,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
     for (auto & attr : tokens) {
 
         /* Is i an index (integer) or a normal attribute name? */
-        enum { apAttr, apIndex } apType = apAttr;
+        auto attrIndex = string2Int<unsigned int>(attr);
-        unsigned int attrIndex;
-        if (string2Int(attr, attrIndex)) apType = apIndex;
 
         /* Evaluate the expression. */
         Value * vNew = state.allocValue();
@@ -65,9 +63,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
         /* It should evaluate to either a set or an expression,
            according to what is specified in the attrPath. */
 
-        if (apType == apAttr) {
+        if (!attrIndex) {
 
-            if (v->type != tAttrs)
+            if (v->type() != nAttrs)
                 throw TypeError(
                     "the expression selected by the selection path '%1%' should be a set but is %2%",
                     attrPath,
@@ -82,17 +80,17 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
             pos = *a->pos;
         }
 
-        else if (apType == apIndex) {
+        else {
 
             if (!v->isList())
                 throw TypeError(
                     "the expression selected by the selection path '%1%' should be a list but is %2%",
                     attrPath,
                     showType(*v));
-            if (attrIndex >= v->listSize())
+            if (*attrIndex >= v->listSize())
-                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
+                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath);
 
-            v = v->listElems()[attrIndex];
+            v = v->listElems()[*attrIndex];
             pos = noPos;
         }
 

src/libexpr/attr-set.cc

@@ -24,9 +24,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
         v = vEmptySet;
         return;
     }
-    clearValue(v);
+    v.mkAttrs(allocBindings(capacity));
-    v.type = tAttrs;
-    v.attrs = allocBindings(capacity);
     nrAttrsets++;
     nrAttrsInAttrsets += capacity;
 }

src/libexpr/common-eval-args.cc

@@ -14,14 +14,14 @@ MixEvalArgs::MixEvalArgs()
 {
     addFlag({
         .longName = "arg",
-        .description = "argument to be passed to Nix functions",
+        .description = "Pass the value *expr* as the argument *name* to Nix functions.",
         .labels = {"name", "expr"},
         .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
     });
 
     addFlag({
         .longName = "argstr",
-        .description = "string-valued argument to be passed to Nix functions",
+        .description = "Pass the string *string* as the argument *name* to Nix functions.",
         .labels = {"name", "string"},
         .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
     });
@@ -29,14 +29,14 @@ MixEvalArgs::MixEvalArgs()
     addFlag({
         .longName = "include",
         .shortName = 'I',
-        .description = "add a path to the list of locations used to look up `<...>` file names",
+        .description = "Add *path* to the list of locations used to look up `<...>` file names.",
         .labels = {"path"},
         .handler = {[&](std::string s) { searchPath.push_back(s); }}
     });
 
     addFlag({
         .longName = "impure",
-        .description = "allow access to mutable paths and repositories",
+        .description = "Allow access to mutable paths and repositories.",
         .handler = {[&]() {
             evalSettings.pureEval = false;
         }},
@@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs()
 
     addFlag({
         .longName = "override-flake",
-        .description = "override a flake registry value",
+        .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.",
         .labels = {"original-ref", "resolved-ref"},
         .handler = {[&](std::string _from, std::string _to) {
             auto from = parseFlakeRef(_from, absPath("."));

src/libexpr/eval-cache.cc

@@ -390,14 +390,14 @@ Value & AttrCursor::forceValue()
     }
 
     if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
-        if (v.type == tString)
+        if (v.type() == nString)
             cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
                            string_t{v.string.s, {}}};
-        else if (v.type == tPath)
+        else if (v.type() == nPath)
-            cachedValue = {root->db->setString(getKey(), v.path), v.path};
+            cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
-        else if (v.type == tBool)
+        else if (v.type() == nBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
-        else if (v.type == tAttrs)
+        else if (v.type() == nAttrs)
             ; // FIXME: do something?
         else
             cachedValue = {root->db->setMisc(getKey()), misc_t()};
@@ -442,7 +442,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
 
     auto & v = forceValue();
 
-    if (v.type != tAttrs)
+    if (v.type() != nAttrs)
         return nullptr;
         //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
 
@@ -512,10 +512,10 @@ std::string AttrCursor::getString()
 
     auto & v = forceValue();
 
-    if (v.type != tString && v.type != tPath)
+    if (v.type() != nString && v.type() != nPath)
-        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
 
-    return v.type == tString ? v.string.s : v.path;
+    return v.type() == nString ? v.string.s : v.path;
 }
 
 string_t AttrCursor::getStringWithContext()
@@ -543,12 +543,12 @@ string_t AttrCursor::getStringWithContext()
 
     auto & v = forceValue();
 
-    if (v.type == tString)
+    if (v.type() == nString)
         return {v.string.s, v.getContext()};
-    else if (v.type == tPath)
+    else if (v.type() == nPath)
         return {v.path, {}};
     else
-        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
 }
 
 bool AttrCursor::getBool()
@@ -567,7 +567,7 @@ bool AttrCursor::getBool()
 
     auto & v = forceValue();
 
-    if (v.type != tBool)
+    if (v.type() != nBool)
         throw TypeError("'%s' is not a Boolean", getAttrPathStr());
 
     return v.boolean;
@@ -589,7 +589,7 @@ std::vector<Symbol> AttrCursor::getAttrs()
 
     auto & v = forceValue();
 
-    if (v.type != tAttrs)
+    if (v.type() != nAttrs)
         throw TypeError("'%s' is not an attribute set", getAttrPathStr());
 
     std::vector<Symbol> attrs;

src/libexpr/eval-inline.hh

@@ -32,23 +32,21 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 
 void EvalState::forceValue(Value & v, const Pos & pos)
 {
-    if (v.type == tThunk) {
+    if (v.isThunk()) {
         Env * env = v.thunk.env;
         Expr * expr = v.thunk.expr;
         try {
-            v.type = tBlackhole;
+            v.mkBlackhole();
             //checkInterrupt();
             expr->eval(*this, *env, v);
         } catch (...) {
-            v.type = tThunk;
+            v.mkThunk(env, expr);
-            v.thunk.env = env;
-            v.thunk.expr = expr;
             throw;
         }
     }
-    else if (v.type == tApp)
+    else if (v.isApp())
         callFunction(*v.app.left, *v.app.right, v, noPos);
-    else if (v.type == tBlackhole)
+    else if (v.isBlackhole())
         throwEvalError(pos, "infinite recursion encountered");
 }
 
@@ -56,7 +54,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
 inline void EvalState::forceAttrs(Value & v)
 {
     forceValue(v);
-    if (v.type != tAttrs)
+    if (v.type() != nAttrs)
         throwTypeError("value is %1% while a set was expected", v);
 }
 
@@ -64,7 +62,7 @@ inline void EvalState::forceAttrs(Value & v)
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tAttrs)
+    if (v.type() != nAttrs)
         throwTypeError(pos, "value is %1% while a set was expected", v);
 }
 

src/libexpr/eval.cc

@@ -68,7 +68,7 @@ RootValue allocRootValue(Value * v)
 }
 
 
-static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
+void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
 
@@ -77,7 +77,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con
         return;
     }
 
-    switch (v.type) {
+    switch (v.internalType) {
     case tInt:
         str << v.integer;
         break;
@@ -158,32 +158,27 @@ std::ostream & operator << (std::ostream & str, const Value & v)
 
 const Value *getPrimOp(const Value &v) {
     const Value * primOp = &v;
-    while (primOp->type == tPrimOpApp) {
+    while (primOp->isPrimOpApp()) {
         primOp = primOp->primOpApp.left;
     }
-    assert(primOp->type == tPrimOp);
+    assert(primOp->isPrimOp());
     return primOp;
 }
 
-
 string showType(ValueType type)
 {
     switch (type) {
-        case tInt: return "an integer";
+        case nInt: return "an integer";
-        case tBool: return "a Boolean";
+        case nBool: return "a Boolean";
-        case tString: return "a string";
+        case nString: return "a string";
-        case tPath: return "a path";
+        case nPath: return "a path";
-        case tNull: return "null";
+        case nNull: return "null";
-        case tAttrs: return "a set";
+        case nAttrs: return "a set";
-        case tList1: case tList2: case tListN: return "a list";
+        case nList: return "a list";
-        case tThunk: return "a thunk";
+        case nFunction: return "a function";
-        case tApp: return "a function application";
+        case nExternal: return "an external value";
-        case tLambda: return "a function";
+        case nFloat: return "a float";
-        case tBlackhole: return "a black hole";
+        case nThunk: return "a thunk";
-        case tPrimOp: return "a built-in function";
-        case tPrimOpApp: return "a partially applied built-in function";
-        case tExternal: return "an external value";
-        case tFloat: return "a float";
     }
     abort();
 }
@@ -191,15 +186,18 @@ string showType(ValueType type)
 
 string showType(const Value & v)
 {
-    switch (v.type) {
+    switch (v.internalType) {
         case tString: return v.string.context ? "a string with context" : "a string";
         case tPrimOp:
             return fmt("the built-in function '%s'", string(v.primOp->name));
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
+        case tThunk: return "a thunk";
+        case tApp: return "a function application";
+        case tBlackhole: return "a black hole";
     default:
-        return showType(v.type);
+        return showType(v.type());
     }
 }
 
@@ -207,9 +205,9 @@ string showType(const Value & v)
 bool Value::isTrivial() const
 {
     return
-        type != tApp
+        internalType != tApp
-        && type != tPrimOpApp
+        && internalType != tPrimOpApp
-        && (type != tThunk
+        && (internalType != tThunk
             || (dynamic_cast<ExprAttrs *>(thunk.expr)
                 && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
             || dynamic_cast<ExprLambda *>(thunk.expr)
@@ -404,11 +402,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
         for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
     }
 
-    try {
-        addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
-    } catch (Error &) {
-    }
-
     if (evalSettings.restrictEval || evalSettings.pureEval) {
         allowedPaths = PathSet();
 
@@ -432,9 +425,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
         }
     }
 
-    clearValue(vEmptySet);
+    vEmptySet.mkAttrs(allocBindings(0));
-    vEmptySet.type = tAttrs;
-    vEmptySet.attrs = allocBindings(0);
 
     createBaseEnv();
 }
@@ -461,6 +452,8 @@ Path EvalState::checkSourcePath(const Path & path_)
      */
     Path abspath = canonPath(path_);
 
+    if (hasPrefix(abspath, corepkgsPrefix)) return abspath;
+
     for (auto & i : *allowedPaths) {
         if (isDirOrInDir(abspath, i)) {
             found = true;
@@ -550,16 +543,14 @@ Value * EvalState::addPrimOp(const string & name,
        the primop to a dummy value. */
     if (arity == 0) {
         auto vPrimOp = allocValue();
-        vPrimOp->type = tPrimOp;
+        vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym });
-        vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym };
         Value v;
         mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(name, v);
     }
 
     Value * v = allocValue();
-    v->type = tPrimOp;
+    v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym });
-    v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym };
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
@@ -574,8 +565,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
     if (primOp.arity == 0) {
         primOp.arity = 1;
         auto vPrimOp = allocValue();
-        vPrimOp->type = tPrimOp;
+        vPrimOp->mkPrimOp(new PrimOp(std::move(primOp)));
-        vPrimOp->primOp = new PrimOp(std::move(primOp));
         Value v;
         mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(primOp.name, v);
@@ -586,8 +576,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
         primOp.name = symbols.create(std::string(primOp.name, 2));
 
     Value * v = allocValue();
-    v->type = tPrimOp;
+    v->mkPrimOp(new PrimOp(std::move(primOp)));
-    v->primOp = new PrimOp(std::move(primOp));
     staticBaseEnv.vars[envName] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
@@ -603,9 +592,9 @@ Value & EvalState::getBuiltin(const string & name)
 
 std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
-    if (v.type == tPrimOp || v.type == tPrimOpApp) {
+    if (v.isPrimOp() || v.isPrimOpApp()) {
         auto v2 = &v;
-        while (v2->type == tPrimOpApp)
+        while (v2->isPrimOpApp())
             v2 = v2->primOpApp.left;
         if (v2->primOp->doc)
             return Doc {
@@ -668,11 +657,6 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
     });
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
-{
-    throw TypeError(s, s1);
-}
-
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
 {
     throw TypeError({
@@ -697,6 +681,14 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
     });
 }
 
+LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1))
+{
+    throw MissingArgumentError({
+        .hint = hintfmt(s, s1),
+        .errPos = pos
+    });
+}
+
 LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
 {
     e.addTrace(std::nullopt, s, s2);
@@ -710,15 +702,13 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con
 
 void mkString(Value & v, const char * s)
 {
-    mkStringNoCopy(v, dupString(s));
+    v.mkString(dupString(s));
 }
 
 
 Value & mkString(Value & v, std::string_view s, const PathSet & context)
 {
-    v.type = tString;
+    v.mkString(dupStringWithLen(s.data(), s.size()));
-    v.string.s = dupStringWithLen(s.data(), s.size());
-    v.string.context = 0;
     if (!context.empty()) {
         size_t n = 0;
         v.string.context = (const char * *)
@@ -733,7 +723,7 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context)
 
 void mkPath(Value & v, const char * s)
 {
-    mkPathNoCopy(v, dupString(s));
+    v.mkPath(dupString(s));
 }
 
 
@@ -794,16 +784,9 @@ Env & EvalState::allocEnv(size_t size)
 
 void EvalState::mkList(Value & v, size_t size)
 {
-    clearValue(v);
+    v.mkList(size);
-    if (size == 1)
+    if (size > 2)
-        v.type = tList1;
+        v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *));
-    else if (size == 2)
-        v.type = tList2;
-    else {
-        v.type = tListN;
-        v.bigList.size = size;
-        v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0;
-    }
     nrListElems += size;
 }
 
@@ -812,9 +795,7 @@ unsigned long nrThunks = 0;
 
 static inline void mkThunk(Value & v, Env & env, Expr * expr)
 {
-    v.type = tThunk;
+    v.mkThunk(&env, expr);
-    v.thunk.env = &env;
-    v.thunk.expr = expr;
     nrThunks++;
 }
 
@@ -949,7 +930,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e)
 {
     Value v;
     e->eval(*this, env, v);
-    if (v.type != tBool)
+    if (v.type() != nBool)
         throwTypeError("value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
@@ -959,7 +940,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
 {
     Value v;
     e->eval(*this, env, v);
-    if (v.type != tBool)
+    if (v.type() != nBool)
         throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
@@ -968,7 +949,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
 inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v)
 {
     e->eval(*this, env, v);
-    if (v.type != tAttrs)
+    if (v.type() != nAttrs)
         throwTypeError("value is %1% while a set was expected", v);
 }
 
@@ -1068,7 +1049,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Value nameVal;
         i.nameExpr->eval(state, *dynamicEnv, nameVal);
         state.forceValue(nameVal, i.pos);
-        if (nameVal.type == tNull)
+        if (nameVal.type() == nNull)
             continue;
         state.forceStringNoCtx(nameVal);
         Symbol nameSym = state.symbols.create(nameVal.string.s);
@@ -1153,7 +1134,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             Symbol name = getName(i, state, env);
             if (def) {
                 state.forceValue(*vAttrs, pos);
-                if (vAttrs->type != tAttrs ||
+                if (vAttrs->type() != nAttrs ||
                     (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
                 {
                     def->eval(state, env, v);
@@ -1193,7 +1174,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
         state.forceValue(*vAttrs);
         Bindings::iterator j;
         Symbol name = getName(i, state, env);
-        if (vAttrs->type != tAttrs ||
+        if (vAttrs->type() != nAttrs ||
             (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
         {
             mkBool(v, false);
@@ -1209,9 +1190,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
 
 void ExprLambda::eval(EvalState & state, Env & env, Value & v)
 {
-    v.type = tLambda;
+    v.mkLambda(&env, this);
-    v.lambda.env = &env;
-    v.lambda.fun = this;
 }
 
 
@@ -1229,11 +1208,11 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
     /* Figure out the number of arguments still needed. */
     size_t argsDone = 0;
     Value * primOp = &fun;
-    while (primOp->type == tPrimOpApp) {
+    while (primOp->isPrimOpApp()) {
         argsDone++;
         primOp = primOp->primOpApp.left;
     }
-    assert(primOp->type == tPrimOp);
+    assert(primOp->isPrimOp());
     auto arity = primOp->primOp->arity;
     auto argsLeft = arity - argsDone;
 
@@ -1244,7 +1223,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
         Value * vArgs[arity];
         auto n = arity - 1;
         vArgs[n--] = &arg;
-        for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left)
+        for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left)
             vArgs[n--] = arg->primOpApp.right;
 
         /* And call the primop. */
@@ -1254,9 +1233,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
     } else {
         Value * fun2 = allocValue();
         *fun2 = fun;
-        v.type = tPrimOpApp;
+        v.mkPrimOpApp(fun2, &arg);
-        v.primOpApp.left = fun2;
-        v.primOpApp.right = &arg;
     }
 }
 
@@ -1266,12 +1243,12 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     forceValue(fun, pos);
 
-    if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
+    if (fun.isPrimOp() || fun.isPrimOpApp()) {
         callPrimOp(fun, arg, v, pos);
         return;
     }
 
-    if (fun.type == tAttrs) {
+    if (fun.type() == nAttrs) {
       auto found = fun.attrs->find(sFunctor);
       if (found != fun.attrs->end()) {
         /* fun may be allocated on the stack of the calling function,
@@ -1287,7 +1264,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
       }
     }
 
-    if (fun.type != tLambda)
+    if (!fun.isLambda())
         throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
 
     ExprLambda & lambda(*fun.lambda.fun);
@@ -1370,7 +1347,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
 {
     forceValue(fun);
 
-    if (fun.type == tAttrs) {
+    if (fun.type() == nAttrs) {
         auto found = fun.attrs->find(sFunctor);
         if (found != fun.attrs->end()) {
             Value * v = allocValue();
@@ -1380,7 +1357,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
         }
     }
 
-    if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) {
+    if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
         res = fun;
         return;
     }
@@ -1402,7 +1379,13 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
             if (j != args.end()) {
                 actualArgs->attrs->push_back(*j);
             } else if (!i.def) {
-                throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
+                throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
+
+nix attempted to evaluate a function as a top level expression; in this case it must have its
+arguments supplied either by default values, or passed explicitly with --arg or --argstr.
+
+https://nixos.org/manual/nix/stable/#ss-functions)", i.name);
+
             }
         }
     }
@@ -1564,7 +1547,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     NixFloat nf = 0;
 
     bool first = !forceString;
-    ValueType firstType = tString;
+    ValueType firstType = nString;
 
     for (auto & i : *es) {
         Value vTmp;
@@ -1575,36 +1558,36 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
            since paths are copied when they are used in a derivation),
            and none of the strings are allowed to have contexts. */
         if (first) {
-            firstType = vTmp.type;
+            firstType = vTmp.type();
             first = false;
         }
 
-        if (firstType == tInt) {
+        if (firstType == nInt) {
-            if (vTmp.type == tInt) {
+            if (vTmp.type() == nInt) {
                 n += vTmp.integer;
-            } else if (vTmp.type == tFloat) {
+            } else if (vTmp.type() == nFloat) {
                 // Upgrade the type from int to float;
-                firstType = tFloat;
+                firstType = nFloat;
                 nf = n;
                 nf += vTmp.fpoint;
             } else
                 throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
-        } else if (firstType == tFloat) {
+        } else if (firstType == nFloat) {
-            if (vTmp.type == tInt) {
+            if (vTmp.type() == nInt) {
                 nf += vTmp.integer;
-            } else if (vTmp.type == tFloat) {
+            } else if (vTmp.type() == nFloat) {
                 nf += vTmp.fpoint;
             } else
                 throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
         } else
-            s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
+            s << state.coerceToString(pos, vTmp, context, false, firstType == nString);
     }
 
-    if (firstType == tInt)
+    if (firstType == nInt)
         mkInt(v, n);
-    else if (firstType == tFloat)
+    else if (firstType == nFloat)
         mkFloat(v, nf);
-    else if (firstType == tPath) {
+    else if (firstType == nPath) {
         if (!context.empty())
             throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
         auto path = canonPath(s.str());
@@ -1631,7 +1614,7 @@ void EvalState::forceValueDeep(Value & v)
 
         forceValue(v);
 
-        if (v.type == tAttrs) {
+        if (v.type() == nAttrs) {
             for (auto & i : *v.attrs)
                 try {
                     recurse(*i.value);
@@ -1654,7 +1637,7 @@ void EvalState::forceValueDeep(Value & v)
 NixInt EvalState::forceInt(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tInt)
+    if (v.type() != nInt)
         throwTypeError(pos, "value is %1% while an integer was expected", v);
     return v.integer;
 }
@@ -1663,9 +1646,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
 NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type == tInt)
+    if (v.type() == nInt)
         return v.integer;
-    else if (v.type != tFloat)
+    else if (v.type() != nFloat)
         throwTypeError(pos, "value is %1% while a float was expected", v);
     return v.fpoint;
 }
@@ -1674,7 +1657,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
 bool EvalState::forceBool(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tBool)
+    if (v.type() != nBool)
         throwTypeError(pos, "value is %1% while a Boolean was expected", v);
     return v.boolean;
 }
@@ -1682,14 +1665,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos)
 
 bool EvalState::isFunctor(Value & fun)
 {
-    return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
+    return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
 }
 
 
 void EvalState::forceFunction(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
+    if (v.type() != nFunction && !isFunctor(v))
         throwTypeError(pos, "value is %1% while a function was expected", v);
 }
 
@@ -1697,7 +1680,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos)
 string EvalState::forceString(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
-    if (v.type != tString) {
+    if (v.type() != nString) {
         if (pos)
             throwTypeError(pos, "value is %1% while a string was expected", v);
         else
@@ -1730,7 +1713,7 @@ void copyContext(const Value & v, PathSet & context)
 std::vector<std::pair<Path, std::string>> Value::getContext()
 {
     std::vector<std::pair<Path, std::string>> res;
-    assert(type == tString);
+    assert(internalType == tString);
     if (string.context)
         for (const char * * p = string.context; *p; ++p)
             res.push_back(decodeContext(*p));
@@ -1763,11 +1746,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
 
 bool EvalState::isDerivation(Value & v)
 {
-    if (v.type != tAttrs) return false;
+    if (v.type() != nAttrs) return false;
     Bindings::iterator i = v.attrs->find(sType);
     if (i == v.attrs->end()) return false;
     forceValue(*i->value);
-    if (i->value->type != tString) return false;
+    if (i->value->type() != nString) return false;
     return strcmp(i->value->string.s, "derivation") == 0;
 }
 
@@ -1792,17 +1775,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 
     string s;
 
-    if (v.type == tString) {
+    if (v.type() == nString) {
         copyContext(v, context);
         return v.string.s;
     }
 
-    if (v.type == tPath) {
+    if (v.type() == nPath) {
         Path path(canonPath(v.path));
         return copyToStore ? copyPathToStore(context, path) : path;
     }
 
-    if (v.type == tAttrs) {
+    if (v.type() == nAttrs) {
         auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
         if (maybeString) {
             return *maybeString;
@@ -1812,18 +1795,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
-    if (v.type == tExternal)
+    if (v.type() == nExternal)
         return v.external->coerceToString(pos, context, coerceMore, copyToStore);
 
     if (coerceMore) {
 
         /* Note that `false' is represented as an empty string for
            shell scripting convenience, just like `null'. */
-        if (v.type == tBool && v.boolean) return "1";
+        if (v.type() == nBool && v.boolean) return "1";
-        if (v.type == tBool && !v.boolean) return "";
+        if (v.type() == nBool && !v.boolean) return "";
-        if (v.type == tInt) return std::to_string(v.integer);
+        if (v.type() == nInt) return std::to_string(v.integer);
-        if (v.type == tFloat) return std::to_string(v.fpoint);
+        if (v.type() == nFloat) return std::to_string(v.fpoint);
-        if (v.type == tNull) return "";
+        if (v.type() == nNull) return "";
 
         if (v.isList()) {
             string result;
@@ -1886,40 +1869,38 @@ bool EvalState::eqValues(Value & v1, Value & v2)
     if (&v1 == &v2) return true;
 
     // Special case type-compatibility between float and int
-    if (v1.type == tInt && v2.type == tFloat)
+    if (v1.type() == nInt && v2.type() == nFloat)
         return v1.integer == v2.fpoint;
-    if (v1.type == tFloat && v2.type == tInt)
+    if (v1.type() == nFloat && v2.type() == nInt)
         return v1.fpoint == v2.integer;
 
     // All other types are not compatible with each other.
-    if (v1.type != v2.type) return false;
+    if (v1.type() != v2.type()) return false;
 
-    switch (v1.type) {
+    switch (v1.type()) {
 
-        case tInt:
+        case nInt:
             return v1.integer == v2.integer;
 
-        case tBool:
+        case nBool:
             return v1.boolean == v2.boolean;
 
-        case tString:
+        case nString:
             return strcmp(v1.string.s, v2.string.s) == 0;
 
-        case tPath:
+        case nPath:
             return strcmp(v1.path, v2.path) == 0;
 
-        case tNull:
+        case nNull:
             return true;
 
-        case tList1:
+        case nList:
-        case tList2:
-        case tListN:
             if (v1.listSize() != v2.listSize()) return false;
             for (size_t n = 0; n < v1.listSize(); ++n)
                 if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false;
             return true;
 
-        case tAttrs: {
+        case nAttrs: {
             /* If both sets denote a derivation (type = "derivation"),
                then compare their outPaths. */
             if (isDerivation(v1) && isDerivation(v2)) {
@@ -1941,15 +1922,13 @@ bool EvalState::eqValues(Value & v1, Value & v2)
         }
 
         /* Functions are incomparable. */
-        case tLambda:
+        case nFunction:
-        case tPrimOp:
-        case tPrimOpApp:
             return false;
 
-        case tExternal:
+        case nExternal:
             return *v1.external == *v2.external;
 
-        case tFloat:
+        case nFloat:
             return v1.fpoint == v2.fpoint;
 
         default:

src/libexpr/eval.hh

@@ -432,4 +432,6 @@ struct EvalSettings : Config
 
 extern EvalSettings evalSettings;
 
+static const std::string corepkgsPrefix{"/__corepkgs__/"};
+
 }

src/libexpr/flake/flake.cc

@@ -73,7 +73,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
 
 static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos)
 {
-    if (value.type == tThunk && value.isTrivial())
+    if (value.isThunk() && value.isTrivial())
         state.forceValue(value, pos);
 }
 
@@ -82,9 +82,9 @@ static void expectType(EvalState & state, ValueType type,
     Value & value, const Pos & pos)
 {
     forceTrivialValue(state, value, pos);
-    if (value.type != type)
+    if (value.type() != type)
         throw Error("expected %s but got %s at %s",
-            showType(type), showType(value.type), pos);
+            showType(type), showType(value.type()), pos);
 }
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
@@ -93,7 +93,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
 static FlakeInput parseFlakeInput(EvalState & state,
     const std::string & inputName, Value * value, const Pos & pos)
 {
-    expectType(state, tAttrs, *value, pos);
+    expectType(state, nAttrs, *value, pos);
 
     FlakeInput input;
 
@@ -108,24 +108,33 @@ static FlakeInput parseFlakeInput(EvalState & state,
     for (nix::Attr attr : *(value->attrs)) {
         try {
             if (attr.name == sUrl) {
-                expectType(state, tString, *attr.value, *attr.pos);
+                expectType(state, nString, *attr.value, *attr.pos);
                 url = attr.value->string.s;
                 attrs.emplace("url", *url);
             } else if (attr.name == sFlake) {
-                expectType(state, tBool, *attr.value, *attr.pos);
+                expectType(state, nBool, *attr.value, *attr.pos);
                 input.isFlake = attr.value->boolean;
             } else if (attr.name == sInputs) {
                 input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
             } else if (attr.name == sFollows) {
-                expectType(state, tString, *attr.value, *attr.pos);
+                expectType(state, nString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
             } else {
-                if (attr.value->type == tString)
+                switch (attr.value->type()) {
+                    case nString:
                         attrs.emplace(attr.name, attr.value->string.s);
-                else
+                        break;
-                    throw TypeError("flake input attribute '%s' is %s while a string is expected",
+                    case nBool:
+                        attrs.emplace(attr.name, Explicit<bool> { attr.value->boolean });
+                        break;
+                    case nInt:
+                        attrs.emplace(attr.name, (long unsigned int)attr.value->integer);
+                        break;
+                    default:
+                        throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
                             attr.name, showType(*attr.value));
                 }
+            }
         } catch (Error & e) {
             e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name));
             throw;
@@ -158,7 +167,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
 {
     std::map<FlakeId, FlakeInput> inputs;
 
-    expectType(state, tAttrs, *value, pos);
+    expectType(state, nAttrs, *value, pos);
 
     for (nix::Attr & inputAttr : *(*value).attrs) {
         inputs.emplace(inputAttr.name,
@@ -199,10 +208,10 @@ static Flake getFlake(
     Value vInfo;
     state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
 
-    expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
+    expectType(state, nAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
 
     if (auto description = vInfo.attrs->get(state.sDescription)) {
-        expectType(state, tString, *description->value, *description->pos);
+        expectType(state, nString, *description->value, *description->pos);
         flake.description = description->value->string.s;
     }
 
@@ -214,9 +223,9 @@ static Flake getFlake(
     auto sOutputs = state.symbols.create("outputs");
 
     if (auto outputs = vInfo.attrs->get(sOutputs)) {
-        expectType(state, tLambda, *outputs->value, *outputs->pos);
+        expectType(state, nFunction, *outputs->value, *outputs->pos);
 
-        if (outputs->value->lambda.fun->matchAttrs) {
+        if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
             for (auto & formal : outputs->value->lambda.fun->formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(formal.name, FlakeInput {
@@ -231,21 +240,21 @@ static Flake getFlake(
     auto sNixConfig = state.symbols.create("nixConfig");
 
     if (auto nixConfig = vInfo.attrs->get(sNixConfig)) {
-        expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos);
+        expectType(state, nAttrs, *nixConfig->value, *nixConfig->pos);
 
         for (auto & setting : *nixConfig->value->attrs) {
             forceTrivialValue(state, *setting.value, *setting.pos);
-            if (setting.value->type == tString)
+            if (setting.value->type() == nString)
                 flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
-            else if (setting.value->type == tInt)
+            else if (setting.value->type() == nInt)
                 flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
-            else if (setting.value->type == tBool)
+            else if (setting.value->type() == nBool)
                 flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
-            else if (setting.value->isList()) {
+            else if (setting.value->type() == nList) {
                 std::vector<std::string> ss;
                 for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
                     auto elem = setting.value->listElems()[n];
-                    if (elem->type != tString)
+                    if (elem->type() != nString)
                         throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
                             setting.name, showType(*setting.value));
                     ss.push_back(state.forceStringNoCtx(*elem, *setting.pos));

src/libexpr/get-drvs.cc

@@ -128,7 +128,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
     if (!outTI->isList()) throw errMsg;
     Outputs result;
     for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
-        if ((*i)->type != tString) throw errMsg;
+        if ((*i)->type() != nString) throw errMsg;
         auto out = outputs.find((*i)->string.s);
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
@@ -172,20 +172,20 @@ StringSet DrvInfo::queryMetaNames()
 bool DrvInfo::checkMeta(Value & v)
 {
     state->forceValue(v);
-    if (v.isList()) {
+    if (v.type() == nList) {
         for (unsigned int n = 0; n < v.listSize(); ++n)
             if (!checkMeta(*v.listElems()[n])) return false;
         return true;
     }
-    else if (v.type == tAttrs) {
+    else if (v.type() == nAttrs) {
         Bindings::iterator i = v.attrs->find(state->sOutPath);
         if (i != v.attrs->end()) return false;
         for (auto & i : *v.attrs)
             if (!checkMeta(*i.value)) return false;
         return true;
     }
-    else return v.type == tInt || v.type == tBool || v.type == tString ||
+    else return v.type() == nInt || v.type() == nBool || v.type() == nString ||
-                v.type == tFloat;
+                v.type() == nFloat;
 }
 
 
@@ -201,7 +201,7 @@ Value * DrvInfo::queryMeta(const string & name)
 string DrvInfo::queryMetaString(const string & name)
 {
     Value * v = queryMeta(name);
-    if (!v || v->type != tString) return "";
+    if (!v || v->type() != nString) return "";
     return v->string.s;
 }
 
@@ -210,12 +210,12 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type == tInt) return v->integer;
+    if (v->type() == nInt) return v->integer;
-    if (v->type == tString) {
+    if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
-        NixInt n;
+        if (auto n = string2Int<NixInt>(v->string.s))
-        if (string2Int(v->string.s, n)) return n;
+            return *n;
     }
     return def;
 }
@@ -224,12 +224,12 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type == tFloat) return v->fpoint;
+    if (v->type() == nFloat) return v->fpoint;
-    if (v->type == tString) {
+    if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
-        NixFloat n;
+        if (auto n = string2Float<NixFloat>(v->string.s))
-        if (string2Float(v->string.s, n)) return n;
+            return *n;
     }
     return def;
 }
@@ -239,8 +239,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
-    if (v->type == tBool) return v->boolean;
+    if (v->type() == nBool) return v->boolean;
-    if (v->type == tString) {
+    if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
         if (strcmp(v->string.s, "true") == 0) return true;
@@ -331,7 +331,7 @@ static void getDerivations(EvalState & state, Value & vIn,
     /* Process the expression. */
     if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
 
-    else if (v.type == tAttrs) {
+    else if (v.type() == nAttrs) {
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
@@ -353,7 +353,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                 /* If the value of this attribute is itself a set,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
-                if (i->value->type == tAttrs) {
+                if (i->value->type() == nAttrs) {
                     Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
                     if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
@@ -362,7 +362,7 @@ static void getDerivations(EvalState & state, Value & vIn,
         }
     }
 
-    else if (v.isList()) {
+    else if (v.type() == nList) {
         for (unsigned int n = 0; n < v.listSize(); ++n) {
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
             if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))

src/libexpr/local.mk

@@ -40,6 +40,6 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
 
 $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh

src/libexpr/nixexpr.hh

@@ -17,6 +17,7 @@ MakeError(ThrownError, AssertionError);
 MakeError(Abort, EvalError);
 MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, Error);
+MakeError(MissingArgumentError, Error);
 MakeError(RestrictedPathError, Error);
 
 
@@ -129,7 +130,7 @@ struct ExprPath : Expr
 {
     string s;
     Value v;
-    ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
+    ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };

src/libexpr/parser.y

@@ -698,6 +698,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
         Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
+
+    if (hasPrefix(path, "nix/"))
+        return corepkgsPrefix + path.substr(4);
+
     throw ThrownError({
         .hint = hintfmt(evalSettings.pureEval
             ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"

src/libexpr/primops.cc

@@ -164,7 +164,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
         state.forceFunction(**fun, pos);
         mkApp(v, **fun, w);
         state.forceAttrs(v, pos);
-    } else {
+    }
+
+    else if (path == corepkgsPrefix + "fetchurl.nix") {
+        state.eval(state.parseExprFromString(
+            #include "fetchurl.nix.gen.hh"
+            , "/"), v);
+    }
+
+    else {
         if (!vScope)
             state.evalFile(realPath, v);
         else {
@@ -356,24 +364,20 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
 {
     state.forceValue(*args[0], pos);
     string t;
-    switch (args[0]->type) {
+    switch (args[0]->type()) {
-        case tInt: t = "int"; break;
+        case nInt: t = "int"; break;
-        case tBool: t = "bool"; break;
+        case nBool: t = "bool"; break;
-        case tString: t = "string"; break;
+        case nString: t = "string"; break;
-        case tPath: t = "path"; break;
+        case nPath: t = "path"; break;
-        case tNull: t = "null"; break;
+        case nNull: t = "null"; break;
-        case tAttrs: t = "set"; break;
+        case nAttrs: t = "set"; break;
-        case tList1: case tList2: case tListN: t = "list"; break;
+        case nList: t = "list"; break;
-        case tLambda:
+        case nFunction: t = "lambda"; break;
-        case tPrimOp:
+        case nExternal:
-        case tPrimOpApp:
-            t = "lambda";
-            break;
-        case tExternal:
             t = args[0]->external->typeOf();
             break;
-        case tFloat: t = "float"; break;
+        case nFloat: t = "float"; break;
-        default: abort();
+        case nThunk: abort();
     }
     mkString(v, state.symbols.create(t));
 }
@@ -393,7 +397,7 @@ static RegisterPrimOp primop_typeOf({
 static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tNull);
+    mkBool(v, args[0]->type() == nNull);
 }
 
 static RegisterPrimOp primop_isNull({
@@ -413,18 +417,7 @@ static RegisterPrimOp primop_isNull({
 static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    bool res;
+    mkBool(v, args[0]->type() == nFunction);
-    switch (args[0]->type) {
-        case tLambda:
-        case tPrimOp:
-        case tPrimOpApp:
-            res = true;
-            break;
-        default:
-            res = false;
-            break;
-    }
-    mkBool(v, res);
 }
 
 static RegisterPrimOp primop_isFunction({
@@ -440,7 +433,7 @@ static RegisterPrimOp primop_isFunction({
 static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tInt);
+    mkBool(v, args[0]->type() == nInt);
 }
 
 static RegisterPrimOp primop_isInt({
@@ -456,7 +449,7 @@ static RegisterPrimOp primop_isInt({
 static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tFloat);
+    mkBool(v, args[0]->type() == nFloat);
 }
 
 static RegisterPrimOp primop_isFloat({
@@ -472,7 +465,7 @@ static RegisterPrimOp primop_isFloat({
 static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tString);
+    mkBool(v, args[0]->type() == nString);
 }
 
 static RegisterPrimOp primop_isString({
@@ -488,7 +481,7 @@ static RegisterPrimOp primop_isString({
 static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tBool);
+    mkBool(v, args[0]->type() == nBool);
 }
 
 static RegisterPrimOp primop_isBool({
@@ -504,7 +497,7 @@ static RegisterPrimOp primop_isBool({
 static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tPath);
+    mkBool(v, args[0]->type() == nPath);
 }
 
 static RegisterPrimOp primop_isPath({
@@ -520,20 +513,20 @@ struct CompareValues
 {
     bool operator () (const Value * v1, const Value * v2) const
     {
-        if (v1->type == tFloat && v2->type == tInt)
+        if (v1->type() == nFloat && v2->type() == nInt)
             return v1->fpoint < v2->integer;
-        if (v1->type == tInt && v2->type == tFloat)
+        if (v1->type() == nInt && v2->type() == nFloat)
             return v1->integer < v2->fpoint;
-        if (v1->type != v2->type)
+        if (v1->type() != v2->type())
             throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
-        switch (v1->type) {
+        switch (v1->type()) {
-            case tInt:
+            case nInt:
                 return v1->integer < v2->integer;
-            case tFloat:
+            case nFloat:
                 return v1->fpoint < v2->fpoint;
-            case tString:
+            case nString:
                 return strcmp(v1->string.s, v2->string.s) < 0;
-            case tPath:
+            case nPath:
                 return strcmp(v1->path, v2->path) < 0;
             default:
                 throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
@@ -777,7 +770,7 @@ static RegisterPrimOp primop_deepSeq({
 static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    if (args[0]->type == tString)
+    if (args[0]->type() == nString)
         printError("trace: %1%", args[0]->string.s);
     else
         printError("trace: %1%", *args[0]);
@@ -902,7 +895,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
             if (ignoreNulls) {
                 state.forceValue(*i->value, pos);
-                if (i->value->type == tNull) continue;
+                if (i->value->type() == nNull) continue;
             }
 
             if (i->name == state.sContentAddressed) {
@@ -1308,7 +1301,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value
 {
     PathSet context;
     Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false));
-    if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
+    if (args[0]->type() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
 }
 
 static RegisterPrimOp primop_dirOf({
@@ -1449,7 +1442,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
         Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name));
         if (ent.type == DT_UNKNOWN)
             ent.type = getFileType(path + "/" + ent.name);
-        mkStringNoCopy(*ent_val,
+        ent_val->mkString(
             ent.type == DT_REG ? "regular" :
             ent.type == DT_DIR ? "directory" :
             ent.type == DT_LNK ? "symlink" :
@@ -1813,7 +1806,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
         });
 
     state.forceValue(*args[0], pos);
-    if (args[0]->type != tLambda)
+    if (args[0]->type() != nFunction)
         throw TypeError({
             .hint = hintfmt(
                 "first argument in call to 'filterSource' is not a function but %1%",
@@ -2079,7 +2072,7 @@ static RegisterPrimOp primop_hasAttr({
 static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->type == tAttrs);
+    mkBool(v, args[0]->type() == nAttrs);
 }
 
 static RegisterPrimOp primop_isAttrs({
@@ -2259,11 +2252,11 @@ static RegisterPrimOp primop_catAttrs({
 static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) {
+    if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) {
         state.mkAttrs(v, 0);
         return;
     }
-    if (args[0]->type != tLambda)
+    if (!args[0]->isLambda())
         throw TypeError({
             .hint = hintfmt("'functionArgs' requires a function"),
             .errPos = pos
@@ -2342,7 +2335,7 @@ static RegisterPrimOp primop_mapAttrs({
 static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    mkBool(v, args[0]->isList());
+    mkBool(v, args[0]->type() == nList);
 }
 
 static RegisterPrimOp primop_isList({
@@ -2694,7 +2687,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
     auto comparator = [&](Value * a, Value * b) {
         /* Optimization: if the comparator is lessThan, bypass
            callFunction. */
-        if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan)
+        if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan)
             return CompareValues()(a, b);
 
         Value vTmp1, vTmp2;
@@ -2836,7 +2829,7 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value &
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    if (args[0]->type == tFloat || args[1]->type == tFloat)
+    if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos));
     else
         mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos));
@@ -2855,7 +2848,7 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value &
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    if (args[0]->type == tFloat || args[1]->type == tFloat)
+    if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos));
     else
         mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos));
@@ -2874,7 +2867,7 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value &
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    if (args[0]->type == tFloat || args[1]->type == tFloat)
+    if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos));
     else
         mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos));
@@ -2901,7 +2894,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
             .errPos = pos
         });
 
-    if (args[0]->type == tFloat || args[1]->type == tFloat) {
+    if (args[0]->type() == nFloat || args[1]->type() == nFloat) {
         mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos));
     } else {
         NixInt i1 = state.forceInt(*args[0], pos);

src/libexpr/primops/fetchMercurial.cc

@@ -17,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
 
     state.forceValue(*args[0]);
 
-    if (args[0]->type == tAttrs) {
+    if (args[0]->type() == nAttrs) {
 
         state.forceAttrs(*args[0], pos);
 

src/libexpr/primops/fetchTree.cc

@@ -85,26 +85,26 @@ static void fetchTree(
 
     state.forceValue(*args[0]);
 
-    if (args[0]->type == tAttrs) {
+    if (args[0]->type() == nAttrs) {
         state.forceAttrs(*args[0], pos);
 
         fetchers::Attrs attrs;
 
         for (auto & attr : *args[0]->attrs) {
             state.forceValue(*attr.value);
-            if (attr.value->type == tPath || attr.value->type == tString)
+            if (attr.value->type() == nPath || attr.value->type() == nString)
                 addURI(
                     state,
                     attrs,
                     attr.name,
                     state.coerceToString(*attr.pos, *attr.value, context, false, false)
                 );
-            else if (attr.value->type == tString)
+            else if (attr.value->type() == nString)
                 addURI(state, attrs, attr.name, attr.value->string.s);
-            else if (attr.value->type == tBool)
+            else if (attr.value->type() == nBool)
                 attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
-            else if (attr.value->type == tInt)
+            else if (attr.value->type() == nInt)
-                attrs.emplace(attr.name, attr.value->integer);
+                attrs.emplace(attr.name, uint64_t(attr.value->integer));
             else
                 throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
                     attr.name, showType(*attr.value));
@@ -163,7 +163,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
 
     state.forceValue(*args[0]);
 
-    if (args[0]->type == tAttrs) {
+    if (args[0]->type() == nAttrs) {
 
         state.forceAttrs(*args[0], pos);
 
@@ -324,6 +324,11 @@ static RegisterPrimOp primop_fetchGit({
           A Boolean parameter that specifies whether submodules should be
           checked out. Defaults to `false`.
 
+        - allRefs  
+          Whether to fetch all refs of the repository. With this argument being
+          true, it's possible to load a `rev` from *any* `ref` (by default only
+          `rev`s from the specified `ref` are supported).
+
       Here are some examples of how to use `fetchGit`.
 
         - To fetch a private repository over SSH:

src/libexpr/value-to-json.cc

@@ -16,30 +16,30 @@ void printValueAsJSON(EvalState & state, bool strict,
 
     if (strict) state.forceValue(v);
 
-    switch (v.type) {
+    switch (v.type()) {
 
-        case tInt:
+        case nInt:
             out.write(v.integer);
             break;
 
-        case tBool:
+        case nBool:
             out.write(v.boolean);
             break;
 
-        case tString:
+        case nString:
             copyContext(v, context);
             out.write(v.string.s);
             break;
 
-        case tPath:
+        case nPath:
             out.write(state.copyPathToStore(context, v.path));
             break;
 
-        case tNull:
+        case nNull:
             out.write(nullptr);
             break;
 
-        case tAttrs: {
+        case nAttrs: {
             auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
             if (maybeString) {
                 out.write(*maybeString);
@@ -61,7 +61,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
-        case tList1: case tList2: case tListN: {
+        case nList: {
             auto list(out.list());
             for (unsigned int n = 0; n < v.listSize(); ++n) {
                 auto placeholder(list.placeholder());
@@ -70,15 +70,18 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
-        case tExternal:
+        case nExternal:
             v.external->printValueAsJSON(state, strict, out, context);
             break;
 
-        case tFloat:
+        case nFloat:
             out.write(v.fpoint);
             break;
 
-        default:
+        case nThunk:
+            throw TypeError("cannot convert %1% to JSON", showType(v));
+
+        case nFunction:
             throw TypeError("cannot convert %1% to JSON", showType(v));
     }
 }

src/libexpr/value-to-xml.cc

@@ -58,31 +58,31 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
     if (strict) state.forceValue(v);
 
-    switch (v.type) {
+    switch (v.type()) {
 
-        case tInt:
+        case nInt:
             doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
             break;
 
-        case tBool:
+        case nBool:
             doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
             break;
 
-        case tString:
+        case nString:
             /* !!! show the context? */
             copyContext(v, context);
             doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
             break;
 
-        case tPath:
+        case nPath:
             doc.writeEmptyElement("path", singletonAttrs("value", v.path));
             break;
 
-        case tNull:
+        case nNull:
             doc.writeEmptyElement("null");
             break;
 
-        case tAttrs:
+        case nAttrs:
             if (state.isDerivation(v)) {
                 XMLAttrs xmlAttrs;
 
@@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 a = v.attrs->find(state.sDrvPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->type == tString)
+                    if (a->value->type() == nString)
                         xmlAttrs["drvPath"] = drvPath = a->value->string.s;
                 }
 
                 a = v.attrs->find(state.sOutPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->type == tString)
+                    if (a->value->type() == nString)
                         xmlAttrs["outPath"] = a->value->string.s;
                 }
 
@@ -118,14 +118,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
             break;
 
-        case tList1: case tList2: case tListN: {
+        case nList: {
             XMLOpenElement _(doc, "list");
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
             break;
         }
 
-        case tLambda: {
+        case nFunction: {
+            if (!v.isLambda()) {
+                // FIXME: Serialize primops and primopapps
+                doc.writeEmptyElement("unevaluated");
+                break;
+            }
             XMLAttrs xmlAttrs;
             if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
             XMLOpenElement _(doc, "function", xmlAttrs);
@@ -143,15 +148,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             break;
         }
 
-        case tExternal:
+        case nExternal:
             v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
             break;
 
-        case tFloat:
+        case nFloat:
             doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
             break;
 
-        default:
+        case nThunk:
             doc.writeEmptyElement("unevaluated");
     }
 }

src/libexpr/value.hh

@@ -27,8 +27,24 @@ typedef enum {
     tPrimOpApp,
     tExternal,
     tFloat
-} ValueType;
+} InternalType;
 
+// This type abstracts over all actual value types in the language,
+// grouping together implementation details like tList*, different function
+// types, and types in non-normal form (so thunks and co.)
+typedef enum {
+    nThunk,
+    nInt,
+    nFloat,
+    nBool,
+    nString,
+    nPath,
+    nNull,
+    nAttrs,
+    nList,
+    nFunction,
+    nExternal
+} ValueType;
 
 class Bindings;
 struct Env;
@@ -90,7 +106,28 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
 
 struct Value
 {
-    ValueType type;
+private:
+    InternalType internalType;
+
+friend std::string showType(const Value & v);
+friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v);
+
+public:
+
+    // Functions needed to distinguish the type
+    // These should be removed eventually, by putting the functionality that's
+    // needed by callers into methods of this type
+
+    // type() == nThunk
+    inline bool isThunk() const { return internalType == tThunk; };
+    inline bool isApp() const { return internalType == tApp; };
+    inline bool isBlackhole() const { return internalType == tBlackhole; };
+
+    // type() == nFunction
+    inline bool isLambda() const { return internalType == tLambda; };
+    inline bool isPrimOp() const { return internalType == tPrimOp; };
+    inline bool isPrimOpApp() const { return internalType == tPrimOpApp; };
+
     union
     {
         NixInt integer;
@@ -147,24 +184,161 @@ struct Value
         NixFloat fpoint;
     };
 
+    // Returns the normal type of a Value. This only returns nThunk if the
+    // Value hasn't been forceValue'd
+    inline ValueType type() const
+    {
+        switch (internalType) {
+            case tInt: return nInt;
+            case tBool: return nBool;
+            case tString: return nString;
+            case tPath: return nPath;
+            case tNull: return nNull;
+            case tAttrs: return nAttrs;
+            case tList1: case tList2: case tListN: return nList;
+            case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
+            case tExternal: return nExternal;
+            case tFloat: return nFloat;
+            case tThunk: case tApp: case tBlackhole: return nThunk;
+        }
+        abort();
+    }
+
+    /* After overwriting an app node, be sure to clear pointers in the
+       Value to ensure that the target isn't kept alive unnecessarily. */
+    inline void clearValue()
+    {
+        app.left = app.right = 0;
+    }
+
+    inline void mkInt(NixInt n)
+    {
+        clearValue();
+        internalType = tInt;
+        integer = n;
+    }
+
+    inline void mkBool(bool b)
+    {
+        clearValue();
+        internalType = tBool;
+        boolean = b;
+    }
+
+    inline void mkString(const char * s, const char * * context = 0)
+    {
+        internalType = tString;
+        string.s = s;
+        string.context = context;
+    }
+
+    inline void mkPath(const char * s)
+    {
+        clearValue();
+        internalType = tPath;
+        path = s;
+    }
+
+    inline void mkNull()
+    {
+        clearValue();
+        internalType = tNull;
+    }
+
+    inline void mkAttrs(Bindings * a)
+    {
+        clearValue();
+        internalType = tAttrs;
+        attrs = a;
+    }
+
+    inline void mkList(size_t size)
+    {
+        clearValue();
+        if (size == 1)
+            internalType = tList1;
+        else if (size == 2)
+            internalType = tList2;
+        else {
+            internalType = tListN;
+            bigList.size = size;
+        }
+    }
+
+    inline void mkThunk(Env * e, Expr * ex)
+    {
+        internalType = tThunk;
+        thunk.env = e;
+        thunk.expr = ex;
+    }
+
+    inline void mkApp(Value * l, Value * r)
+    {
+        internalType = tApp;
+        app.left = l;
+        app.right = r;
+    }
+
+    inline void mkLambda(Env * e, ExprLambda * f)
+    {
+        internalType = tLambda;
+        lambda.env = e;
+        lambda.fun = f;
+    }
+
+    inline void mkBlackhole()
+    {
+        internalType = tBlackhole;
+        // Value will be overridden anyways
+    }
+
+    inline void mkPrimOp(PrimOp * p)
+    {
+        clearValue();
+        internalType = tPrimOp;
+        primOp = p;
+    }
+
+
+    inline void mkPrimOpApp(Value * l, Value * r)
+    {
+        internalType = tPrimOpApp;
+        app.left = l;
+        app.right = r;
+    }
+
+    inline void mkExternal(ExternalValueBase * e)
+    {
+        clearValue();
+        internalType = tExternal;
+        external = e;
+    }
+
+    inline void mkFloat(NixFloat n)
+    {
+        clearValue();
+        internalType = tFloat;
+        fpoint = n;
+    }
+
     bool isList() const
     {
-        return type == tList1 || type == tList2 || type == tListN;
+        return internalType == tList1 || internalType == tList2 || internalType == tListN;
     }
 
     Value * * listElems()
     {
-        return type == tList1 || type == tList2 ? smallList : bigList.elems;
+        return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
     }
 
     const Value * const * listElems() const
     {
-        return type == tList1 || type == tList2 ? smallList : bigList.elems;
+        return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
     }
 
     size_t listSize() const
     {
-        return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
+        return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size;
     }
 
     /* Check whether forcing this value requires a trivial amount of
@@ -176,86 +350,42 @@ struct Value
 };
 
 
-/* After overwriting an app node, be sure to clear pointers in the
-   Value to ensure that the target isn't kept alive unnecessarily. */
-static inline void clearValue(Value & v)
-{
-    v.app.left = v.app.right = 0;
-}
-
 
+// TODO: Remove these static functions, replace call sites with v.mk* instead
 static inline void mkInt(Value & v, NixInt n)
 {
-    clearValue(v);
+    v.mkInt(n);
-    v.type = tInt;
-    v.integer = n;
 }
 
-
 static inline void mkFloat(Value & v, NixFloat n)
 {
-    clearValue(v);
+    v.mkFloat(n);
-    v.type = tFloat;
-    v.fpoint = n;
 }
 
-
 static inline void mkBool(Value & v, bool b)
 {
-    clearValue(v);
+    v.mkBool(b);
-    v.type = tBool;
-    v.boolean = b;
 }
 
-
 static inline void mkNull(Value & v)
 {
-    clearValue(v);
+    v.mkNull();
-    v.type = tNull;
 }
 
-
 static inline void mkApp(Value & v, Value & left, Value & right)
 {
-    v.type = tApp;
+    v.mkApp(&left, &right);
-    v.app.left = &left;
-    v.app.right = &right;
 }
 
-
-static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
-{
-    v.type = tPrimOpApp;
-    v.app.left = &left;
-    v.app.right = &right;
-}
-
-
-static inline void mkStringNoCopy(Value & v, const char * s)
-{
-    v.type = tString;
-    v.string.s = s;
-    v.string.context = 0;
-}
-
-
 static inline void mkString(Value & v, const Symbol & s)
 {
-    mkStringNoCopy(v, ((const string &) s).c_str());
+    v.mkString(((const string &) s).c_str());
 }
 
 
 void mkString(Value & v, const char * s);
 
 
-static inline void mkPathNoCopy(Value & v, const char * s)
-{
-    clearValue(v);
-    v.type = tPath;
-    v.path = s;
-}
-
-
 void mkPath(Value & v, const char * s);
 
 

src/libfetchers/attrs.cc

@@ -11,11 +11,11 @@ Attrs jsonToAttrs(const nlohmann::json & json)
 
     for (auto & i : json.items()) {
         if (i.value().is_number())
-            attrs.emplace(i.key(), i.value().get<int64_t>());
+            attrs.emplace(i.key(), i.value().get<uint64_t>());
         else if (i.value().is_string())
             attrs.emplace(i.key(), i.value().get<std::string>());
         else if (i.value().is_boolean())
-            attrs.emplace(i.key(), i.value().get<bool>());
+            attrs.emplace(i.key(), Explicit<bool> { i.value().get<bool>() });
         else
             throw Error("unsupported input attribute type in lock file");
     }

src/libfetchers/git.cc

@@ -59,12 +59,13 @@ struct GitInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
                 throw Error("unsupported Git input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
         maybeGetBoolAttr(attrs, "shallow");
         maybeGetBoolAttr(attrs, "submodules");
+        maybeGetBoolAttr(attrs, "allRefs");
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
             if (std::regex_search(*ref, badGitRefRegex))
@@ -169,10 +170,12 @@ struct GitInputScheme : InputScheme
 
         bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
         bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
+        bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
 
         std::string cacheType = "git";
         if (shallow) cacheType += "-shallow";
         if (submodules) cacheType += "-submodules";
+        if (allRefs) cacheType += "-all-refs";
 
         auto getImmutableAttrs = [&]()
         {
@@ -337,6 +340,9 @@ struct GitInputScheme : InputScheme
                         throw;
                     }
                 }
+            } else {
+                if (allRefs) {
+                    doFetch = true;
                 } else {
                     /* If the local ref is older than ‘tarball-ttl’ seconds, do a
                        git fetch to update the local ref to the remote ref. */
@@ -344,6 +350,7 @@ struct GitInputScheme : InputScheme
                     doFetch = stat(localRefFile.c_str(), &st) != 0 ||
                         (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
                 }
+            }
 
             if (doFetch) {
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
@@ -352,7 +359,9 @@ struct GitInputScheme : InputScheme
                 // we're using --quiet for now. Should process its stderr.
                 try {
                     auto ref = input.getRef();
-                    auto fetchRef = ref->compare(0, 5, "refs/") == 0
+                    auto fetchRef = allRefs
+                        ? "refs/*"
+                        : ref->compare(0, 5, "refs/") == 0
                             ? *ref
                             : "refs/heads/" + *ref;
                     runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
@@ -392,6 +401,28 @@ struct GitInputScheme : InputScheme
         AutoDelete delTmpDir(tmpDir, true);
         PathFilter filter = defaultPathFilter;
 
+        RunOptions checkCommitOpts(
+            "git",
+            { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
+        );
+        checkCommitOpts.searchPath = true;
+        checkCommitOpts.mergeStderrToStdout = true;
+
+        auto result = runProgram(checkCommitOpts);
+        if (WEXITSTATUS(result.first) == 128
+            && result.second.find("bad file") != std::string::npos
+        ) {
+            throw Error(
+                "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
+                    "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
+                    ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
+                    "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
+                input.getRev()->gitRev(),
+                *input.getRef(),
+                actualUrl
+            );
+        }
+
         if (submodules) {
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);

src/libfetchers/github.cc

@@ -37,15 +37,29 @@ struct GitArchiveInputScheme : InputScheme
         std::optional<std::string> ref;
         std::optional<std::string> host_url;
 
-        if (path.size() == 2) {
+        auto size = path.size();
-        } else if (path.size() == 3) {
+        if (size == 3) {
             if (std::regex_match(path[2], revRegex))
                 rev = Hash::parseAny(path[2], htSHA1);
             else if (std::regex_match(path[2], refRegex))
                 ref = path[2];
             else
                 throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
-        } else
+        } else if (size > 3) {
+            std::string rs;
+            for (auto i = std::next(path.begin(), 2); i != path.end(); i++) {
+                rs += *i;
+                if (std::next(i) != path.end()) {
+                    rs += "/";
+                }
+            }
+
+            if (std::regex_match(rs, refRegex)) {
+                ref = rs;
+            } else {
+                throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs);
+            }
+        } else if (size < 2)
             throw BadURL("URL '%s' is invalid", url.url);
 
         for (auto &[name, value] : url.query) {
@@ -195,14 +209,14 @@ struct GitArchiveInputScheme : InputScheme
 
         auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
 
-        input.attrs.insert_or_assign("lastModified", lastModified);
+        input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
 
         getCache()->add(
             store,
             immutableAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", lastModified}
+                {"lastModified", uint64_t(lastModified)}
             },
             tree.storePath,
             true);

src/libfetchers/mercurial.cc

@@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme
 
         Attrs infoAttrs({
             {"rev", input.getRev()->gitRev()},
-            {"revCount", (int64_t) revCount},
+            {"revCount", (uint64_t) revCount},
         });
 
         if (!_input.getRev())

src/libfetchers/path.cc

@@ -20,10 +20,10 @@ struct PathInputScheme : InputScheme
             if (name == "rev" || name == "narHash")
                 input.attrs.insert_or_assign(name, value);
             else if (name == "revCount" || name == "lastModified") {
-                uint64_t n;
+                if (auto n = string2Int<uint64_t>(value))
-                if (!string2Int(value, n))
+                    input.attrs.insert_or_assign(name, *n);
+                else
                     throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
-                input.attrs.insert_or_assign(name, n);
             }
             else
                 throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);

src/libfetchers/tarball.cc

@@ -152,7 +152,7 @@ std::pair<Tree, time_t> downloadTarball(
     }
 
     Attrs infoAttrs({
-        {"lastModified", lastModified},
+        {"lastModified", uint64_t(lastModified)},
         {"etag", res.etag},
     });
 

src/libmain/common-args.cc

@@ -10,25 +10,25 @@ MixCommonArgs::MixCommonArgs(const string & programName)
     addFlag({
         .longName = "verbose",
         .shortName = 'v',
-        .description = "increase verbosity level",
+        .description = "Increase the logging verbosity level.",
         .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
     });
 
     addFlag({
         .longName = "quiet",
-        .description = "decrease verbosity level",
+        .description = "Decrease the logging verbosity level.",
         .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
     });
 
     addFlag({
         .longName = "debug",
-        .description = "enable debug output",
+        .description = "Set the logging verbosity level to 'debug'.",
         .handler = {[]() { verbosity = lvlDebug; }},
     });
 
     addFlag({
         .longName = "option",
-        .description = "set a Nix configuration option (overriding `nix.conf`)",
+        .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).",
         .labels = {"name", "value"},
         .handler = {[](std::string name, std::string value) {
             try {
@@ -51,8 +51,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
 
     addFlag({
         .longName = "log-format",
-        .description = "format of log output; `raw`, `internal-json`, `bar` "
+        .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.",
-                        "or `bar-with-logs`",
         .labels = {"format"},
         .handler = {[](std::string format) { setLogFormat(format); }},
     });
@@ -60,7 +59,7 @@ MixCommonArgs::MixCommonArgs(const string & programName)
     addFlag({
         .longName = "max-jobs",
         .shortName = 'j',
-        .description = "maximum number of parallel builds",
+        .description = "The maximum number of parallel builds.",
         .labels = Strings{"jobs"},
         .handler = {[=](std::string s) {
             settings.set("max-jobs", s);

src/libmain/common-args.hh

@@ -16,7 +16,7 @@ struct MixDryRun : virtual Args
 
     MixDryRun()
     {
-        mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun);
+        mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun);
     }
 };
 
@@ -26,7 +26,7 @@ struct MixJSON : virtual Args
 
     MixJSON()
     {
-        mkFlag(0, "json", "produce JSON output", &json);
+        mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json);
     }
 };
 

src/libmain/shared.cc

@@ -18,6 +18,8 @@
 
 #include <openssl/crypto.h>
 
+#include <sodium.h>
+
 
 namespace nix {
 
@@ -126,6 +128,9 @@ void initNix()
     CRYPTO_set_locking_callback(opensslLockCallback);
 #endif
 
+    if (sodium_init() == -1)
+        throw Error("could not initialise libsodium");
+
     loadConfFile();
 
     startSignalHandlerThread();
@@ -181,50 +186,58 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     addFlag({
         .longName = "no-build-output",
         .shortName = 'Q',
-        .description = "do not show build output",
+        .description = "Do not show build output.",
         .handler = {[&]() {setLogFormat(LogFormat::raw); }},
     });
 
     addFlag({
         .longName = "keep-failed",
         .shortName ='K',
-        .description = "keep temporary directories of failed builds",
+        .description = "Keep temporary directories of failed builds.",
         .handler = {&(bool&) settings.keepFailed, true},
     });
 
     addFlag({
         .longName = "keep-going",
         .shortName ='k',
-        .description = "keep going after a build fails",
+        .description = "Keep going after a build fails.",
         .handler = {&(bool&) settings.keepGoing, true},
     });
 
     addFlag({
         .longName = "fallback",
-        .description = "build from source if substitution fails",
+        .description = "Build from source if substitution fails.",
         .handler = {&(bool&) settings.tryFallback, true},
     });
 
     auto intSettingAlias = [&](char shortName, const std::string & longName,
-        const std::string & description, const std::string & dest) {
+        const std::string & description, const std::string & dest)
-        mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
+    {
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .labels = {"n"},
+            .handler = {[=](std::string s) {
+                auto n = string2IntWithUnitPrefix<uint64_t>(s);
                 settings.set(dest, std::to_string(n));
+            }}
         });
     };
 
-    intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores");
+    intSettingAlias(0, "cores", "Maximum number of CPU cores to use inside a build.", "cores");
-    intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time");
+    intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time");
-    intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
+    intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout");
 
-    mkFlag(0, "readonly-mode", "do not write to the Nix store",
+    mkFlag(0, "readonly-mode", "Do not write to the Nix store.",
         &settings.readOnlyMode);
 
-    mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
+    mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.",
         &gcWarning, false);
 
     addFlag({
         .longName = "store",
-        .description = "URI of the Nix store to use",
+        .description = "The URL of the Nix store to use.",
         .labels = {"store-uri"},
         .handler = {&(std::string&) settings.storeUri},
     });
@@ -274,9 +287,7 @@ void printVersion(const string & programName)
 #if HAVE_BOEHMGC
         cfg.push_back("gc");
 #endif
-#if HAVE_SODIUM
         cfg.push_back("signed-caches");
-#endif
         std::cout << "System type: " << settings.thisSystem << "\n";
         std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
         std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";

src/libmain/shared.hh

@@ -57,23 +57,7 @@ template<class N> N getIntArg(const string & opt,
 {
     ++i;
     if (i == end) throw UsageError("'%1%' requires an argument", opt);
-    string s = *i;
+    return string2IntWithUnitPrefix<N>(*i);
-    N multiplier = 1;
-    if (allowUnit && !s.empty()) {
-        char u = std::toupper(*s.rbegin());
-        if (std::isalpha(u)) {
-            if (u == 'K') multiplier = 1ULL << 10;
-            else if (u == 'M') multiplier = 1ULL << 20;
-            else if (u == 'G') multiplier = 1ULL << 30;
-            else if (u == 'T') multiplier = 1ULL << 40;
-            else throw UsageError("invalid unit specifier '%1%'", u);
-            s.resize(s.size() - 1);
-        }
-    }
-    N n;
-    if (!string2Int(s, n))
-        throw UsageError("'%1%' requires an integer argument", opt);
-    return n * multiplier;
 }
 
 

src/libstore/binary-cache-store.hh

@@ -24,7 +24,7 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
         "enable multi-threading compression, available for xz only currently"};
 };
 
-class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
+class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
 {
 
 private:

src/libstore/build/derivation-goal.cc

@@ -50,6 +50,11 @@
 #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
 #endif
 
+#if __APPLE__
+#include <spawn.h>
+#include <sys/sysctl.h>
+#endif
+
 #include <pwd.h>
 #include <grp.h>
 
@@ -675,13 +680,9 @@ void DerivationGoal::tryToBuild()
 }
 
 void DerivationGoal::tryLocalBuild() {
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
+    /* Make sure that we are allowed to start a build. */
-
-    /* Make sure that we are allowed to start a build.  If this
-       derivation prefers to be done locally, do it even if
-       maxBuildJobs is 0. */
     unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
+    if (curBuilds >= settings.maxBuildJobs) {
         worker.waitForBuildSlot(shared_from_this());
         outputLocks.unlock();
         return;
@@ -1695,12 +1696,10 @@ void DerivationGoal::startBuilder()
             userNamespaceSync.writeSide = -1;
         });
 
-        pid_t tmp;
         auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
         assert(ss.size() == 2);
         usingUserNamespace = ss[0] == "1";
-        if (!string2Int<pid_t>(ss[1], tmp)) abort();
+        pid = string2Int<pid_t>(ss[1]).value();
-        pid = tmp;
 
         if (usingUserNamespace) {
             /* Set the UID/GID mapping of the builder's user namespace
@@ -1985,7 +1984,7 @@ void DerivationGoal::writeStructuredAttrs()
     chownToBuilder(tmpDir + "/.attrs.sh");
 }
 
-struct RestrictedStoreConfig : LocalFSStoreConfig
+struct RestrictedStoreConfig : virtual LocalFSStoreConfig
 {
     using LocalFSStoreConfig::LocalFSStoreConfig;
     const std::string name() { return "Restricted Store"; }
@@ -1994,14 +1993,19 @@ struct RestrictedStoreConfig : LocalFSStoreConfig
 /* A wrapper around LocalStore that only allows building/querying of
    paths that are in the input closures of the build or were added via
    recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
+struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
 {
     ref<LocalStore> next;
 
     DerivationGoal & goal;
 
     RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
-        : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , RestrictedStoreConfig(params)
+        , Store(params)
+        , LocalFSStore(params)
+        , next(next), goal(goal)
     { }
 
     Path getRealStoreDir() override
@@ -2852,7 +2856,31 @@ void DerivationGoal::runChild()
             }
         }
 
+#if __APPLE__
+        posix_spawnattr_t attrp;
+
+        if (posix_spawnattr_init(&attrp))
+            throw SysError("failed to initialize builder");
+
+        if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC))
+            throw SysError("failed to initialize builder");
+
+        if (drv->platform == "aarch64-darwin") {
+            // Unset kern.curproc_arch_affinity so we can escape Rosetta
+            int affinity = 0;
+            sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity));
+
+            cpu_type_t cpu = CPU_TYPE_ARM64;
+            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+        } else if (drv->platform == "x86_64-darwin") {
+            cpu_type_t cpu = CPU_TYPE_X86_64;
+            posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+        }
+
+        posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+#else
         execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+#endif
 
         throw SysError("executing '%1%'", drv->builder);
 

src/libstore/crypto.cc

@@ -2,21 +2,19 @@
 #include "util.hh"
 #include "globals.hh"
 
-#if HAVE_SODIUM
 #include <sodium.h>
-#endif
 
 namespace nix {
 
-static std::pair<std::string, std::string> split(const string & s)
+static std::pair<std::string_view, std::string_view> split(std::string_view s)
 {
     size_t colon = s.find(':');
     if (colon == std::string::npos || colon == 0)
         return {"", ""};
-    return {std::string(s, 0, colon), std::string(s, colon + 1)};
+    return {s.substr(0, colon), s.substr(colon + 1)};
 }
 
-Key::Key(const string & s)
+Key::Key(std::string_view s)
 {
     auto ss = split(s);
 
@@ -29,62 +27,57 @@ Key::Key(const string & s)
     key = base64Decode(key);
 }
 
-SecretKey::SecretKey(const string & s)
+std::string Key::to_string() const
+{
+    return name + ":" + base64Encode(key);
+}
+
+SecretKey::SecretKey(std::string_view s)
     : Key(s)
 {
-#if HAVE_SODIUM
     if (key.size() != crypto_sign_SECRETKEYBYTES)
         throw Error("secret key is not valid");
-#endif
 }
 
-#if !HAVE_SODIUM
+std::string SecretKey::signDetached(std::string_view data) const
-[[noreturn]] static void noSodium()
 {
-    throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
-}
-#endif
-
-std::string SecretKey::signDetached(const std::string & data) const
-{
-#if HAVE_SODIUM
     unsigned char sig[crypto_sign_BYTES];
     unsigned long long sigLen;
     crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(),
         (unsigned char *) key.data());
     return name + ":" + base64Encode(std::string((char *) sig, sigLen));
-#else
-    noSodium();
-#endif
 }
 
 PublicKey SecretKey::toPublicKey() const
 {
-#if HAVE_SODIUM
     unsigned char pk[crypto_sign_PUBLICKEYBYTES];
     crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data());
     return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES));
-#else
-    noSodium();
-#endif
 }
 
-PublicKey::PublicKey(const string & s)
+SecretKey SecretKey::generate(std::string_view name)
+{
+    unsigned char pk[crypto_sign_PUBLICKEYBYTES];
+    unsigned char sk[crypto_sign_SECRETKEYBYTES];
+    if (crypto_sign_keypair(pk, sk) != 0)
+        throw Error("key generation failed");
+
+    return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES));
+}
+
+PublicKey::PublicKey(std::string_view s)
     : Key(s)
 {
-#if HAVE_SODIUM
     if (key.size() != crypto_sign_PUBLICKEYBYTES)
         throw Error("public key is not valid");
-#endif
 }
 
 bool verifyDetached(const std::string & data, const std::string & sig,
     const PublicKeys & publicKeys)
 {
-#if HAVE_SODIUM
     auto ss = split(sig);
 
-    auto key = publicKeys.find(ss.first);
+    auto key = publicKeys.find(std::string(ss.first));
     if (key == publicKeys.end()) return false;
 
     auto sig2 = base64Decode(ss.second);
@@ -94,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig,
     return crypto_sign_verify_detached((unsigned char *) sig2.data(),
         (unsigned char *) data.data(), data.size(),
         (unsigned char *) key->second.key.data()) == 0;
-#else
-    noSodium();
-#endif
 }
 
 PublicKeys getDefaultPublicKeys()

src/libstore/crypto.hh

@@ -13,32 +13,40 @@ struct Key
 
     /* Construct Key from a string in the format
        ‘<name>:<key-in-base64>’. */
-    Key(const std::string & s);
+    Key(std::string_view s);
+
+    std::string to_string() const;
 
 protected:
-    Key(const std::string & name, const std::string & key)
+    Key(std::string_view name, std::string && key)
-        : name(name), key(key) { }
+        : name(name), key(std::move(key)) { }
 };
 
 struct PublicKey;
 
 struct SecretKey : Key
 {
-    SecretKey(const std::string & s);
+    SecretKey(std::string_view s);
 
     /* Return a detached signature of the given string. */
-    std::string signDetached(const std::string & s) const;
+    std::string signDetached(std::string_view s) const;
 
     PublicKey toPublicKey() const;
+
+    static SecretKey generate(std::string_view name);
+
+private:
+    SecretKey(std::string_view name, std::string && key)
+        : Key(name, std::move(key)) { }
 };
 
 struct PublicKey : Key
 {
-    PublicKey(const std::string & data);
+    PublicKey(std::string_view data);
 
 private:
-    PublicKey(const std::string & name, const std::string & key)
+    PublicKey(std::string_view name, std::string && key)
-        : Key(name, key) { }
+        : Key(name, std::move(key)) { }
     friend struct SecretKey;
 };
 

src/libstore/dummy-store.cc

@@ -9,7 +9,7 @@ struct DummyStoreConfig : virtual StoreConfig {
     const std::string name() override { return "Dummy Store"; }
 };
 
-struct DummyStore : public Store, public virtual DummyStoreConfig
+struct DummyStore : public virtual DummyStoreConfig, public virtual Store
 {
     DummyStore(const std::string scheme, const std::string uri, const Params & params)
         : DummyStore(params)
@@ -17,6 +17,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
 
     DummyStore(const Params & params)
         : StoreConfig(params)
+        , DummyStoreConfig(params)
         , Store(params)
     { }
 

src/libstore/filetransfer.hh

@@ -63,7 +63,7 @@ struct FileTransferRequest
     std::string mimeType;
     std::function<void(std::string_view data)> dataCallback;
 
-    FileTransferRequest(const std::string & uri)
+    FileTransferRequest(std::string_view uri)
         : uri(uri), parentAct(getCurActivity()) { }
 
     std::string verb()

src/libstore/fs-accessor.hh

@@ -25,7 +25,14 @@ public:
 
     virtual StringSet readDirectory(const Path & path) = 0;
 
-    virtual std::string readFile(const Path & path) = 0;
+    /**
+     * Read a file inside the store.
+     *
+     * If `requireValidPath` is set to `true` (the default), the path must be
+     * inside a valid store path, otherwise it just needs to be physically
+     * present (but not necessarily properly registered)
+     */
+    virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0;
 
     virtual std::string readLink(const Path & path) = 0;
 };

src/libstore/globals.cc

@@ -131,6 +131,28 @@ StringSet Settings::getDefaultSystemFeatures()
     return features;
 }
 
+StringSet Settings::getDefaultExtraPlatforms()
+{
+    if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1())
+        return StringSet{"i686-linux"};
+#if __APPLE__
+    // Rosetta 2 emulation layer can run x86_64 binaries on aarch64
+    // machines. Note that we can’t force processes from executing
+    // x86_64 in aarch64 environments or vice versa since they can
+    // always exec with their own binary preferences.
+    else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
+        if (std::string{SYSTEM} == "x86_64-darwin")
+            return StringSet{"aarch64-darwin"};
+        else if (std::string{SYSTEM} == "aarch64-darwin")
+            return StringSet{"x86_64-darwin"};
+        else
+            return StringSet{};
+    }
+#endif
+    else
+        return StringSet{};
+}
+
 bool Settings::isExperimentalFeatureEnabled(const std::string & name)
 {
     auto & f = experimentalFeatures.get();
@@ -206,9 +228,13 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
 void MaxBuildJobsSetting::set(const std::string & str, bool append)
 {
     if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
-    else if (!string2Int(str, value))
+    else {
+        if (auto n = string2Int<decltype(value)>(str))
+            value = *n;
+        else
             throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
     }
+}
 
 
 void initPlugins()

src/libstore/globals.hh

@@ -34,6 +34,8 @@ class Settings : public Config {
 
     StringSet getDefaultSystemFeatures();
 
+    StringSet getDefaultExtraPlatforms();
+
     bool isWSL1();
 
 public:
@@ -545,7 +547,7 @@ public:
 
     Setting<StringSet> extraPlatforms{
         this,
-        std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
+        getDefaultExtraPlatforms(),
         "extra-platforms",
         R"(
           Platforms other than the native one which this machine is capable of

src/libstore/http-binary-cache-store.cc

@@ -15,7 +15,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
     const std::string name() override { return "Http Binary Cache Store"; }
 };
 
-class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
+class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore
 {
 private:
 
@@ -36,6 +36,9 @@ public:
         const Path & _cacheUri,
         const Params & params)
         : StoreConfig(params)
+        , BinaryCacheStoreConfig(params)
+        , HttpBinaryCacheStoreConfig(params)
+        , Store(params)
         , BinaryCacheStore(params)
         , cacheUri(scheme + "://" + _cacheUri)
     {

src/libstore/legacy-ssh-store.cc

@@ -22,7 +22,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig
     const std::string name() override { return "Legacy SSH Store"; }
 };
 
-struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
+struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
 {
     // Hack for getting remote build log output.
     // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
@@ -48,6 +48,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
 
     LegacySSHStore(const string & scheme, const string & host, const Params & params)
         : StoreConfig(params)
+        , LegacySSHStoreConfig(params)
         , Store(params)
         , host(host)
         , connections(make_ref<Pool<Connection>>(

src/libstore/local-binary-cache-store.cc

@@ -11,7 +11,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
     const std::string name() override { return "Local Binary Cache Store"; }
 };
 
-class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
+class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore
 {
 private:
 
@@ -24,6 +24,9 @@ public:
         const Path & binaryCacheDir,
         const Params & params)
         : StoreConfig(params)
+        , BinaryCacheStoreConfig(params)
+        , LocalBinaryCacheStoreConfig(params)
+        , Store(params)
         , BinaryCacheStore(params)
         , binaryCacheDir(binaryCacheDir)
     {

src/libstore/local-fs-store.cc

@@ -19,10 +19,10 @@ struct LocalStoreAccessor : public FSAccessor
 
     LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { }
 
-    Path toRealPath(const Path & path)
+    Path toRealPath(const Path & path, bool requireValidPath = true)
     {
         auto storePath = store->toStorePath(path).first;
-        if (!store->isValidPath(storePath))
+        if (requireValidPath && !store->isValidPath(storePath))
             throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
@@ -61,9 +61,9 @@ struct LocalStoreAccessor : public FSAccessor
         return res;
     }
 
-    std::string readFile(const Path & path) override
+    std::string readFile(const Path & path, bool requireValidPath = true) override
     {
-        return nix::readFile(toRealPath(path));
+        return nix::readFile(toRealPath(path, requireValidPath));
     }
 
     std::string readLink(const Path & path) override

src/libstore/local-fs-store.hh

@@ -20,7 +20,7 @@ struct LocalFSStoreConfig : virtual StoreConfig
         "log", "directory where Nix will store state"};
 };
 
-class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
+class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
 {
 public:
 

src/libstore/local-store.cc

@@ -66,8 +66,10 @@ int getSchema(Path schemaPath)
     int curSchema = 0;
     if (pathExists(schemaPath)) {
         string s = readFile(schemaPath);
-        if (!string2Int(s, curSchema))
+        auto n = string2Int<int>(s);
+        if (!n)
             throw Error("'%1%' is corrupt", schemaPath);
+        curSchema = *n;
     }
     return curSchema;
 }
@@ -100,6 +102,8 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
 
 LocalStore::LocalStore(const Params & params)
     : StoreConfig(params)
+    , LocalFSStoreConfig(params)
+    , LocalStoreConfig(params)
     , Store(params)
     , LocalFSStore(params)
     , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
@@ -734,11 +738,19 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
+        callback(retrySQLite<std::shared_ptr<const ValidPathInfo>>([&]() {
             auto state(_state.lock());
+            return queryPathInfoInternal(*state, path);
+        }));
 
+    } catch (...) { callback.rethrow(); }
+}
+
+
+std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & state, const StorePath & path)
+{
     /* Get the path info. */
-            auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path)));
+    auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path)));
 
     if (!useQueryPathInfo.next())
         return std::shared_ptr<ValidPathInfo>();
@@ -758,7 +770,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
 
     info->registrationTime = useQueryPathInfo.getInt(2);
 
-            auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3);
+    auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3);
     if (s) info->deriver = parseStorePath(s);
 
     /* Note that narSize = NULL yields 0. */
@@ -766,22 +778,19 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
 
     info->ultimate = useQueryPathInfo.getInt(5) == 1;
 
-            s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6);
+    s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6);
     if (s) info->sigs = tokenizeString<StringSet>(s, " ");
 
-            s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7);
+    s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7);
     if (s) info->ca = parseContentAddressOpt(s);
 
     /* Get the references. */
-            auto useQueryReferences(state->stmts->QueryReferences.use()(info->id));
+    auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
 
     while (useQueryReferences.next())
         info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
 
     return info;
-        }));
-
-    } catch (...) { callback.rethrow(); }
 }
 
 
@@ -905,7 +914,7 @@ LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_)
         if (realisation)
             outputs.insert_or_assign(outputName, realisation->outPath);
         else
-            outputs.insert_or_assign(outputName, std::nullopt);
+            outputs.insert({outputName, std::nullopt});
     }
 
     return outputs;
@@ -1606,7 +1615,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si
 
         SQLiteTxn txn(state->db);
 
-        auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath)));
+        auto info = std::const_pointer_cast<ValidPathInfo>(queryPathInfoInternal(*state, storePath));
 
         info->sigs.insert(sigs.begin(), sigs.end());
 

src/libstore/local-store.hh

@@ -43,7 +43,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
 };
 
 
-class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
+class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore
 {
 private:
 
@@ -198,9 +198,7 @@ public:
 
     void vacuumDB();
 
-    /* Repair the contents of the given path by redownloading it using
+    void repairPath(const StorePath & path) override;
-       a substituter (if available). */
-    void repairPath(const StorePath & path);
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
@@ -235,6 +233,8 @@ private:
     void verifyPath(const Path & path, const StringSet & store,
         PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
 
+    std::shared_ptr<const ValidPathInfo> queryPathInfoInternal(State & state, const StorePath & path);
+
     void updatePathInfo(State & state, const ValidPathInfo & info);
 
     void upgradeStore6();

src/libstore/names.cc

@@ -80,16 +80,16 @@ string nextComponent(string::const_iterator & p,
 
 static bool componentsLT(const string & c1, const string & c2)
 {
-    int n1, n2;
+    auto n1 = string2Int<int>(c1);
-    bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2);
+    auto n2 = string2Int<int>(c2);
 
-    if (c1Num && c2Num) return n1 < n2;
+    if (n1 && n2) return *n1 < *n2;
-    else if (c1 == "" && c2Num) return true;
+    else if (c1 == "" && n2) return true;
     else if (c1 == "pre" && c2 != "pre") return true;
     else if (c2 == "pre") return false;
     /* Assume that `2.3a' < `2.3.1'. */
-    else if (c2Num) return true;
+    else if (n2) return true;
-    else if (c1Num) return false;
+    else if (n1) return false;
     else return c1 < c2;
 }
 

src/libstore/nar-accessor.cc

@@ -203,7 +203,7 @@ struct NarAccessor : public FSAccessor
         return res;
     }
 
-    std::string readFile(const Path & path) override
+    std::string readFile(const Path & path, bool requireValidPath = true) override
     {
         auto i = get(path);
         if (i.type != FSAccessor::Type::tRegular)

src/libstore/nar-info-disk-cache.cc

@@ -109,8 +109,10 @@ public:
                 SQLiteStmt(state->db,
                     "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
                     .use()
-                    (now - settings.ttlNegativeNarInfoCache)
+                    // Use a minimum TTL to prevent --refresh from
-                    (now - settings.ttlPositiveNarInfoCache)
+                    // nuking the entire disk cache.
+                    (now - std::max(settings.ttlNegativeNarInfoCache.get(), 3600U))
+                    (now - std::max(settings.ttlPositiveNarInfoCache.get(), 30 * 24 * 3600U))
                     .exec();
 
                 debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));

src/libstore/nar-info.cc

@@ -46,14 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
         else if (name == "FileHash")
             fileHash = parseHashField(value);
         else if (name == "FileSize") {
-            if (!string2Int(value, fileSize)) throw corrupt();
+            auto n = string2Int<decltype(fileSize)>(value);
+            if (!n) throw corrupt();
+            fileSize = *n;
         }
         else if (name == "NarHash") {
             narHash = parseHashField(value);
             haveNarHash = true;
         }
         else if (name == "NarSize") {
-            if (!string2Int(value, narSize)) throw corrupt();
+            auto n = string2Int<decltype(narSize)>(value);
+            if (!n) throw corrupt();
+            narSize = *n;
         }
         else if (name == "References") {
             auto refs = tokenizeString<Strings>(value, " ");

src/libstore/parsed-derivations.cc

@@ -101,6 +101,10 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const
         && !drv.isBuiltin())
         return false;
 
+    if (settings.maxBuildJobs.get() == 0
+        && !drv.isBuiltin())
+        return false;
+
     for (auto & feature : getRequiredSystemFeatures())
         if (!localStore.systemFeatures.get().count(feature)) return false;
 

src/libstore/profiles.cc

@@ -21,9 +21,8 @@ static std::optional<GenerationNumber> parseName(const string & profileName, con
     string s = string(name, profileName.size() + 1);
     string::size_type p = s.find("-link");
     if (p == string::npos) return {};
-    unsigned int n;
+    if (auto n = string2Int<unsigned int>(s.substr(0, p)))
-    if (string2Int(string(s, 0, p), n) && n >= 0)
+        return *n;
-        return n;
     else
         return {};
 }
@@ -214,12 +213,12 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
 {
     time_t curTime = time(0);
     string strDays = string(timeSpec, 0, timeSpec.size() - 1);
-    int days;
+    auto days = string2Int<int>(strDays);
 
-    if (!string2Int(strDays, days) || days < 1)
+    if (!days || *days < 1)
         throw Error("invalid number of days specifier '%1%'", timeSpec);
 
-    time_t oldTime = curTime - days * 24 * 3600;
+    time_t oldTime = curTime - *days * 24 * 3600;
 
     deleteGenerationsOlderThan(profile, oldTime, dryRun);
 }

src/libstore/references.cc

@@ -88,9 +88,6 @@ PathSet scanForReferences(Sink & toTee,
     TeeSink sink { refsSink, toTee };
     std::map<string, Path> backMap;
 
-    /* For efficiency (and a higher hit rate), just search for the
-       hash part of the file name.  (This assumes that all references
-       have the form `HASH-bla'). */
     for (auto & i : refs) {
         auto baseName = std::string(baseNameOf(i));
         string::size_type pos = baseName.find('-');

src/libstore/remote-fs-accessor.cc

@@ -43,13 +43,13 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string &
     }
 }
 
-std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
+std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath)
 {
     auto path = canonPath(path_);
 
     auto [storePath, restPath] = store->toStorePath(path);
 
-    if (!store->isValidPath(storePath))
+    if (requireValidPath && !store->isValidPath(storePath))
         throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
 
     auto i = nars.find(std::string(storePath.hashPart()));
@@ -113,9 +113,9 @@ StringSet RemoteFSAccessor::readDirectory(const Path & path)
     return res.first->readDirectory(res.second);
 }
 
-std::string RemoteFSAccessor::readFile(const Path & path)
+std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath)
 {
-    auto res = fetch(path);
+    auto res = fetch(path, requireValidPath);
     return res.first->readFile(res.second);
 }
 

src/libstore/remote-fs-accessor.hh

@@ -14,7 +14,7 @@ class RemoteFSAccessor : public FSAccessor
 
     Path cacheDir;
 
-    std::pair<ref<FSAccessor>, Path> fetch(const Path & path_);
+    std::pair<ref<FSAccessor>, Path> fetch(const Path & path_, bool requireValidPath = true);
 
     friend class BinaryCacheStore;
 
@@ -32,7 +32,7 @@ public:
 
     StringSet readDirectory(const Path & path) override;
 
-    std::string readFile(const Path & path) override;
+    std::string readFile(const Path & path, bool requireValidPath = true) override;
 
     std::string readLink(const Path & path) override;
 };

src/libstore/remote-store.cc

@@ -77,8 +77,8 @@ void write(const Store & store, Sink & out, const std::optional<ContentAddress>
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
-    : Store(params)
+    : RemoteStoreConfig(params)
-    , RemoteStoreConfig(params)
+    , Store(params)
     , connections(make_ref<Pool<Connection>>(
             std::max(1, (int) maxConnections),
             [this]() {

src/libstore/remote-store.hh

@@ -29,7 +29,7 @@ struct RemoteStoreConfig : virtual StoreConfig
 
 /* FIXME: RemoteStore is a misnomer - should be something like
    DaemonStore. */
-class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
+class RemoteStore : public virtual RemoteStoreConfig, public virtual Store
 {
 public:
 

src/libstore/s3-binary-cache-store.cc

@@ -177,6 +177,11 @@ S3Helper::FileTransferResult S3Helper::getObject(
     return res;
 }
 
+S3BinaryCacheStore::S3BinaryCacheStore(const Params & params)
+    : BinaryCacheStoreConfig(params)
+    , BinaryCacheStore(params)
+{ }
+
 struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
@@ -195,7 +200,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
     const std::string name() override { return "S3 Binary Cache Store"; }
 };
 
-struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
+struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore
 {
     std::string bucketName;
 
@@ -208,6 +213,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
         const std::string & bucketName,
         const Params & params)
         : StoreConfig(params)
+        , BinaryCacheStoreConfig(params)
+        , S3BinaryCacheStoreConfig(params)
+        , Store(params)
+        , BinaryCacheStore(params)
         , S3BinaryCacheStore(params)
         , bucketName(bucketName)
         , s3Helper(profile, region, scheme, endpoint)

src/libstore/s3-binary-cache-store.hh

@@ -6,13 +6,11 @@
 
 namespace nix {
 
-class S3BinaryCacheStore : public BinaryCacheStore
+class S3BinaryCacheStore : public virtual BinaryCacheStore
 {
 protected:
 
-    S3BinaryCacheStore(const Params & params)
+    S3BinaryCacheStore(const Params & params);
-        : BinaryCacheStore(params)
-    { }
 
 public:
 

src/libstore/ssh-store.cc

@@ -20,12 +20,14 @@ struct SSHStoreConfig : virtual RemoteStoreConfig
     const std::string name() override { return "SSH Store"; }
 };
 
-class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
+class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
 {
 public:
 
     SSHStore(const std::string & scheme, const std::string & host, const Params & params)
         : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , SSHStoreConfig(params)
         , Store(params)
         , RemoteStore(params)
         , host(host)

src/libstore/store-api.cc

@@ -949,19 +949,20 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
         getline(str, s);
         auto narHash = Hash::parseAny(s, htSHA256);
         getline(str, s);
-        uint64_t narSize;
+        auto narSize = string2Int<uint64_t>(s);
-        if (!string2Int(s, narSize)) throw Error("number expected");
+        if (!narSize) throw Error("number expected");
-        hashGiven = { narHash, narSize };
+        hashGiven = { narHash, *narSize };
     }
     ValidPathInfo info(store.parseStorePath(path), hashGiven->first);
     info.narSize = hashGiven->second;
     std::string deriver;
     getline(str, deriver);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
-    string s; int n;
+    string s;
     getline(str, s);
-    if (!string2Int(s, n)) throw Error("number expected");
+    auto n = string2Int<int>(s);
-    while (n--) {
+    if (!n) throw Error("number expected");
+    while ((*n)--) {
         getline(str, s);
         info.references.insert(store.parseStorePath(s));
     }
@@ -1066,26 +1067,23 @@ Derivation Store::derivationFromPath(const StorePath & drvPath)
     return readDerivation(drvPath);
 }
 
-
+Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath)
-Derivation Store::readDerivation(const StorePath & drvPath)
 {
-    auto accessor = getFSAccessor();
+    auto accessor = store.getFSAccessor();
     try {
-        return parseDerivation(*this,
+        return parseDerivation(store,
-            accessor->readFile(printStorePath(drvPath)),
+            accessor->readFile(store.printStorePath(drvPath), requireValidPath),
             Derivation::nameFromPath(drvPath));
     } catch (FormatError & e) {
-        throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
+        throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg());
     }
 }
 
+Derivation Store::readDerivation(const StorePath & drvPath)
+{ return readDerivationCommon(*this, drvPath, true); }
+
 Derivation Store::readInvalidDerivation(const StorePath & drvPath)
-{
+{ return readDerivationCommon(*this, drvPath, false); }
-    return parseDerivation(
-        *this,
-        readFile(Store::toRealPath(drvPath)),
-        Derivation::nameFromPath(drvPath));
-}
 
 }
 

src/libstore/store-api.hh

@@ -175,25 +175,7 @@ struct StoreConfig : public Config
 {
     using Config::Config;
 
-    /**
+    StoreConfig() = delete;
-     * When constructing a store implementation, we pass in a map `params` of
-     * parameters that's supposed to initialize the associated config.
-     * To do that, we must use the `StoreConfig(StringMap & params)`
-     * constructor, so we'd like to `delete` its default constructor to enforce
-     * it.
-     *
-     * However, actually deleting it means that all the subclasses of
-     * `StoreConfig` will have their default constructor deleted (because it's
-     * supposed to call the deleted default constructor of `StoreConfig`). But
-     * because we're always using virtual inheritance, the constructors of
-     * child classes will never implicitely call this one, so deleting it will
-     * be more painful than anything else.
-     *
-     * So we `assert(false)` here to ensure at runtime that the right
-     * constructor is always called without having to redefine a custom
-     * constructor for each `*Config` class.
-     */
-    StoreConfig() { assert(false); }
 
     virtual ~StoreConfig() { }
 
@@ -624,6 +606,11 @@ public:
     virtual ref<FSAccessor> getFSAccessor()
     { unsupported("getFSAccessor"); }
 
+    /* Repair the contents of the given path by redownloading it using
+       a substituter (if available). */
+    virtual void repairPath(const StorePath & path)
+    { unsupported("repairPath"); }
+
     /* Add signatures to the specified store path. The signatures are
        not verified. */
     virtual void addSignatures(const StorePath & storePath, const StringSet & sigs)

src/libstore/uds-remote-store.cc

@@ -15,6 +15,9 @@ namespace nix {
 
 UDSRemoteStore::UDSRemoteStore(const Params & params)
     : StoreConfig(params)
+    , LocalFSStoreConfig(params)
+    , RemoteStoreConfig(params)
+    , UDSRemoteStoreConfig(params)
     , Store(params)
     , LocalFSStore(params)
     , RemoteStore(params)

src/libstore/uds-remote-store.hh

@@ -14,15 +14,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
     {
     }
 
-    UDSRemoteStoreConfig()
-        : UDSRemoteStoreConfig(Store::Params({}))
-    {
-    }
-
     const std::string name() override { return "Local Daemon Store"; }
 };
 
-class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
+class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore
 {
 public:
 

src/libutil/args.cc

@@ -254,6 +254,8 @@ nlohmann::json Args::toJSON()
     res["description"] = description();
     res["flags"] = std::move(flags);
     res["args"] = std::move(args);
+    auto s = doc();
+    if (s != "") res.emplace("doc", stripIndentation(s));
     return res;
 }
 
@@ -351,38 +353,6 @@ void printTable(std::ostream & out, const Table2 & table)
     }
 }
 
-void Command::printHelp(const string & programName, std::ostream & out)
-{
-    Args::printHelp(programName, out);
-
-    auto exs = examples();
-    if (!exs.empty()) {
-        out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n";
-        for (auto & ex : exs)
-            out << "\n"
-                << "  " << ex.description << "\n" // FIXME: wrap
-                << "  $ " << ex.command << "\n";
-    }
-}
-
-nlohmann::json Command::toJSON()
-{
-    auto exs = nlohmann::json::array();
-
-    for (auto & example : examples()) {
-        auto ex = nlohmann::json::object();
-        ex["description"] = example.description;
-        ex["command"] = chomp(stripIndentation(example.command));
-        exs.push_back(std::move(ex));
-    }
-
-    auto res = Args::toJSON();
-    res["examples"] = std::move(exs);
-    auto s = doc();
-    if (s != "") res.emplace("doc", stripIndentation(s));
-    return res;
-}
-
 MultiCommand::MultiCommand(const Commands & commands)
     : commands(commands)
 {

src/libutil/args.hh

@@ -25,6 +25,9 @@ public:
     /* Return a short one-line description of the command. */
     virtual std::string description() { return ""; }
 
+    /* Return documentation about this command, in Markdown format. */
+    virtual std::string doc() { return ""; }
+
 protected:
 
     static const size_t ArityAny = std::numeric_limits<size_t>::max();
@@ -65,8 +68,12 @@ protected:
             , arity(ArityAny)
         { }
 
-        template<class T>
+        Handler(std::string * dest)
-        Handler(T * dest)
+            : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
+            , arity(1)
+        { }
+
+        Handler(std::optional<std::string> * dest)
             : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
             , arity(1)
         { }
@@ -76,6 +83,14 @@ protected:
             : fun([=](std::vector<std::string> ss) { *dest = val; })
             , arity(0)
         { }
+
+        template<class I>
+        Handler(I * dest)
+            : fun([=](std::vector<std::string> ss) {
+                *dest = string2IntWithUnitPrefix<I>(ss[0]);
+              })
+            , arity(1)
+        { }
     };
 
     /* Flags. */
@@ -127,19 +142,6 @@ public:
     /* Helper functions for constructing flags / positional
        arguments. */
 
-    void mkFlag1(char shortName, const std::string & longName,
-        const std::string & label, const std::string & description,
-        std::function<void(std::string)> fun)
-    {
-        addFlag({
-            .longName = longName,
-            .shortName = shortName,
-            .description = description,
-            .labels = {label},
-            .handler = {[=](std::string s) { fun(s); }}
-        });
-    }
-
     void mkFlag(char shortName, const std::string & name,
         const std::string & description, bool * dest)
     {
@@ -158,33 +160,6 @@ public:
         });
     }
 
-    template<class I>
-    void mkIntFlag(char shortName, const std::string & longName,
-        const std::string & description, I * dest)
-    {
-        mkFlag<I>(shortName, longName, description, [=](I n) {
-            *dest = n;
-        });
-    }
-
-    template<class I>
-    void mkFlag(char shortName, const std::string & longName,
-        const std::string & description, std::function<void(I)> fun)
-    {
-        addFlag({
-            .longName = longName,
-            .shortName = shortName,
-            .description = description,
-            .labels = {"N"},
-            .handler = {[=](std::string s) {
-                I n;
-                if (!string2Int(s, n))
-                    throw UsageError("flag '--%s' requires a integer argument", longName);
-                fun(n);
-            }}
-        });
-    }
-
     void expectArgs(ExpectedArg && arg)
     {
         expectedArgs.emplace_back(std::move(arg));
@@ -225,28 +200,11 @@ struct Command : virtual Args
     virtual void prepare() { };
     virtual void run() = 0;
 
-    /* Return documentation about this command, in Markdown format. */
-    virtual std::string doc() { return ""; }
-
-    struct Example
-    {
-        std::string description;
-        std::string command;
-    };
-
-    typedef std::list<Example> Examples;
-
-    virtual Examples examples() { return Examples(); }
-
     typedef int Category;
 
     static constexpr Category catDefault = 0;
 
     virtual Category category() { return catDefault; }
-
-    void printHelp(const string & programName, std::ostream & out) override;
-
-    nlohmann::json toJSON() override;
 };
 
 typedef std::map<std::string, std::function<ref<Command>()>> Commands;

src/libutil/config.cc

@@ -230,7 +230,9 @@ template<typename T>
 void BaseSetting<T>::set(const std::string & str, bool append)
 {
     static_assert(std::is_integral<T>::value, "Integer required.");
-    if (!string2Int(str, value))
+    if (auto n = string2Int<T>(str))
+        value = *n;
+    else
         throw UsageError("setting '%s' has invalid value '%s'", name, str);
 }
 

src/libutil/error.cc

@@ -61,13 +61,11 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
     if (errPos.origin == foFile) {
         LinesOfCode loc;
         try {
+            // FIXME: when running as the daemon, make sure we don't
+            // open a file to which the client doesn't have access.
             AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
-            if (!fd) {
+            if (!fd) return {};
-                logError(SysError("opening file '%1%'", errPos.file).info());
+
-                return std::nullopt;
-            }
-            else
-            {
             // count the newlines.
             int count = 0;
             string line;
@@ -77,21 +75,18 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
                 line = readLine(fd.get());
                 ++count;
                 if (count < pl)
-                    {
                     ;
-                    }
+                else if (count == pl)
-                    else if (count == pl) {
                     loc.prevLineOfCode = line;
-                    } else if (count == pl + 1) {
+                else if (count == pl + 1)
                     loc.errLineOfCode = line;
-                    } else if (count == pl + 2) {
+                else if (count == pl + 2) {
                     loc.nextLineOfCode = line;
                     break;
                 }
             } while (true);
             return loc;
         }
-        }
         catch (EndOfFile & eof) {
             if (loc.errLineOfCode.has_value())
                 return loc;
@@ -99,7 +94,6 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
                 return std::nullopt;
         }
         catch (std::exception & e) {
-            printError("error reading nix file: %s\n%s", errPos.file, e.what());
             return std::nullopt;
         }
     } else {

src/libutil/error.hh

@@ -38,7 +38,7 @@ namespace nix {
    ErrorInfo structs are sent to the logger as part of an exception, or directly with the
    logError or logWarning macros.
 
-   See the error-demo.cc program for usage examples.
+   See libutil/tests/logging.cc for usage examples.
 
  */
 

src/libutil/tests/logging.cc

@@ -49,7 +49,7 @@ namespace nix {
             });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
+        ASSERT_STREQ(str.c_str(), "@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
     }
 
     TEST(logEI, appendingHintsToPreviousError) {
@@ -208,7 +208,7 @@ namespace nix {
             });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
     TEST(logError, logErrorWithOnlyHintAndName) {
@@ -290,7 +290,7 @@ namespace nix {
         logError(e.info());
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n     1| this is the other problem line of code\n      |                   \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n     1| this is the other problem line of code\n      |                   \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
     }
 
     TEST(addTrace, hideTracesWithoutShowTrace) {

src/libutil/tests/tests.cc

@@ -320,20 +320,15 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(string2Float, emptyString) {
-        double n;
+        ASSERT_EQ(string2Float<double>(""), std::nullopt);
-        ASSERT_EQ(string2Float("", n), false);
     }
 
     TEST(string2Float, trivialConversions) {
-        double n;
+        ASSERT_EQ(string2Float<double>("1.0"), 1.0);
-        ASSERT_EQ(string2Float("1.0", n), true);
-        ASSERT_EQ(n, 1.0);
 
-        ASSERT_EQ(string2Float("0.0", n), true);
+        ASSERT_EQ(string2Float<double>("0.0"), 0.0);
-        ASSERT_EQ(n, 0.0);
 
-        ASSERT_EQ(string2Float("-100.25", n), true);
+        ASSERT_EQ(string2Float<double>("-100.25"), -100.25);
-        ASSERT_EQ(n, (-100.25));
     }
 
     /* ----------------------------------------------------------------------------
@@ -341,20 +336,15 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(string2Int, emptyString) {
-        double n;
+        ASSERT_EQ(string2Int<int>(""), std::nullopt);
-        ASSERT_EQ(string2Int("", n), false);
     }
 
     TEST(string2Int, trivialConversions) {
-        double n;
+        ASSERT_EQ(string2Int<int>("1"), 1);
-        ASSERT_EQ(string2Int("1", n), true);
-        ASSERT_EQ(n, 1);
 
-        ASSERT_EQ(string2Int("0", n), true);
+        ASSERT_EQ(string2Int<int>("0"), 0);
-        ASSERT_EQ(n, 0);
 
-        ASSERT_EQ(string2Int("-100", n), true);
+        ASSERT_EQ(string2Int<int>("-100"), -100);
-        ASSERT_EQ(n, (-100));
     }
 
     /* ----------------------------------------------------------------------------

src/libutil/url-parts.hh

@@ -23,7 +23,7 @@ const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
 const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
 
 // A Git ref (i.e. branch or tag name).
-const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
+const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check
 extern std::regex refRegex;
 
 // Instead of defining what a good Git Ref is, we define what a bad Git Ref is

src/libutil/util.hh

@@ -397,21 +397,49 @@ bool statusOk(int status);
 
 
 /* Parse a string into an integer. */
-template<class N> bool string2Int(const string & s, N & n)
+template<class N>
+std::optional<N> string2Int(const std::string & s)
 {
-    if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed)
+    if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
-        return false;
+        return std::nullopt;
     std::istringstream str(s);
+    N n;
     str >> n;
-    return str && str.get() == EOF;
+    if (str && str.get() == EOF) return n;
+    return std::nullopt;
+}
+
+/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or
+   'T' denoting a binary unit prefix. */
+template<class N>
+N string2IntWithUnitPrefix(std::string s)
+{
+    N multiplier = 1;
+    if (!s.empty()) {
+        char u = std::toupper(*s.rbegin());
+        if (std::isalpha(u)) {
+            if (u == 'K') multiplier = 1ULL << 10;
+            else if (u == 'M') multiplier = 1ULL << 20;
+            else if (u == 'G') multiplier = 1ULL << 30;
+            else if (u == 'T') multiplier = 1ULL << 40;
+            else throw UsageError("invalid unit specifier '%1%'", u);
+            s.resize(s.size() - 1);
+        }
+    }
+    if (auto n = string2Int<N>(s))
+        return *n * multiplier;
+    throw UsageError("'%s' is not an integer", s);
 }
 
 /* Parse a string into a float. */
-template<class N> bool string2Float(const string & s, N & n)
+template<class N>
+std::optional<N> string2Float(const string & s)
 {
     std::istringstream str(s);
+    N n;
     str >> n;
-    return str && str.get() == EOF;
+    if (str && str.get() == EOF) return n;
+    return std::nullopt;
 }
 
 

src/nix-build/nix-build.cc

@@ -217,9 +217,9 @@ static void main_nix_build(int argc, char * * argv)
                 // read the shebang to understand which packages to read from. Since
                 // this is handled via nix-shell -p, we wrap our ruby script execution
                 // in ruby -e 'load' which ignores the shebangs.
-                envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str();
+                envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str();
             } else {
-                envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str();
+                envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str();
             }
         }
 

src/nix-env/nix-env.cc

@@ -1138,38 +1138,38 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                                         i.queryName(), j)
                                 });
                             else {
-                                if (v->type == tString) {
+                                if (v->type() == nString) {
                                     attrs2["type"] = "string";
                                     attrs2["value"] = v->string.s;
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->type == tInt) {
+                                } else if (v->type() == nInt) {
                                     attrs2["type"] = "int";
                                     attrs2["value"] = (format("%1%") % v->integer).str();
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->type == tFloat) {
+                                } else if (v->type() == nFloat) {
                                     attrs2["type"] = "float";
                                     attrs2["value"] = (format("%1%") % v->fpoint).str();
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->type == tBool) {
+                                } else if (v->type() == nBool) {
                                     attrs2["type"] = "bool";
                                     attrs2["value"] = v->boolean ? "true" : "false";
                                     xml.writeEmptyElement("meta", attrs2);
-                                } else if (v->isList()) {
+                                } else if (v->type() == nList) {
                                     attrs2["type"] = "strings";
                                     XMLOpenElement m(xml, "meta", attrs2);
                                     for (unsigned int j = 0; j < v->listSize(); ++j) {
-                                        if (v->listElems()[j]->type != tString) continue;
+                                        if (v->listElems()[j]->type() != nString) continue;
                                         XMLAttrs attrs3;
                                         attrs3["value"] = v->listElems()[j]->string.s;
                                         xml.writeEmptyElement("string", attrs3);
                                     }
-                              } else if (v->type == tAttrs) {
+                              } else if (v->type() == nAttrs) {
                                   attrs2["type"] = "strings";
                                   XMLOpenElement m(xml, "meta", attrs2);
                                   Bindings & attrs = *v->attrs;
                                   for (auto &i : attrs) {
                                       Attr & a(*attrs.find(i.name));
-                                      if(a.value->type != tString) continue;
+                                      if(a.value->type() != nString) continue;
                                       XMLAttrs attrs3;
                                       attrs3["type"] = i.name;
                                       attrs3["value"] = a.value->string.s;
@@ -1250,11 +1250,10 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg
     if (opArgs.size() != 1)
         throw UsageError("exactly one argument expected");
 
-    GenerationNumber dstGen;
+    if (auto dstGen = string2Int<GenerationNumber>(opArgs.front()))
-    if (!string2Int(opArgs.front(), dstGen))
+        switchGeneration(globals, *dstGen);
+    else
         throw UsageError("expected a generation number");
-
-    switchGeneration(globals, dstGen);
 }
 
 
@@ -1308,17 +1307,17 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
         if(opArgs.front().size() < 2)
             throw Error("invalid number of generations ‘%1%’", opArgs.front());
         string str_max = string(opArgs.front(), 1, opArgs.front().size());
-        GenerationNumber max;
+        auto max = string2Int<GenerationNumber>(str_max);
-        if (!string2Int(str_max, max) || max == 0)
+        if (!max || *max == 0)
             throw Error("invalid number of generations to keep ‘%1%’", opArgs.front());
-        deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun);
+        deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun);
     } else {
         std::set<GenerationNumber> gens;
         for (auto & i : opArgs) {
-            GenerationNumber n;
+            if (auto n = string2Int<GenerationNumber>(i))
-            if (!string2Int(i, n))
+                gens.insert(*n);
+            else
                 throw UsageError("invalid generation number '%1%'", i);
-            gens.insert(n);
         }
         deleteGenerations(globals.profile, gens, globals.dryRun);
     }

src/nix-env/user-env.cc

@@ -53,10 +53,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
            output paths, and optionally the derivation path, as well
            as the meta attributes. */
         Path drvPath = keepDerivations ? i.queryDrvPath() : "";
+        DrvInfo::Outputs outputs = i.queryOutputs(true);
+        StringSet metaNames = i.queryMetaNames();
 
         Value & v(*state.allocValue());
         manifest.listElems()[n++] = &v;
-        state.mkAttrs(v, 16);
+        state.mkAttrs(v, 7 + outputs.size());
 
         mkString(*state.allocAttr(v, state.sType), "derivation");
         mkString(*state.allocAttr(v, state.sName), i.queryName());
@@ -68,7 +70,6 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
             mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath());
 
         // Copy each output meant for installation.
-        DrvInfo::Outputs outputs = i.queryOutputs(true);
         Value & vOutputs = *state.allocAttr(v, state.sOutputs);
         state.mkList(vOutputs, outputs.size());
         unsigned int m = 0;
@@ -88,8 +89,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
         // Copy the meta attributes.
         Value & vMeta = *state.allocAttr(v, state.sMeta);
-        state.mkAttrs(vMeta, 16);
+        state.mkAttrs(vMeta, metaNames.size());
-        StringSet metaNames = i.queryMetaNames();
         for (auto & j : metaNames) {
             Value * v = i.queryMeta(j);
             if (!v) continue;

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -1,232 +0,0 @@
-#include "hash.hh"
-#include "shared.hh"
-#include "filetransfer.hh"
-#include "store-api.hh"
-#include "eval.hh"
-#include "eval-inline.hh"
-#include "common-eval-args.hh"
-#include "attr-path.hh"
-#include "finally.hh"
-#include "../nix/legacy.hh"
-#include "progress-bar.hh"
-#include "tarfile.hh"
-
-#include <iostream>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-using namespace nix;
-
-
-/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of
-   mirrors defined in Nixpkgs. */
-string resolveMirrorUri(EvalState & state, string uri)
-{
-    if (string(uri, 0, 9) != "mirror://") return uri;
-
-    string s(uri, 9);
-    auto p = s.find('/');
-    if (p == string::npos) throw Error("invalid mirror URI");
-    string mirrorName(s, 0, p);
-
-    Value vMirrors;
-    state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors);
-    state.forceAttrs(vMirrors);
-
-    auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName));
-    if (mirrorList == vMirrors.attrs->end())
-        throw Error("unknown mirror name '%1%'", mirrorName);
-    state.forceList(*mirrorList->value);
-
-    if (mirrorList->value->listSize() < 1)
-        throw Error("mirror URI '%1%' did not expand to anything", uri);
-
-    string mirror = state.forceString(*mirrorList->value->listElems()[0]);
-    return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1);
-}
-
-
-static int main_nix_prefetch_url(int argc, char * * argv)
-{
-    {
-        HashType ht = htSHA256;
-        std::vector<string> args;
-        bool printPath = getEnv("PRINT_PATH") == "1";
-        bool fromExpr = false;
-        string attrPath;
-        bool unpack = false;
-        bool executable = false;
-        string name;
-
-        struct MyArgs : LegacyArgs, MixEvalArgs
-        {
-            using LegacyArgs::LegacyArgs;
-        };
-
-        MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) {
-            if (*arg == "--help")
-                showManPage("nix-prefetch-url");
-            else if (*arg == "--version")
-                printVersion("nix-prefetch-url");
-            else if (*arg == "--type") {
-                string s = getArg(*arg, arg, end);
-                ht = parseHashType(s);
-            }
-            else if (*arg == "--print-path")
-                printPath = true;
-            else if (*arg == "--attr" || *arg == "-A") {
-                fromExpr = true;
-                attrPath = getArg(*arg, arg, end);
-            }
-            else if (*arg == "--unpack")
-                unpack = true;
-            else if (*arg == "--executable")
-                executable = true;
-            else if (*arg == "--name")
-                name = getArg(*arg, arg, end);
-            else if (*arg != "" && arg->at(0) == '-')
-                return false;
-            else
-                args.push_back(*arg);
-            return true;
-        });
-
-        myArgs.parseCmdline(argvToStrings(argc, argv));
-
-        initPlugins();
-
-        if (args.size() > 2)
-            throw UsageError("too many arguments");
-
-        Finally f([]() { stopProgressBar(); });
-
-        if (isatty(STDERR_FILENO))
-          startProgressBar();
-
-        auto store = openStore();
-        auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
-
-        Bindings & autoArgs = *myArgs.getAutoArgs(*state);
-
-        /* If -A is given, get the URI from the specified Nix
-           expression. */
-        string uri;
-        if (!fromExpr) {
-            if (args.empty())
-                throw UsageError("you must specify a URI");
-            uri = args[0];
-        } else {
-            Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0]));
-            Value vRoot;
-            state->evalFile(path, vRoot);
-            Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first);
-            state->forceAttrs(v);
-
-            /* Extract the URI. */
-            auto attr = v.attrs->find(state->symbols.create("urls"));
-            if (attr == v.attrs->end())
-                throw Error("attribute set does not contain a 'urls' attribute");
-            state->forceList(*attr->value);
-            if (attr->value->listSize() < 1)
-                throw Error("'urls' list is empty");
-            uri = state->forceString(*attr->value->listElems()[0]);
-
-            /* Extract the hash mode. */
-            attr = v.attrs->find(state->symbols.create("outputHashMode"));
-            if (attr == v.attrs->end())
-                printInfo("warning: this does not look like a fetchurl call");
-            else
-                unpack = state->forceString(*attr->value) == "recursive";
-
-            /* Extract the name. */
-            if (name.empty()) {
-                attr = v.attrs->find(state->symbols.create("name"));
-                if (attr != v.attrs->end())
-                    name = state->forceString(*attr->value);
-            }
-        }
-
-        /* Figure out a name in the Nix store. */
-        if (name.empty())
-            name = baseNameOf(uri);
-        if (name.empty())
-            throw Error("cannot figure out file name for '%1%'", uri);
-
-        /* If an expected hash is given, the file may already exist in
-           the store. */
-        std::optional<Hash> expectedHash;
-        Hash hash(ht);
-        std::optional<StorePath> storePath;
-        if (args.size() == 2) {
-            expectedHash = Hash::parseAny(args[1], ht);
-            const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
-            if (store->isValidPath(*storePath))
-                hash = *expectedHash;
-            else
-                storePath.reset();
-        }
-
-        if (!storePath) {
-
-            auto actualUri = resolveMirrorUri(*state, uri);
-
-            AutoDelete tmpDir(createTempDir(), true);
-            Path tmpFile = (Path) tmpDir + "/tmp";
-
-            /* Download the file. */
-            {
-                auto mode = 0600;
-                if (executable)
-                    mode = 0700;
-
-                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode);
-                if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
-
-                FdSink sink(fd.get());
-
-                FileTransferRequest req(actualUri);
-                req.decompress = false;
-                getFileTransfer()->download(std::move(req), sink);
-            }
-
-            /* Optionally unpack the file. */
-            if (unpack) {
-                printInfo("unpacking...");
-                Path unpacked = (Path) tmpDir + "/unpacked";
-                createDirs(unpacked);
-                unpackTarfile(tmpFile, unpacked);
-
-                /* If the archive unpacks to a single file/directory, then use
-                   that as the top-level. */
-                auto entries = readDirectory(unpacked);
-                if (entries.size() == 1)
-                    tmpFile = unpacked + "/" + entries[0].name;
-                else
-                    tmpFile = unpacked;
-            }
-
-            const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-
-            auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
-            storePath = info.path;
-            assert(info.ca);
-            hash = getContentAddressHash(*info.ca);
-        }
-
-        stopProgressBar();
-
-        if (!printPath)
-            printInfo("path is '%s'", store->printStorePath(*storePath));
-
-        std::cout << printHash16or32(hash) << std::endl;
-        if (printPath)
-            std::cout << store->printStorePath(*storePath) << std::endl;
-
-        return 0;
-    }
-}
-
-static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url);

src/nix-store/nix-store.cc

@@ -19,10 +19,6 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
-#if HAVE_SODIUM
-#include <sodium.h>
-#endif
-
 
 namespace nix_store {
 
@@ -761,7 +757,7 @@ static void opRepairPath(Strings opFlags, Strings opArgs)
         throw UsageError("no flags expected");
 
     for (auto & i : opArgs)
-        ensureLocalStore()->repairPath(store->followLinksToStorePath(i));
+        store->repairPath(store->followLinksToStorePath(i));
 }
 
 /* Optimise the disk space usage of the Nix store by hard-linking
@@ -980,21 +976,11 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs)
     string secretKeyFile = *i++;
     string publicKeyFile = *i++;
 
-#if HAVE_SODIUM
+    auto secretKey = SecretKey::generate(keyName);
-    if (sodium_init() == -1)
-        throw Error("could not initialise libsodium");
 
-    unsigned char pk[crypto_sign_PUBLICKEYBYTES];
+    writeFile(publicKeyFile, secretKey.toPublicKey().to_string());
-    unsigned char sk[crypto_sign_SECRETKEYBYTES];
-    if (crypto_sign_keypair(pk, sk) != 0)
-        throw Error("key generation failed");
-
-    writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES)));
     umask(0077);
-    writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES)));
+    writeFile(secretKeyFile, secretKey.to_string());
-#else
-    throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
-#endif
 }
 
 

src/nix/add-to-store.cc

@@ -19,7 +19,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         addFlag({
             .longName = "name",
             .shortName = 'n',
-            .description = "name component of the store path",
+            .description = "Override the name component of the store path. It defaults to the base name of *path*.",
             .labels = {"name"},
             .handler = {&namePart},
         });

src/nix/build.cc

@@ -19,7 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
         addFlag({
             .longName = "out-link",
             .shortName = 'o',
-            .description = "path of the symlink to the build result",
+            .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.",
             .labels = {"path"},
             .handler = {&outLink},
             .completer = completePath
@@ -27,13 +27,13 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 
         addFlag({
             .longName = "no-link",
-            .description = "do not create a symlink to the build result",
+            .description = "Do not create symlinks to the build results.",
             .handler = {&outLink, Path("")},
         });
 
         addFlag({
             .longName = "rebuild",
-            .description = "rebuild an already built package and compare the result to the existing store paths",
+            .description = "Rebuild an already built package and compare the result to the existing store paths.",
             .handler = {&buildMode, bmCheck},
         });
     }
@@ -43,22 +43,11 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
         return "build a derivation or fetch a store path";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
+        return
-            Example{
+          #include "build.md"
-                "To build and run GNU Hello from NixOS 17.03:",
+          ;
-                "nix build -f channel:nixos-17.03 hello; ./result/bin/hello"
-            },
-            Example{
-                "To build the build.x86_64-linux attribute from release.nix:",
-                "nix build -f release.nix build.x86_64-linux"
-            },
-            Example{
-                "To make a profile point at GNU Hello:",
-                "nix build --profile /tmp/profile nixpkgs#hello"
-            },
-        };
     }
 
     void run(ref<Store> store) override
@@ -69,7 +58,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                for (size_t i = 0; i < buildables.size(); ++i)
+                for (const auto & [_i, buildable] : enumerate(buildables)) {
+                    auto i = _i;
                     std::visit(overloaded {
                         [&](BuildableOpaque bo) {
                             std::string symlink = outLink;
@@ -85,7 +75,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                                 store2->addPermRoot(output.second, absPath(symlink));
                             }
                         },
-                    }, buildables[i]);
+                    }, buildable);
+                }
 
         updateProfile(buildables);
 

src/nix/build.md

@@ -0,0 +1,92 @@
+R""(
+
+# Examples
+
+* Build the default package from the flake in the current directory:
+
+  ```console
+  # nix build
+  ```
+
+* Build and run GNU Hello from the `nixpkgs` flake:
+
+  ```console
+  # nix build nixpkgs#hello
+  # ./result/bin/hello
+  Hello, world!
+  ```
+
+* Build GNU Hello and Cowsay, leaving two result symlinks:
+
+  ```console
+  # nix build nixpkgs#hello nixpkgs#cowsay
+  # ls -l result*
+  lrwxrwxrwx 1 … result -> /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10
+  lrwxrwxrwx 1 … result-1 -> /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2
+  ```
+
+* Build a specific output:
+
+  ```console
+  # nix build nixpkgs#glibc.dev
+  # ls -ld ./result-dev
+  lrwxrwxrwx 1 … ./result-dev -> /nix/store/dkm3gwl0xrx0wrw6zi5x3px3lpgjhlw4-glibc-2.32-dev
+  ```
+
+* Build attribute `build.x86_64-linux` from (non-flake) Nix expression
+  `release.nix`:
+
+  ```console
+  # nix build -f release.nix build.x86_64-linux
+  ```
+
+* Build a NixOS system configuration from a flake, and make a profile
+  point to the result:
+
+  ```console
+  # nix build --profile /nix/var/nix/profiles/system \
+      ~/my-configurations#nixosConfigurations.machine.config.system.build.toplevel
+  ```
+
+  (This is essentially what `nixos-rebuild` does.)
+
+* Build an expression specified on the command line:
+
+  ```console
+  # nix build --impure --expr \
+      'with import <nixpkgs> {};
+       runCommand "foo" {
+         buildInputs = [ hello ];
+       }
+       "hello > $out"'
+  # cat ./result
+  Hello, world!
+  ```
+
+  Note that `--impure` is needed because we're using `<nixpkgs>`,
+  which relies on the `$NIX_PATH` environment variable.
+
+* Fetch a store path from the configured substituters, if it doesn't
+  already exist:
+
+  ```console
+  # nix build /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2
+  ```
+
+# Description
+
+`nix build` builds the specified *installables*. Installables that
+resolve to derivations are built (or substituted if possible). Store
+path installables are substituted.
+
+Unless `--no-link` is specified, after a successful build, it creates
+symlinks to the store paths of the installables. These symlinks have
+the prefix `./result` by default; this can be overriden using the
+`--out-link` option. Each symlink has a suffix `-<N>-<outname>`, where
+*N* is the index of the installable (with the left-most installable
+having index 0), and *outname* is the symbolic derivation output name
+(e.g. `bin`, `dev` or `lib`). `-<N>` is omitted if *N* = 0, and
+`-<outname>` is omitted if *outname* = `out` (denoting the default
+output).
+
+)""

src/nix/bundle.cc

@@ -16,7 +16,7 @@ struct CmdBundle : InstallableCommand
     {
         addFlag({
             .longName = "bundler",
-            .description = "use custom bundler",
+            .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler),
             .labels = {"flake-url"},
             .handler = {&bundler},
             .completer = {[&](size_t, std::string_view prefix) {
@@ -27,7 +27,7 @@ struct CmdBundle : InstallableCommand
         addFlag({
             .longName = "out-link",
             .shortName = 'o',
-            .description = "path of the symlink to the build result",
+            .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.",
             .labels = {"path"},
             .handler = {&outLink},
             .completer = completePath
@@ -40,14 +40,11 @@ struct CmdBundle : InstallableCommand
         return "bundle an application so that it works outside of the Nix store";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
+        return
-            Example{
+          #include "bundle.md"
-                "To bundle Hello:",
+          ;
-                "nix bundle hello"
-            },
-        };
     }
 
     Category category() override { return catSecondary; }

src/nix/bundle.md

@@ -0,0 +1,36 @@
+R""(
+
+# Examples
+
+* Bundle Hello:
+
+  ```console
+  # nix bundle nixpkgs#hello
+  # ./hello
+  Hello, world!
+  ```
+
+* Bundle a specific version of Nix:
+
+  ```console
+  # nix bundle github:NixOS/nix/e3ddffb27e5fc37a209cfd843c6f7f6a9460a8ec
+  # ./nix --version
+  nix (Nix) 2.4pre20201215_e3ddffb
+  ```
+
+# Description
+
+`nix bundle` packs the closure of the [Nix app](./nix3-run.md)
+*installable* into a single self-extracting executable. See the
+[`nix-bundle` homepage](https://github.com/matthewbauer/nix-bundle)
+for more details.
+
+> **Note**
+>
+> This command only works on Linux.
+
+# Bundler definitions
+
+TODO
+
+)""

src/nix/cat.cc

@@ -37,6 +37,13 @@ struct CmdCatStore : StoreCommand, MixCat
         return "print the contents of a file in the Nix store on stdout";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "store-cat.md"
+          ;
+    }
+
     void run(ref<Store> store) override
     {
         cat(store->getFSAccessor());
@@ -62,6 +69,13 @@ struct CmdCatNar : StoreCommand, MixCat
         return "print the contents of a file inside a NAR file on stdout";
     }
 
+    std::string doc() override
+    {
+        return
+          #include "nar-cat.md"
+          ;
+    }
+
     void run(ref<Store> store) override
     {
         cat(makeNarAccessor(make_ref<std::string>(readFile(narPath))));

src/nix/command.cc

@@ -65,18 +65,18 @@ StorePathsCommand::StorePathsCommand(bool recursive)
     if (recursive)
         addFlag({
             .longName = "no-recursive",
-            .description = "apply operation to specified paths only",
+            .description = "Apply operation to specified paths only.",
             .handler = {&this->recursive, false},
         });
     else
         addFlag({
             .longName = "recursive",
             .shortName = 'r',
-            .description = "apply operation to closure of the specified paths",
+            .description = "Apply operation to closure of the specified paths.",
             .handler = {&this->recursive, true},
         });
 
-    mkFlag(0, "all", "apply operation to the entire store", &all);
+    mkFlag(0, "all", "Apply the operation to every store path.", &all);
 }
 
 void StorePathsCommand::run(ref<Store> store)
@@ -133,7 +133,7 @@ MixProfile::MixProfile()
 {
     addFlag({
         .longName = "profile",
-        .description = "profile to update",
+        .description = "The profile to update.",
         .labels = {"path"},
         .handler = {&profile},
         .completer = completePath
@@ -190,14 +190,14 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
     addFlag({
         .longName = "ignore-environment",
         .shortName = 'i',
-        .description = "clear the entire environment (except those specified with --keep)",
+        .description = "Clear the entire environment (except those specified with `--keep`).",
         .handler = {&ignoreEnvironment, true},
     });
 
     addFlag({
         .longName = "keep",
         .shortName = 'k',
-        .description = "keep specified environment variable",
+        .description = "Keep the environment variable *name*.",
         .labels = {"name"},
         .handler = {[&](std::string s) { keep.insert(s); }},
     });
@@ -205,7 +205,7 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
     addFlag({
         .longName = "unset",
         .shortName = 'u',
-        .description = "unset specified environment variable",
+        .description = "Unset the environment variable *name*.",
         .labels = {"name"},
         .handler = {[&](std::string s) { unset.insert(s); }},
     });

src/nix/command.hh

@@ -13,6 +13,8 @@ namespace nix {
 
 extern std::string programPath;
 
+extern char * * savedArgv;
+
 class EvalState;
 struct Pos;
 class Store;
@@ -261,6 +263,8 @@ void completeFlakeRefWithFragment(
     const Strings & defaultFlakeAttrPaths,
     std::string_view prefix);
 
+std::string showVersions(const std::set<std::string> & versions);
+
 void printClosureDiff(
     ref<Store> store,
     const StorePath & beforePath,

src/nix/copy.cc

@@ -21,28 +21,28 @@ struct CmdCopy : StorePathsCommand
     {
         addFlag({
             .longName = "from",
-            .description = "URI of the source Nix store",
+            .description = "URL of the source Nix store.",
             .labels = {"store-uri"},
             .handler = {&srcUri},
         });
 
         addFlag({
             .longName = "to",
-            .description = "URI of the destination Nix store",
+            .description = "URL of the destination Nix store.",
             .labels = {"store-uri"},
             .handler = {&dstUri},
         });
 
         addFlag({
             .longName = "no-check-sigs",
-            .description = "do not require that paths are signed by trusted keys",
+            .description = "Do not require that paths are signed by trusted keys.",
             .handler = {&checkSigs, NoCheckSigs},
         });
 
         addFlag({
             .longName = "substitute-on-destination",
             .shortName = 's',
-            .description = "whether to try substitutes on the destination store (only supported by SSH)",
+            .description = "Whether to try substitutes on the destination store (only supported by SSH stores).",
             .handler = {&substitute, Substitute},
         });
 
@@ -54,32 +54,11 @@ struct CmdCopy : StorePathsCommand
         return "copy paths between Nix stores";
     }
 
-    Examples examples() override
+    std::string doc() override
     {
-        return {
+        return
-            Example{
+          #include "copy.md"
-                "To copy Firefox from the local store to a binary cache in file:///tmp/cache:",
+          ;
-                "nix copy --to file:///tmp/cache $(type -p firefox)"
-            },
-            Example{
-                "To copy the entire current NixOS system closure to another machine via SSH:",
-                "nix copy --to ssh://server /run/current-system"
-            },
-            Example{
-                "To copy a closure from another machine via SSH:",
-                "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"
-            },
-#ifdef ENABLE_S3
-            Example{
-                "To copy Hello to an S3 binary cache:",
-                "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello"
-            },
-            Example{
-                "To copy Hello to an S3-compatible binary cache:",
-                "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello"
-            },
-#endif
-        };
     }
 
     Category category() override { return catSecondary; }

src/nix/copy.md

@@ -0,0 +1,58 @@
+R""(
+
+# Examples
+
+* Copy Firefox from the local store to a binary cache in `/tmp/cache`:
+
+  ```console
+  # nix copy --to file:///tmp/cache $(type -p firefox)
+  ```
+
+  Note the `file://` - without this, the destination is a chroot
+  store, not a binary cache.
+
+* Copy the entire current NixOS system closure to another machine via
+  SSH:
+
+  ```console
+  # nix copy -s --to ssh://server /run/current-system
+  ```
+
+  The `-s` flag causes the remote machine to try to substitute missing
+  store paths, which may be faster if the link between the local and
+  remote machines is slower than the link between the remote machine
+  and its substituters (e.g. `https://cache.nixos.org`).
+
+* Copy a closure from another machine via SSH:
+
+  ```console
+  # nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2
+  ```
+
+* Copy Hello to a binary cache in an Amazon S3 bucket:
+
+  ```console
+  # nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello
+  ```
+
+  or to an S3-compatible storage system:
+
+  ```console
+  # nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello
+  ```
+
+  Note that this only works if Nix is built with AWS support.
+
+* Copy a closure from `/nix/store` to the chroot store `/tmp/nix/nix/store`:
+
+  ```console
+  # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs
+  ```
+
+# Description
+
+`nix copy` copies store path closures between two Nix stores. The
+source store is specified using `--from` and the destination using
+`--to`. If one of these is omitted, it defaults to the local store.
+
+)""