From 703d863a48f549b2626382eda407ffae779f8725 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Dec 2022 12:58:58 +0100 Subject: [PATCH] Trivial changes from the lazy-trees branch --- configure.ac | 2 +- doc/manual/src/command-ref/env-common.md | 39 ++------------ src/libcmd/common-eval-args.cc | 47 +++++++++++++++- src/libcmd/installables.cc | 3 +- src/libcmd/repl.cc | 2 +- src/libexpr/eval-cache.cc | 6 +-- src/libexpr/eval.cc | 2 +- src/libexpr/flake/flake.cc | 47 ++++++++++------ src/libexpr/flake/flake.hh | 6 +-- src/libexpr/flake/flakeref.hh | 2 +- src/libexpr/flake/lockfile.cc | 48 +++++++++-------- src/libexpr/flake/lockfile.hh | 9 ++-- src/libexpr/get-drvs.cc | 2 +- src/libexpr/nixexpr.cc | 1 - src/libexpr/primops.cc | 8 +-- src/libfetchers/fetchers.cc | 6 +-- src/libfetchers/fetchers.hh | 13 +++-- src/libfetchers/git.cc | 60 ++++++++++----------- src/libfetchers/github.cc | 53 +++++++++++------- src/libfetchers/indirect.cc | 10 ++-- src/libfetchers/mercurial.cc | 10 ++-- src/libfetchers/path.cc | 8 +-- src/libfetchers/tarball.cc | 11 ++-- src/libmain/progress-bar.cc | 2 +- src/libstore/binary-cache-store.cc | 2 +- src/libstore/build/derivation-goal.cc | 6 +-- src/libstore/build/entry-points.cc | 6 +-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/derivations.cc | 2 +- src/libstore/derivations.hh | 2 +- src/libstore/filetransfer.cc | 6 +-- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 8 +-- src/libstore/store-api.hh | 4 +- src/libutil/archive.cc | 4 -- src/libutil/archive.hh | 4 +- src/libutil/fmt.hh | 2 +- src/libutil/logging.cc | 12 +---- src/libutil/logging.hh | 8 ++- src/libutil/ref.hh | 5 ++ src/libutil/serialise.cc | 2 +- src/libutil/serialise.hh | 12 +---- src/libutil/util.cc | 15 ++++++ src/libutil/util.hh | 24 +++++++++ src/nix-store/nix-store.cc | 1 - src/nix/daemon.cc | 2 +- src/nix/flake-update.md | 2 +- src/nix/flake.cc | 4 +- src/nix/profile-list.md | 6 +-- src/nix/profile-upgrade.md | 6 +-- src/nix/profile.md | 3 +- src/nix/registry.cc | 8 ++- tests/eval.sh | 4 ++ tests/fetchGit.sh | 1 + tests/flakes/absolute-paths.sh | 17 ++++++ tests/flakes/flakes.sh | 13 +++-- tests/flakes/unlocked-override.sh | 30 +++++++++++ tests/function-trace.sh | 2 +- tests/local.mk | 5 +- tests/nix_path.sh | 3 ++ tests/restricted.sh | 2 +- tests/toString-path.sh | 8 +++ 62 files changed, 394 insertions(+), 248 deletions(-) create mode 100644 tests/flakes/absolute-paths.sh create mode 100644 tests/flakes/unlocked-override.sh create mode 100644 tests/toString-path.sh diff --git a/configure.ac b/configure.ac index 64fa12fc7..c0e989d85 100644 --- a/configure.ac +++ b/configure.ac @@ -177,7 +177,7 @@ fi PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) -# Checks for libarchive +# Look for libarchive. PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed if test "$shared" != yes; then diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 3f3eb6915..6947dbf4c 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -8,41 +8,10 @@ Most Nix commands interpret the following environment variables: - [`NIX_PATH`]{#env-NIX_PATH}\ A colon-separated list of directories used to look up Nix - expressions enclosed in angle brackets (i.e., ``). For - instance, the value - - /home/eelco/Dev:/etc/nixos - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in this order. It is also possible to match paths - against a prefix. For example, the value - - nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or - `https://`, it is interpreted as the URL of a tarball that will be - downloaded and unpacked to a temporary location. The tarball must - consist of a single top-level directory. For example, setting - `NIX_PATH` to - - nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - - tells Nix to download and use the current contents of the - `master` branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official nixos.org channels (see - [the manual for `nix-channel`](nix-channel.md)) can be abbreviated - as `channel:`. For instance, the following two - values of `NIX_PATH` are equivalent: - - nixpkgs=channel:nixos-21.05 - nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - - The Nix search path can also be extended using the `-I` option to - many Nix commands, which takes precedence over `NIX_PATH`. + expressions enclosed in angle brackets (i.e., ``), + e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the + `-I` option. For more information about the semantics of the Nix + search path, see the documentation for `-I`. - [`NIX_IGNORE_SYMLINK_STORE`]{#env-NIX_IGNORE_SYMLINK_STORE}\ Normally, the Nix store directory (typically `/nix/store`) is not diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 140ed3b88..2c94d7e6c 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -32,7 +32,52 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "include", .shortName = 'I', - .description = "Add *path* to the list of locations used to look up `<...>` file names.", + .description = R"( + Add *path* to the Nix search path. The Nix search path is + initialized from the colon-separated `NIX_PATH` environment + variable, and is used to look up Nix expressions enclosed in angle + brackets (i.e., ``). For instance, if the Nix search path + consists of the entries + + ``` + /home/eelco/Dev + /etc/nixos + ``` + + Nix will look for paths relative to `/home/eelco/Dev` and + `/etc/nixos`, in this order. It is also possible to match paths + against a prefix. For example, the search path + + ``` + nixpkgs=/home/eelco/Dev/nixpkgs-branch + /etc/nixos + ``` + + will cause Nix to search for `` in + `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. + + If a path in the Nix search path starts with `http://` or `https://`, + it is interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must consist of a single + top-level directory. For example, setting `NIX_PATH` to + + ``` + nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz + ``` + + tells Nix to download and use the current contents of the `master` + branch in the `nixpkgs` repository. + + The URLs of the tarballs from the official `nixos.org` channels + (see [the manual page for `nix-channel`](nix-channel.md)) can be + abbreviated as `channel:`. For instance, the + following two values of `NIX_PATH` are equivalent: + + ``` + nixpkgs=channel:nixos-21.05 + nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz + ``` + )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index dbe4a449d..f8adbf90d 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -781,7 +781,8 @@ std::vector> SourceExprCommand::parseInstallables( if (file == "-") { auto e = state->parseStdin(); state->eval(e, *vFile); - } else if (file) + } + else if (file) state->evalFile(lookupFileArg(*state, *file), *vFile); else { auto e = state->parseExprFromString(*expr, absPath(".")); diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 557952277..c704fcfb1 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -787,7 +787,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS) flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, - .allowMutable = !evalSettings.pureEval, + .allowUnlocked = !evalSettings.pureEval, }), v); addAttrsToScope(v); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index b259eec63..3e2a8665e 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -645,17 +645,17 @@ NixInt AttrCursor::getInt() cachedValue = root->db->getAttr(getKey()); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto i = std::get_if(&cachedValue->second)) { - debug("using cached Integer attribute '%s'", getAttrPathStr()); + debug("using cached integer attribute '%s'", getAttrPathStr()); return i->x; } else - throw TypeError("'%s' is not an Integer", getAttrPathStr()); + throw TypeError("'%s' is not an integer", getAttrPathStr()); } } auto & v = forceValue(); if (v.type() != nInt) - throw TypeError("'%s' is not an Integer", getAttrPathStr()); + throw TypeError("'%s' is not an integer", getAttrPathStr()); return v.integer; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 76a10b9f8..538a739af 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1806,7 +1806,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) Nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with '--arg' or '--argstr'. See -https://nixos.org/manual/nix/stable/expressions/language-constructs.html#functions.)", symbols[i.name], +https://nixos.org/manual/nix/stable/expressions/language-constructs.html#functions.)", symbols[i.name], *fun.lambda.env, *fun.lambda.fun); } } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 119c556ac..8d7d08928 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -353,7 +353,7 @@ LockedFlake lockFlake( std::function node, + ref node, const InputPath & inputPathPrefix, std::shared_ptr oldNode, const InputPath & lockRootPath, @@ -362,9 +362,15 @@ LockedFlake lockFlake( computeLocks; computeLocks = [&]( + /* The inputs of this node, either from flake.nix or + flake.lock. */ const FlakeInputs & flakeInputs, - std::shared_ptr node, + /* The node whose locks are to be updated.*/ + ref node, + /* The path to this node in the lock file graph. */ const InputPath & inputPathPrefix, + /* The old node, if any, from which locks can be + copied. */ std::shared_ptr oldNode, const InputPath & lockRootPath, const Path & parentPath, @@ -452,7 +458,7 @@ LockedFlake lockFlake( /* Copy the input from the old lock since its flakeref didn't change and there is no override from a higher level flake. */ - auto childNode = std::make_shared( + auto childNode = make_ref( oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); node->inputs.insert_or_assign(id, childNode); @@ -481,7 +487,7 @@ LockedFlake lockFlake( .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { - if (! trustLock) { + if (!trustLock) { // It is possible that the flake has changed, // so we must confirm all the follows that are in the lock file are also in the flake. auto overridePath(inputPath); @@ -521,8 +527,8 @@ LockedFlake lockFlake( this input. */ debug("creating new input '%s'", inputPathS); - if (!lockFlags.allowMutable && !input.ref->input.isLocked()) - throw Error("cannot update flake input '%s' in pure mode", inputPathS); + if (!lockFlags.allowUnlocked && !input.ref->input.isLocked()) + throw Error("cannot update unlocked flake input '%s' in pure mode", inputPathS); /* Note: in case of an --override-input, we use the *original* ref (input2.ref) for the @@ -544,7 +550,7 @@ LockedFlake lockFlake( auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath); - auto childNode = std::make_shared(inputFlake.lockedRef, ref); + auto childNode = make_ref(inputFlake.lockedRef, ref); node->inputs.insert_or_assign(id, childNode); @@ -563,16 +569,20 @@ LockedFlake lockFlake( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : LockFile::read( + : (std::shared_ptr) LockFile::read( inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, - oldLock ? lockRootPath : inputPath, localPath, false); + oldLock ? lockRootPath : inputPath, + localPath, + false); } else { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, *input.ref, useRegistries, flakeCache); - node->inputs.insert_or_assign(id, - std::make_shared(lockedRef, ref, false)); + + auto childNode = make_ref(lockedRef, ref, false); + + node->inputs.insert_or_assign(id, childNode); } } @@ -587,8 +597,13 @@ LockedFlake lockFlake( auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); computeLocks( - flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root, {}, parentPath, false); + flake.inputs, + newLockFile.root, + {}, + lockFlags.recreateLockFile ? nullptr : (std::shared_ptr) oldLockFile.root, + {}, + parentPath, + false); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) @@ -611,9 +626,9 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (auto sourcePath = topRef.input.getSourcePath()) { - if (!newLockFile.isImmutable()) { + if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) - warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); @@ -737,7 +752,7 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V .updateLockFile = false, .writeLockFile = false, .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, - .allowMutable = !evalSettings.pureEval, + .allowUnlocked = !evalSettings.pureEval, }), v); } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 524b18af1..10301d8aa 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -108,11 +108,11 @@ struct LockFlags bool applyNixConfig = false; - /* Whether mutable flake references (i.e. those without a Git + /* Whether unlocked flake references (i.e. those without a Git revision or similar) without a corresponding lock are - allowed. Mutable flake references with a lock are always + allowed. Unlocked flake references with a lock are always allowed. */ - bool allowMutable = true; + bool allowUnlocked = true; /* Whether to commit changes to flake.lock. */ bool commitLockFile = false; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index fe4f67193..a36d852a8 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -35,7 +35,7 @@ typedef std::string FlakeId; struct FlakeRef { - /* fetcher-specific representation of the input, sufficient to + /* Fetcher-specific representation of the input, sufficient to perform the fetch operation. */ fetchers::Input input; diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 629d2e669..a3ed90e1f 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -31,7 +31,7 @@ FlakeRef getFlakeRef( } LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked", "info")) + : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { @@ -49,15 +49,15 @@ std::shared_ptr LockFile::findInput(const InputPath & path) { auto pos = root; - if (!pos) return {}; - for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { if (auto node = std::get_if<0>(&*i)) pos = *node; else if (auto follows = std::get_if<1>(&*i)) { - pos = findInput(*follows); - if (!pos) return {}; + if (auto p = findInput(*follows)) + pos = ref(p); + else + return {}; } } else return {}; @@ -72,7 +72,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) if (version < 5 || version > 7) throw Error("lock file '%s' has unsupported version %d", path, version); - std::unordered_map> nodeMap; + std::map> nodeMap; std::function getInputs; @@ -93,12 +93,12 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) auto jsonNode2 = nodes.find(inputKey); if (jsonNode2 == nodes.end()) throw Error("lock file references missing node '%s'", inputKey); - auto input = std::make_shared(*jsonNode2); + auto input = make_ref(*jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; getInputs(*input, *jsonNode2); } - if (auto child = std::dynamic_pointer_cast(k->second)) - node.inputs.insert_or_assign(i.key(), child); + if (auto child = k->second.dynamic_pointer_cast()) + node.inputs.insert_or_assign(i.key(), ref(child)); else // FIXME: replace by follows node throw Error("lock file contains cycle to root node"); @@ -122,9 +122,9 @@ nlohmann::json LockFile::toJSON() const std::unordered_map, std::string> nodeKeys; std::unordered_set keys; - std::function node)> dumpNode; + std::function node)> dumpNode; - dumpNode = [&](std::string key, std::shared_ptr node) -> std::string + dumpNode = [&](std::string key, ref node) -> std::string { auto k = nodeKeys.find(node); if (k != nodeKeys.end()) @@ -159,10 +159,11 @@ nlohmann::json LockFile::toJSON() const n["inputs"] = std::move(inputs); } - if (auto lockedNode = std::dynamic_pointer_cast(node)) { + if (auto lockedNode = node.dynamic_pointer_cast()) { n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); - if (!lockedNode->isFlake) n["flake"] = false; + if (!lockedNode->isFlake) + n["flake"] = false; } nodes[key] = std::move(n); @@ -201,13 +202,13 @@ void LockFile::write(const Path & path) const writeFile(path, fmt("%s\n", *this)); } -bool LockFile::isImmutable() const +std::optional LockFile::isUnlocked() const { - std::unordered_set> nodes; + std::set> nodes; - std::function node)> visit; + std::function node)> visit; - visit = [&](std::shared_ptr node) + visit = [&](ref node) { if (!nodes.insert(node).second) return; for (auto & i : node->inputs) @@ -219,11 +220,12 @@ bool LockFile::isImmutable() const for (auto & i : nodes) { if (i == root) continue; - auto lockedNode = std::dynamic_pointer_cast(i); - if (lockedNode && !lockedNode->lockedRef.input.isLocked()) return false; + auto node = i.dynamic_pointer_cast(); + if (node && !node->lockedRef.input.isLocked()) + return node->lockedRef; } - return true; + return {}; } bool LockFile::operator ==(const LockFile & other) const @@ -247,12 +249,12 @@ InputPath parseInputPath(std::string_view s) std::map LockFile::getAllInputs() const { - std::unordered_set> done; + std::set> done; std::map res; - std::function node)> recurse; + std::function node)> recurse; - recurse = [&](const InputPath & prefix, std::shared_ptr node) + recurse = [&](const InputPath & prefix, ref node) { if (!done.insert(node).second) return; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 96f1edc76..02e9bdfbc 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -20,7 +20,7 @@ struct LockedNode; type LockedNode. */ struct Node : std::enable_shared_from_this { - typedef std::variant, InputPath> Edge; + typedef std::variant, InputPath> Edge; std::map inputs; @@ -47,11 +47,13 @@ struct LockedNode : Node struct LockFile { - std::shared_ptr root = std::make_shared(); + ref root = make_ref(); LockFile() {}; LockFile(const nlohmann::json & json, const Path & path); + typedef std::map, std::string> KeyMap; + nlohmann::json toJSON() const; std::string to_string() const; @@ -60,7 +62,8 @@ struct LockFile void write(const Path & path) const; - bool isImmutable() const; + /* Check whether this lock file has any unlocked inputs. */ + std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 346741dd5..5ad5d1fd4 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -150,7 +150,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ const Value * outTI = queryMeta("outputsToInstall"); if (!outTI) return outputs; - const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); + auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) throw errMsg; Outputs result; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 7c623a07d..2be560d76 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -289,7 +289,6 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) } - /* Computing levels/displacements for variables. */ void Expr::bindVars(EvalState & es, const std::shared_ptr & env) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a4c19f7c..283d2746b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1461,10 +1461,10 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { /* We don’t check the path right now, because we don’t want to - throw if the path isn’t allowed, but just return false (and we - can’t just catch the exception here because we still want to - throw if something in the evaluation of `*args[0]` tries to - access an unauthorized path). */ + throw if the path isn’t allowed, but just return false (and we + can’t just catch the exception here because we still want to + throw if something in the evaluation of `*args[0]` tries to + access an unauthorized path). */ auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); try { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 6957d2da4..c767e72e5 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -266,7 +266,7 @@ std::optional Input::getLastModified() const return {}; } -ParsedURL InputScheme::toURL(const Input & input) +ParsedURL InputScheme::toURL(const Input & input) const { throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs)); } @@ -274,7 +274,7 @@ ParsedURL InputScheme::toURL(const Input & input) Input InputScheme::applyOverrides( const Input & input, std::optional ref, - std::optional rev) + std::optional rev) const { if (ref) throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); @@ -293,7 +293,7 @@ void InputScheme::markChangedFile(const Input & input, std::string_view file, st assert(false); } -void InputScheme::clone(const Input & input, const Path & destDir) +void InputScheme::clone(const Input & input, const Path & destDir) const { throw Error("do not know how to clone input '%s'", input.to_string()); } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index bc9a76b0b..17da37f47 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -107,26 +107,25 @@ public: * recognized. The Input object contains the information the fetcher * needs to actually perform the "fetch()" when called. */ - struct InputScheme { virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url) = 0; + virtual std::optional inputFromURL(const ParsedURL & url) const = 0; - virtual std::optional inputFromAttrs(const Attrs & attrs) = 0; + virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; - virtual ParsedURL toURL(const Input & input); + virtual ParsedURL toURL(const Input & input) const; - virtual bool hasAllInfo(const Input & input) = 0; + virtual bool hasAllInfo(const Input & input) const = 0; virtual Input applyOverrides( const Input & input, std::optional ref, - std::optional rev); + std::optional rev) const; - virtual void clone(const Input & input, const Path & destDir); + virtual void clone(const Input & input, const Path & destDir) const; virtual std::optional getSourcePath(const Input & input); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7b7a1be35..1f7d7c07d 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -18,6 +18,7 @@ using namespace std::string_literals; namespace nix::fetchers { + namespace { // Explicit initial branch of our bare repo to suppress warnings from new version of git. @@ -26,23 +27,23 @@ namespace { // old version of git, which will ignore unrecognized `-c` options. const std::string gitInitialBranch = "__nix_dummy_branch"; -bool isCacheFileWithinTtl(const time_t now, const struct stat & st) +bool isCacheFileWithinTtl(time_t now, const struct stat & st) { return st.st_mtime + settings.tarballTtl > now; } -bool touchCacheFile(const Path& path, const time_t& touch_time) +bool touchCacheFile(const Path & path, time_t touch_time) { - struct timeval times[2]; - times[0].tv_sec = touch_time; - times[0].tv_usec = 0; - times[1].tv_sec = touch_time; - times[1].tv_usec = 0; + struct timeval times[2]; + times[0].tv_sec = touch_time; + times[0].tv_usec = 0; + times[1].tv_sec = touch_time; + times[1].tv_usec = 0; - return lutimes(path.c_str(), times) == 0; + return lutimes(path.c_str(), times) == 0; } -Path getCachePath(std::string key) +Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, key).to_string(Base32, false); @@ -57,13 +58,12 @@ Path getCachePath(std::string key) // ... std::optional readHead(const Path & path) { - auto [exit_code, output] = runProgram(RunOptions { + auto [status, output] = runProgram(RunOptions { .program = "git", + // FIXME: use 'HEAD' to avoid returning all refs .args = {"ls-remote", "--symref", path}, }); - if (exit_code != 0) { - return std::nullopt; - } + if (status != 0) return std::nullopt; std::string_view line = output; line = line.substr(0, line.find("\n")); @@ -82,12 +82,11 @@ std::optional readHead(const Path & path) } // Persist the HEAD ref from the remote repo in the local cached repo. -bool storeCachedHead(const std::string& actualUrl, const std::string& headRef) +bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) { Path cacheDir = getCachePath(actualUrl); - auto gitDir = "."; try { - runProgram("git", true, { "-C", cacheDir, "--git-dir", gitDir, "symbolic-ref", "--", "HEAD", headRef }); + runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { if (!WIFEXITED(e.status)) throw; return false; @@ -96,7 +95,7 @@ bool storeCachedHead(const std::string& actualUrl, const std::string& headRef) return true; } -std::optional readHeadCached(const std::string& actualUrl) +std::optional readHeadCached(const std::string & actualUrl) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. @@ -110,16 +109,15 @@ std::optional readHeadCached(const std::string& actualUrl) cachedRef = readHead(cacheDir); if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && - isCacheFileWithinTtl(now, st)) { + isCacheFileWithinTtl(now, st)) + { debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl); return cachedRef; } } auto ref = readHead(actualUrl); - if (ref) { - return ref; - } + if (ref) return ref; if (cachedRef) { // If the cached git ref is expired in fetch() below, and the 'git fetch' @@ -250,7 +248,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -265,7 +263,7 @@ struct GitInputScheme : InputScheme Attrs attrs; attrs.emplace("type", "git"); - for (auto &[name, value] : url.query) { + for (auto & [name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); else if (name == "shallow" || name == "submodules") @@ -279,7 +277,7 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "git") return {}; @@ -302,7 +300,7 @@ struct GitInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme != "git") url.scheme = "git+" + url.scheme; @@ -313,7 +311,7 @@ struct GitInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { bool maybeDirty = !input.getRef(); bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); @@ -325,7 +323,7 @@ struct GitInputScheme : InputScheme Input applyOverrides( const Input & input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto res(input); if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); @@ -335,7 +333,7 @@ struct GitInputScheme : InputScheme return res; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto [isLocal, actualUrl] = getActualUrl(input); @@ -603,9 +601,9 @@ struct GitInputScheme : InputScheme { throw Error( "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " - "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " - ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD - "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", input.getRev()->gitRev(), *input.getRef(), actualUrl diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 2115ce2f5..1ed09d30d 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -26,11 +26,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() = 0; + virtual std::string type() const = 0; virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != type()) return {}; @@ -100,7 +100,7 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != type()) return {}; @@ -116,7 +116,7 @@ struct GitArchiveInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto owner = getStrAttr(input.attrs, "owner"); auto repo = getStrAttr(input.attrs, "repo"); @@ -132,7 +132,7 @@ struct GitArchiveInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); } @@ -140,7 +140,7 @@ struct GitArchiveInputScheme : InputScheme Input applyOverrides( const Input & _input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto input(_input); if (rev && ref) @@ -227,7 +227,7 @@ struct GitArchiveInputScheme : InputScheme struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() override { return "github"; } + std::string type() const override { return "github"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -240,14 +240,29 @@ struct GitHubInputScheme : GitArchiveInputScheme return std::pair("Authorization", fmt("token %s", token)); } + std::string getHost(const Input & input) const + { + return maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + } + + std::string getOwner(const Input & input) const + { + return getStrAttr(input.attrs, "owner"); + } + + std::string getRepo(const Input & input) const + { + return getStrAttr(input.attrs, "repo"); + } + Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); auto url = fmt( host == "github.com" ? "https://api.%s/repos/%s/%s/commits/%s" : "https://%s/api/v3/repos/%s/%s/commits/%s", - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + host, getOwner(input), getRepo(input), *input.getRef()); Headers headers = makeHeadersWithAuthTokens(host); @@ -262,8 +277,10 @@ struct GitHubInputScheme : GitArchiveInputScheme DownloadUrl getDownloadUrl(const Input & input) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); + Headers headers = makeHeadersWithAuthTokens(host); + // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. const auto urlFmt = @@ -273,17 +290,17 @@ struct GitHubInputScheme : GitArchiveInputScheme ? "https://%s/%s/%s/archive/%s.tar.gz" : "https://api.%s/repos/%s/%s/tarball/%s"; - const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), input.getRev()->to_string(Base16, false)); return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); Input::fromURL(fmt("git+https://%s/%s/%s.git", - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); } @@ -291,7 +308,7 @@ struct GitHubInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() override { return "gitlab"; } + std::string type() const override { return "gitlab"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -346,7 +363,7 @@ struct GitLabInputScheme : GitArchiveInputScheme return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere @@ -359,7 +376,7 @@ struct GitLabInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() override { return "sourcehut"; } + std::string type() const override { return "sourcehut"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -433,7 +450,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); Input::fromURL(fmt("git+https://%s/%s/%s", diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 9288fc6cf..b99504a16 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -7,7 +7,7 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "flake") return {}; @@ -50,7 +50,7 @@ struct IndirectInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; @@ -68,7 +68,7 @@ struct IndirectInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { ParsedURL url; url.scheme = "flake"; @@ -78,7 +78,7 @@ struct IndirectInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return false; } @@ -86,7 +86,7 @@ struct IndirectInputScheme : InputScheme Input applyOverrides( const Input & _input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto input(_input); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 5c5671681..86e8f81f4 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -43,7 +43,7 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -69,7 +69,7 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "hg") return {}; @@ -89,7 +89,7 @@ struct MercurialInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); url.scheme = "hg+" + url.scheme; @@ -98,7 +98,7 @@ struct MercurialInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { // FIXME: ugly, need to distinguish between dirty and clean // default trees. @@ -108,7 +108,7 @@ struct MercurialInputScheme : InputScheme Input applyOverrides( const Input & input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto res(input); if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index f0ef97da5..61541e69d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -6,7 +6,7 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "path") return {}; @@ -32,7 +32,7 @@ struct PathInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "path") return {}; @@ -54,7 +54,7 @@ struct PathInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto query = attrsToQuery(input.attrs); query.erase("path"); @@ -66,7 +66,7 @@ struct PathInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return true; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 6c551bd93..e9686262a 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -185,7 +185,7 @@ struct CurlInputScheme : InputScheme virtual bool isValidURL(const ParsedURL & url) const = 0; - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (!isValidURL(url)) return std::nullopt; @@ -203,7 +203,7 @@ struct CurlInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { auto type = maybeGetStrAttr(attrs, "type"); if (type != inputType()) return {}; @@ -220,16 +220,17 @@ struct CurlInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); - // NAR hashes are preferred over file hashes since tar/zip files // don't have a canonical representation. + // NAR hashes are preferred over file hashes since tar/zip + // files don't have a canonical representation. if (auto narHash = input.getNarHash()) url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return true; } diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 961f4e18a..9855bd2aa 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -132,7 +132,7 @@ public: log(*state, lvl, fs.s); } - void logEI(const ErrorInfo &ei) override + void logEI(const ErrorInfo & ei) override { auto state(state_.lock()); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 12d0c32fb..149d414d3 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -346,7 +346,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) try { getFile(info->url, *decompressor); } catch (NoSuchBinaryCacheFile & e) { - throw SubstituteGone(e.info()); + throw SubstituteGone(std::move(e.info())); } decompressor->finish(); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5aed51bcd..2949a0a1f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -134,7 +134,7 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, {}, ex); + done(BuildResult::TimedOut, {}, std::move(ex)); } @@ -984,7 +984,7 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, {}, e); + done(st, {}, std::move(e)); return; } } @@ -1435,7 +1435,7 @@ void DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } - amDone(buildResult.success() ? ecSuccess : ecFailed, ex); + amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index bea7363db..e1b80165e 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -30,7 +30,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod if (ex) logError(i->ex->info()); else - ex = i->ex; + ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); @@ -40,7 +40,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod if (failed.size() == 1 && ex) { ex->status = worker.exitStatus(); - throw *ex; + throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); @@ -109,7 +109,7 @@ void Store::ensurePath(const StorePath & path) if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { goal->ex->status = worker.exitStatus(); - throw *goal->ex; + throw std::move(*goal->ex); } else throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d2798888b..dc6f8eeba 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -230,7 +230,7 @@ void LocalDerivationGoal::tryLocalBuild() { outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, {}, e); + done(BuildResult::InputRejected, {}, std::move(e)); return; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fe99c3c5e..42a53912e 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -448,7 +448,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, // FIXME: remove -bool isDerivation(const std::string & fileName) +bool isDerivation(std::string_view fileName) { return hasSuffix(fileName, drvExtension); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index af198a767..f3cd87fb1 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -224,7 +224,7 @@ StorePath writeDerivation(Store & store, Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); // FIXME: remove -bool isDerivation(const std::string & fileName); +bool isDerivation(std::string_view fileName); /* Calculate the name that will be used for the store path for this output. diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 5746c32a3..2ff411e18 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -142,9 +142,9 @@ struct curlFileTransfer : public FileTransfer } template - void fail(const T & e) + void fail(T && e) { - failEx(std::make_exception_ptr(e)); + failEx(std::make_exception_ptr(std::move(e))); } LambdaSink finalSink; @@ -472,7 +472,7 @@ struct curlFileTransfer : public FileTransfer fileTransfer.enqueueItem(shared_from_this()); } else - fail(exc); + fail(std::move(exc)); } } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 96a29155c..48cf731a8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -447,7 +447,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.info()); + throw InvalidPath(std::move(e.info())); throw; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8811ab578..80b60ca1b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -19,21 +19,21 @@ using json = nlohmann::json; namespace nix { -bool Store::isInStore(const Path & path) const +bool Store::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair Store::toStorePath(const Path & path) const +std::pair Store::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); - Path::size_type slash = path.find('/', storeDir.size() + 1); + auto slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) return {parseStorePath(path), ""}; else - return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; + return {parseStorePath(path.substr(0, slash)), (Path) path.substr(slash)}; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 151ec10d6..4a88d7216 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -179,7 +179,7 @@ public: /* Return true if ‘path’ is in the Nix store (but not the Nix store itself). */ - bool isInStore(const Path & path) const; + bool isInStore(PathView path) const; /* Return true if ‘path’ is a store path, i.e. a direct child of the Nix store. */ @@ -187,7 +187,7 @@ public: /* Split a path like /nix/store/-/ into /nix/store/- and /. */ - std::pair toStorePath(const Path & path) const; + std::pair toStorePath(PathView path) const; /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(std::string_view path) const; diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 4b0636129..0e2b9d12c 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -35,10 +35,6 @@ static ArchiveSettings archiveSettings; static GlobalConfig::Register rArchiveSettings(&archiveSettings); -const std::string narVersionMagic1 = "nix-archive-1"; - -static std::string caseHackSuffix = "~nix~case~hack~"; - PathFilter defaultPathFilter = [](const Path &) { return true; }; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index ac4183bf5..e42dea540 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -103,7 +103,9 @@ void copyNAR(Source & source, Sink & sink); void copyPath(const Path & from, const Path & to); -extern const std::string narVersionMagic1; +inline constexpr std::string_view narVersionMagic1 = "nix-archive-1"; + +inline constexpr std::string_view caseHackSuffix = "~nix~case~hack~"; } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 7664e5c04..e879fd3b8 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -148,7 +148,7 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args) return f; } -inline hintformat hintfmt(std::string plain_string) +inline hintformat hintfmt(const std::string & plain_string) { // we won't be receiving any args in this case, so just print the original string return hintfmt("%s", normaltxt(plain_string)); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index cb2b15b41..ac86d8ac2 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -105,14 +105,6 @@ public: Verbosity verbosity = lvlInfo; -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - warn(fs.s); - haveWarned = true; - } -} - void writeToStderr(std::string_view s) { try { @@ -130,11 +122,11 @@ Logger * makeSimpleLogger(bool printBuildLogs) return new SimpleLogger(printBuildLogs); } -std::atomic nextId{(uint64_t) getpid() << 32}; +std::atomic nextId{0}; Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s, const Logger::Fields & fields, ActivityId parent) - : logger(logger), id(nextId++) + : logger(logger), id(nextId++ + (((uint64_t) getpid()) << 32)) { logger.startActivity(id, lvl, type, s, fields, parent); } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index d0817b4a9..4642c49f7 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -82,7 +82,7 @@ public: log(lvlInfo, fs); } - virtual void logEI(const ErrorInfo &ei) = 0; + virtual void logEI(const ErrorInfo & ei) = 0; void logEI(Verbosity lvl, ErrorInfo ei) { @@ -225,7 +225,11 @@ inline void warn(const std::string & fs, const Args & ... args) logger->warn(f.str()); } -void warnOnce(bool & haveWarned, const FormatOrString & fs); +#define warnOnce(haveWarned, args...) \ + if (!haveWarned) { \ + haveWarned = true; \ + warn(args); \ + } void writeToStderr(std::string_view s); diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index bf26321db..7d38b059c 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -83,6 +83,11 @@ public: return p != other.p; } + bool operator < (const ref & other) const + { + return p < other.p; + } + private: template diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 2c3597775..c653db9d0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -338,7 +338,7 @@ Sink & operator << (Sink & sink, const StringSet & s) Sink & operator << (Sink & sink, const Error & ex) { - auto info = ex.info(); + auto & info = ex.info(); sink << "Error" << info.level diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 84847835a..7da5b07fd 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -331,17 +331,9 @@ T readNum(Source & source) unsigned char buf[8]; source((char *) buf, sizeof(buf)); - uint64_t n = - ((uint64_t) buf[0]) | - ((uint64_t) buf[1] << 8) | - ((uint64_t) buf[2] << 16) | - ((uint64_t) buf[3] << 24) | - ((uint64_t) buf[4] << 32) | - ((uint64_t) buf[5] << 40) | - ((uint64_t) buf[6] << 48) | - ((uint64_t) buf[7] << 56); + auto n = readLittleEndian(buf); - if (n > (uint64_t)std::numeric_limits::max()) + if (n > (uint64_t) std::numeric_limits::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); return (T) n; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 4f2caaa40..993dc1cb6 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1594,6 +1594,21 @@ std::string stripIndentation(std::string_view s) } +std::pair getLine(std::string_view s) +{ + auto newline = s.find('\n'); + + if (newline == s.npos) { + return {s, ""}; + } else { + auto line = s.substr(0, newline); + if (!line.empty() && line[line.size() - 1] == '\r') + line = line.substr(0, line.size() - 1); + return {line, s.substr(newline + 1)}; + } +} + + ////////////////////////////////////////////////////////////////////// static Sync> windowSize{{0, 0}}; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 94d8cc555..3caa95fca 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -510,6 +510,17 @@ std::optional string2Float(const std::string_view s) } +/* Convert a little-endian integer to host order. */ +template +T readLittleEndian(unsigned char * p) +{ + T x = 0; + for (size_t i = 0; i < sizeof(x); ++i) + x |= ((T) *p++) << (i * 8); + return x; +} + + /* Return true iff `s' starts with `prefix'. */ bool hasPrefix(std::string_view s, std::string_view prefix); @@ -563,6 +574,12 @@ std::string base64Decode(std::string_view s); std::string stripIndentation(std::string_view s); +/* Get the prefix of 's' up to and excluding the next line break (LF + optionally preceded by CR), and the remainder following the line + break. */ +std::pair getLine(std::string_view s); + + /* Get a value for the specified key from an associate container. */ template const typename T::mapped_type * get(const T & map, const typename T::key_type & key) @@ -737,4 +754,11 @@ inline std::string operator + (std::string && s, std::string_view s2) return std::move(s); } +inline std::string operator + (std::string_view s1, const char * s2) +{ + std::string s(s1); + s.append(s2); + return s; +} + } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b59a6d026..7bb9c630f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -926,7 +926,6 @@ static void opServe(Strings opFlags, Strings opArgs) worker_proto::write(*store, out, status.builtOutputs); } - break; } diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 940923d3b..c527fdb0a 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -257,7 +257,7 @@ static void daemonLoop() } catch (Interrupted & e) { return; } catch (Error & error) { - ErrorInfo ei = error.info(); + auto ei = error.info(); // FIXME: add to trace? ei.msg = hintfmt("error processing connection: %1%", ei.msg.str()); logError(ei); diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 2ee8a707d..8c6042d94 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -16,7 +16,7 @@ R""( # Description This command recreates the lock file of a flake (`flake.lock`), thus -updating the lock for every mutable input (like `nixpkgs`) to its +updating the lock for every unlocked input (like `nixpkgs`) to its current version. This is equivalent to passing `--recreate-lock-file` to any command that operates on a flake. That is, diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 336f6723a..96f035117 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -215,7 +215,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (!lockedFlake.lockFile.root->inputs.empty()) logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); - std::unordered_set> visited; + std::set> visited; std::function recurse; @@ -227,7 +227,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (auto lockedNode = std::get_if<0>(&input.second)) { logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, - *lockedNode ? (*lockedNode)->lockedRef : flake.lockedRef); + (*lockedNode)->lockedRef); bool firstVisit = visited.insert(*lockedNode).second; diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index bdab9a208..fa786162f 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -20,11 +20,11 @@ following fields: * An integer that can be used to unambiguously identify the package in invocations of `nix profile remove` and `nix profile upgrade`. -* The original ("mutable") flake reference and output attribute path +* The original ("unlocked") flake reference and output attribute path used at installation time. -* The immutable flake reference to which the mutable flake reference - was resolved. +* The locked flake reference to which the unlocked flake reference was + resolved. * The store path(s) of the package. diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index e06e74abe..39cca428b 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -2,7 +2,7 @@ R""( # Examples -* Upgrade all packages that were installed using a mutable flake +* Upgrade all packages that were installed using an unlocked flake reference: ```console @@ -32,9 +32,9 @@ the package was installed. > **Warning** > -> This only works if you used a *mutable* flake reference at +> This only works if you used an *unlocked* flake reference at > installation time, e.g. `nixpkgs#hello`. It does not work if you -> used an *immutable* flake reference +> used a *locked* flake reference > (e.g. `github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a#hello`), > since in that case the "latest version" is always the same. diff --git a/src/nix/profile.md b/src/nix/profile.md index be3c5ba1a..273e02280 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -88,8 +88,7 @@ has the following fields: the user at the time of installation (e.g. `nixpkgs`). This is also the flake reference that will be used by `nix profile upgrade`. -* `uri`: The immutable flake reference to which `originalUrl` - resolved. +* `uri`: The locked flake reference to which `originalUrl` resolved. * `attrPath`: The flake output attribute that provided this package. Note that this is not necessarily the attribute that the diff --git a/src/nix/registry.cc b/src/nix/registry.cc index c496f94f8..b5bdfba95 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -183,14 +183,12 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand void run(nix::ref store) override { - if (locked.empty()) { - locked = url; - } + if (locked.empty()) locked = url; auto registry = getRegistry(); auto ref = parseFlakeRef(url); - auto locked_ref = parseFlakeRef(locked); + auto lockedRef = parseFlakeRef(locked); registry->remove(ref.input); - auto [tree, resolved] = locked_ref.resolve(store).input.fetch(store); + auto [tree, resolved] = lockedRef.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; registry->add(ref.input, resolved, extraAttrs); diff --git a/tests/eval.sh b/tests/eval.sh index d74976019..ffae08a6a 100644 --- a/tests/eval.sh +++ b/tests/eval.sh @@ -29,3 +29,7 @@ nix-instantiate --eval -E 'assert 1 + 2 == 3; true' [[ $(nix-instantiate -A attr --eval "./eval.nix") == '{ foo = "bar"; }' ]] [[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]] + +# Check that symlink cycles don't cause a hang. +ln -sfn cycle.nix $TEST_ROOT/cycle.nix +(! nix eval --file $TEST_ROOT/cycle.nix) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 4ceba0293..da09c3f37 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -122,6 +122,7 @@ git -C $repo commit -m 'Bla3' -a path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] +status=0 nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$? [[ "$status" = "102" ]] diff --git a/tests/flakes/absolute-paths.sh b/tests/flakes/absolute-paths.sh new file mode 100644 index 000000000..e7bfba12d --- /dev/null +++ b/tests/flakes/absolute-paths.sh @@ -0,0 +1,17 @@ +source ./common.sh + +requireGit + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 + +createGitRepo $flake1Dir +cat > $flake1Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/default.nix < $nonFlakeDir/README.md < $badFlakeDir/flake.nix diff --git a/tests/flakes/unlocked-override.sh b/tests/flakes/unlocked-override.sh new file mode 100644 index 000000000..8abc8b7d3 --- /dev/null +++ b/tests/flakes/unlocked-override.sh @@ -0,0 +1,30 @@ +source ./common.sh + +requireGit + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 + +createGitRepo $flake1Dir +cat > $flake1Dir/flake.nix < $flake1Dir/x.nix +git -C $flake1Dir add flake.nix x.nix +git -C $flake1Dir commit -m Initial + +createGitRepo $flake2Dir +cat > $flake2Dir/flake.nix < $flake1Dir/x.nix + +[[ $(nix eval --json $flake2Dir#x --override-input flake1 $TEST_ROOT/flake1) = 456 ]] diff --git a/tests/function-trace.sh b/tests/function-trace.sh index 0b7f49d82..d68e10df5 100755 --- a/tests/function-trace.sh +++ b/tests/function-trace.sh @@ -11,7 +11,7 @@ expect_trace() { --expr "$expr" 2>&1 \ | grep "function-trace" \ | sed -e 's/ [0-9]*$//' - ); + ) echo -n "Tracing expression '$expr'" set +e diff --git a/tests/local.mk b/tests/local.mk index 340817ec3..2f7f76261 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -7,6 +7,8 @@ nix_tests = \ flakes/follow-paths.sh \ flakes/bundle.sh \ flakes/check.sh \ + flakes/unlocked-override.sh \ + flakes/absolute-paths.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ @@ -110,7 +112,8 @@ nix_tests = \ fetchClosure.sh \ completions.sh \ impure-derivations.sh \ - path-from-hash-part.sh + path-from-hash-part.sh \ + toString-path.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/nix_path.sh b/tests/nix_path.sh index d3657abf0..2b222b4a1 100644 --- a/tests/nix_path.sh +++ b/tests/nix_path.sh @@ -9,3 +9,6 @@ nix-instantiate --eval -E '' --restrict-eval # Should ideally also test this, but there’s no pure way to do it, so just trust me that it works # nix-instantiate --eval -E '' -I nixpkgs=channel:nixos-unstable --restrict-eval + +[[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] +[[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] diff --git a/tests/restricted.sh b/tests/restricted.sh index 242b901dd..9bd16cf51 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -3,7 +3,7 @@ source common.sh clearStore nix-instantiate --restrict-eval --eval -E '1 + 2' -(! nix-instantiate --restrict-eval ./restricted.nix) +(! nix-instantiate --eval --restrict-eval ./restricted.nix) (! nix-instantiate --eval --restrict-eval <(echo '1 + 2')) nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh diff --git a/tests/toString-path.sh b/tests/toString-path.sh new file mode 100644 index 000000000..07eb87465 --- /dev/null +++ b/tests/toString-path.sh @@ -0,0 +1,8 @@ +source common.sh + +mkdir -p $TEST_ROOT/foo +echo bla > $TEST_ROOT/foo/bar + +[[ $(nix eval --raw --impure --expr "builtins.readFile (builtins.toString (builtins.fetchTree { type = \"path\"; path = \"$TEST_ROOT/foo\"; } + \"/bar\"))") = bla ]] + +[[ $(nix eval --json --impure --expr "builtins.readDir (builtins.toString (builtins.fetchTree { type = \"path\"; path = \"$TEST_ROOT/foo\"; }))") = '{"bar":"regular"}' ]]