Merge remote-tracking branch 'upstream/master' into indexed-store-path-outputs

.github/assign-by-files.yml

@@ -1,5 +0,0 @@
----
-# This files is used by https://github.com/marketplace/actions/auto-assign-reviewer-by-files
-# to assign maintainers
-"doc/**/*":
-  - fricklerhandwerk

.github/workflows/assign-reviewer.yml

@@ -1,12 +0,0 @@
-name: "Auto Assign"
-on:
-  - pull_request
-
-jobs:
-  assign_reviewer:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: shufo/auto-assign-reviewer-by-files@v1.1.4
-        with:
-          config: ".github/assign-by-files.yml"
-          token: ${{ secrets.GITHUB_TOKEN }}

doc/manual/src/release-notes/rl-next.md

@@ -3,6 +3,18 @@
 * The `repeat` and `enforce-determinism` options have been removed
   since they had been broken under many circumstances for a long time.
 
+* You can now use [flake references] in the [old command line interface], e.g.
+
+   [flake references]: ../command-ref/new-cli/nix3-flake.md#flake-references
+   [old command line interface]: ../command-ref/main-commands.md
+
+  ```
+  # nix-build flake:nixpkgs -A hello
+  # nix-build -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 \
+      '<nixpkgs>' -A hello
+  # NIX_PATH=nixpkgs=flake:nixpkgs nix-build '<nixpkgs>' -A hello
+  ```
+
 * Allow explicitly selecting outputs in a store derivation installable, just like we can do with other sorts of installables.
   For example,
   ```shell-session

src/libcmd/common-eval-args.cc

@@ -85,6 +85,23 @@ MixEvalArgs::MixEvalArgs()
   -I nixpkgs=channel:nixos-21.05
   -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
   ```
+
+  You can also fetch source trees using flake URLs and add them to the
+  search path. For instance,
+
+  ```
+  -I nixpkgs=flake:nixpkgs
+  ```
+
+  specifies that the prefix `nixpkgs` shall refer to the source tree
+  downloaded from the `nixpkgs` entry in the flake registry. Similarly,
+
+  ```
+  -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05
+  ```
+
+  makes `<nixpkgs>` refer to a particular branch of the
+  `NixOS/nixpkgs` repository on GitHub.
   )",
         .category = category,
         .labels = {"path"},
@@ -142,14 +159,25 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 
 Path lookupFileArg(EvalState & state, std::string_view s)
 {
-    if (isUri(s)) {
+    if (EvalSettings::isPseudoUrl(s)) {
-        return state.store->toRealPath(
+        auto storePath = fetchers::downloadTarball(
-            fetchers::downloadTarball(
+            state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath;
-                state.store, resolveUri(s), "source", false).first.storePath);
+        return state.store->toRealPath(storePath);
-    } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
+    }
+
+    else if (hasPrefix(s, "flake:")) {
+        settings.requireExperimentalFeature(Xp::Flakes);
+        auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
+        auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath;
+        return state.store->toRealPath(storePath);
+    }
+
+    else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p(s.substr(1, s.size() - 2));
         return state.findFile(p);
-    } else
+    }
+
+    else
         return absPath(std::string(s));
 }
 

src/libexpr/eval.cc

@@ -402,7 +402,8 @@ static Strings parseNixPath(const std::string & s)
         }
 
         if (*p == ':') {
-            if (isUri(std::string(start2, s.end()))) {
+            auto prefix = std::string(start2, s.end());
+            if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) {
                 ++p;
                 while (p != s.end() && *p != ':') ++p;
             }
@@ -1659,7 +1660,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                         (lambda.name
                             ? concatStrings("'", symbols[lambda.name], "'")
                             : "anonymous lambda"));
-                    addErrorTrace(e, pos, "from call site%s", "");
+                    addErrorTrace(e, pos, "while evaluating call site%s", "");
                 }
                 throw;
             }
@@ -2583,6 +2584,23 @@ Strings EvalSettings::getDefaultNixPath()
     return res;
 }
 
+bool EvalSettings::isPseudoUrl(std::string_view s)
+{
+    if (s.compare(0, 8, "channel:") == 0) return true;
+    size_t pos = s.find("://");
+    if (pos == std::string::npos) return false;
+    std::string scheme(s, 0, pos);
+    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
+}
+
+std::string EvalSettings::resolvePseudoUrl(std::string_view url)
+{
+    if (hasPrefix(url, "channel:"))
+        return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
+    else
+        return std::string(url);
+}
+
 EvalSettings evalSettings;
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);

src/libexpr/eval.hh

@@ -590,6 +590,10 @@ struct EvalSettings : Config
 
     static Strings getDefaultNixPath();
 
+    static bool isPseudoUrl(std::string_view s);
+
+    static std::string resolvePseudoUrl(std::string_view url);
+
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 

src/libexpr/flake/flake.cc

@@ -143,7 +143,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         } catch (Error & e) {
             e.addTrace(
                 state.positions[attr.pos],
-                hintfmt("in flake attribute '%s'", state.symbols[attr.name]));
+                hintfmt("while evaluating flake attribute '%s'", state.symbols[attr.name]));
             throw;
         }
     }
@@ -152,7 +152,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         try {
             input.ref = FlakeRef::fromAttrs(attrs);
         } catch (Error & e) {
-            e.addTrace(state.positions[pos], hintfmt("in flake input"));
+            e.addTrace(state.positions[pos], hintfmt("while evaluating flake input"));
             throw;
         }
     else {

src/libexpr/parser.y

@@ -643,6 +643,7 @@ formal
 #include "filetransfer.hh"
 #include "fetchers.hh"
 #include "store-api.hh"
+#include "flake/flake.hh"
 
 
 namespace nix {
@@ -805,17 +806,28 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
 
     std::pair<bool, std::string> res;
 
-    if (isUri(elem.second)) {
+    if (EvalSettings::isPseudoUrl(elem.second)) {
         try {
-            res = { true, store->toRealPath(fetchers::downloadTarball(
+            auto storePath = fetchers::downloadTarball(
-                        store, resolveUri(elem.second), "source", false).first.storePath) };
+                store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).first.storePath;
+            res = { true, store->toRealPath(storePath) };
         } catch (FileTransferError & e) {
             logWarning({
                 .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
             });
             res = { false, "" };
         }
-    } else {
+    }
+
+    else if (hasPrefix(elem.second, "flake:")) {
+        settings.requireExperimentalFeature(Xp::Flakes);
+        auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false);
+        debug("fetching flake search path element '%s''", elem.second);
+        auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath;
+        res = { true, store->toRealPath(storePath) };
+    }
+
+    else {
         auto path = absPath(elem.second);
         if (pathExists(path))
             res = { true, path };

src/libexpr/primops/fetchTree.cc

@@ -220,8 +220,6 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
-    url = resolveUri(*url);
-
     state.checkURI(*url);
 
     if (name == "")

src/libfetchers/fetch-settings.hh

@@ -71,7 +71,12 @@ struct FetchSettings : public Config
         "Whether to warn about dirty Git/Mercurial trees."};
 
     Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry",
-        "Path or URI of the global flake registry."};
+        R"(
+          Path or URI of the global flake registry.
+
+          When empty, disables the global flake registry.
+        )"};
+
 
     Setting<bool> useRegistries{this, true, "use-registries",
         "Whether to use flake registries to resolve flake references."};

src/libfetchers/registry.cc

@@ -153,6 +153,9 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
 {
     static auto reg = [&]() {
         auto path = fetchSettings.flakeRegistry.get();
+        if (path == "") {
+            return std::make_shared<Registry>(Registry::Global); // empty registry
+        }
 
         if (!hasPrefix(path, "/")) {
             auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;

src/libstore/filetransfer.cc

@@ -33,14 +33,6 @@ FileTransferSettings fileTransferSettings;
 
 static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
 
-std::string resolveUri(std::string_view uri)
-{
-    if (uri.compare(0, 8, "channel:") == 0)
-        return "https://nixos.org/channels/" + std::string(uri.substr(8)) + "/nixexprs.tar.xz";
-    else
-        return std::string(uri);
-}
-
 struct curlFileTransfer : public FileTransfer
 {
     CURLM * curlm = 0;
@@ -873,14 +865,4 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::optional<st
         err.msg = hf;
 }
 
-bool isUri(std::string_view s)
-{
-    if (s.compare(0, 8, "channel:") == 0) return true;
-    size_t pos = s.find("://");
-    if (pos == std::string::npos) return false;
-    std::string scheme(s, 0, pos);
-    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
-}
-
-
 }

src/libstore/filetransfer.hh

@@ -125,9 +125,4 @@ public:
     FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args);
 };
 
-bool isUri(std::string_view s);
-
-/* Resolve deprecated 'channel:<foo>' URLs. */
-std::string resolveUri(std::string_view uri);
-
 }

src/libutil/error.cc

@@ -262,6 +262,28 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
         prefix += ":" ANSI_NORMAL " ";
 
     std::ostringstream oss;
+
+    // traces
+    if (showTrace && !einfo.traces.empty()) {
+        for (const auto & trace : einfo.traces) {
+            oss << "\n" << "… " << trace.hint.str() << "\n";
+
+            if (trace.pos.has_value() && (*trace.pos)) {
+                auto pos = trace.pos.value();
+                oss << "\n";
+                printAtPos(pos, oss);
+
+                auto loc = getCodeLines(pos);
+                if (loc.has_value()) {
+                    oss << "\n";
+                    printCodeLines(oss, "", pos, *loc);
+                    oss << "\n";
+                }
+            }
+        }
+        oss << "\n" << prefix;
+    }
+
     oss << einfo.msg << "\n";
 
     if (einfo.errPos.has_value() && *einfo.errPos) {
@@ -285,26 +307,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
             "?" << std::endl;
     }
 
-    // traces
-    if (showTrace && !einfo.traces.empty()) {
-        for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) {
-            oss << "\n" << "… " << iter->hint.str() << "\n";
-
-            if (iter->pos.has_value() && (*iter->pos)) {
-                auto pos = iter->pos.value();
-                oss << "\n";
-                printAtPos(pos, oss);
-
-                auto loc = getCodeLines(pos);
-                if (loc.has_value()) {
-                    oss << "\n";
-                    printCodeLines(oss, "", pos, *loc);
-                    oss << "\n";
-                }
-            }
-        }
-    }
-
     out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str()));
 
     return out;

src/nix-store/nix-store.cc

@@ -812,10 +812,13 @@ static void opServe(Strings opFlags, Strings opArgs)
             if (nrRepeats != 0) {
                 throw Error("client requested repeating builds, but this is not currently implemented");
             }
-            // Ignore. It used to be true by default, but also only never had any effect when `nrRepeats == 0`.
+            // Ignore 'enforceDeterminism'. It used to be true by
-            // We have already asserted that `nrRepeats` in fact is 0, so we can safely ignore this without
+            // default, but also only never had any effect when
-            // doing something other than what the client asked for.
+            // `nrRepeats == 0`.  We have already asserted that
-            auto _enforceDeterminism = readInt(in);
+            // `nrRepeats` in fact is 0, so we can safely ignore this
+            // without doing something other than what the client
+            // asked for.
+            readInt(in);
 
             settings.runDiffHook = true;
         }

tests/flakes/flakes.sh

@@ -74,8 +74,10 @@ nix registry add --registry $registry flake3 git+file://$flake3Dir
 nix registry add --registry $registry flake4 flake3
 nix registry add --registry $registry nixpkgs flake1
 
-# Test 'nix flake list'.
+# Test 'nix registry list'.
 [[ $(nix registry list | wc -l) == 5 ]]
+nix registry list | grep -q    '^global'
+nix registry list | grep -q -v '^user' # nothing in user registry
 
 # Test 'nix flake metadata'.
 nix flake metadata flake1
@@ -340,6 +342,16 @@ nix registry pin flake1 flake3
 nix registry remove flake1
 [[ $(nix registry list | wc -l) == 5 ]]
 
+# Test 'nix registry list' with a disabled global registry.
+nix registry add user-flake1 git+file://$flake1Dir
+nix registry add user-flake2 git+file://$flake2Dir
+[[ $(nix --flake-registry "" registry list | wc -l) == 2 ]]
+nix --flake-registry "" registry list | grep -q -v '^global' # nothing in global registry
+nix --flake-registry "" registry list | grep -q    '^user'
+nix registry remove user-flake1
+nix registry remove user-flake2
+[[ $(nix registry list | wc -l) == 5 ]]
+
 # Test 'nix flake clone'.
 rm -rf $TEST_ROOT/flake1-v2
 nix flake clone flake1 --dest $TEST_ROOT/flake1-v2
@@ -473,3 +485,9 @@ nix store delete $(nix store add-path $badFlakeDir)
 
 [[ $(nix path-info      $(nix store add-path $flake1Dir)) =~ flake1 ]]
 [[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]]
+
+# Test fetching flakerefs in the legacy CLI.
+[[ $(nix-instantiate --eval flake:flake3 -A x) = 123 ]]
+[[ $(nix-instantiate --eval flake:git+file://$flake3Dir -A x) = 123 ]]
+[[ $(nix-instantiate -I flake3=flake:flake3 --eval '<flake3>' -A x) = 123 ]]
+[[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '<flake3>' -A x) = 123 ]]

tests/lang.sh

@@ -2,6 +2,7 @@ source common.sh
 
 export TEST_VAR=foo # for eval-okay-getenv.nix
 export NIX_REMOTE=dummy://
+export NIX_STORE_DIR=/nix/store
 
 nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello
 nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1
@@ -50,10 +51,10 @@ for i in lang/eval-okay-*.nix; do
         if test -e lang/$i.flags; then
             flags=$(cat lang/$i.flags)
         fi
-        if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
+        if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
             echo "FAIL: $i should evaluate"
             fail=1
-        elif ! diff lang/$i.out lang/$i.exp; then
+        elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then
             echo "FAIL: evaluation result of $i not as expected"
             fail=1
         fi

tests/lang/eval-okay-closure.exp

@@ -0,0 +1 @@
+[ { foo = true; key = -13; } { foo = true; key = -12; } { foo = true; key = -11; } { foo = true; key = -9; } { foo = true; key = -8; } { foo = true; key = -7; } { foo = true; key = -5; } { foo = true; key = -4; } { foo = true; key = -3; } { key = -1; } { foo = true; key = 0; } { foo = true; key = 1; } { foo = true; key = 2; } { foo = true; key = 4; } { foo = true; key = 5; } { foo = true; key = 6; } { key = 8; } { foo = true; key = 9; } { foo = true; key = 10; } { foo = true; key = 13; } { foo = true; key = 14; } { foo = true; key = 15; } { key = 17; } { foo = true; key = 18; } { foo = true; key = 19; } { foo = true; key = 22; } { foo = true; key = 23; } { key = 26; } { foo = true; key = 27; } { foo = true; key = 28; } { foo = true; key = 31; } { foo = true; key = 32; } { key = 35; } { foo = true; key = 36; } { foo = true; key = 40; } { foo = true; key = 41; } { key = 44; } { foo = true; key = 45; } { foo = true; key = 49; } { key = 53; } { foo = true; key = 54; } { foo = true; key = 58; } { key = 62; } { foo = true; key = 67; } { key = 71; } { key = 80; } ]

tests/lang/eval-okay-functionargs.exp

@@ -0,0 +1 @@
+[ "stdenv" "fetchurl" "aterm-stdenv" "aterm-stdenv2" "libX11" "libXv" "mplayer-stdenv2.libXv-libX11" "mplayer-stdenv2.libXv-libX11_2" "nix-stdenv-aterm-stdenv" "nix-stdenv2-aterm2-stdenv2" ]

tests/lang/eval-okay-path-antiquotation.exp

@@ -0,0 +1 @@
+{ absolute = /foo; expr = /pwd/lang/foo/bar; home = /fake-home/foo; notfirst = /pwd/lang/bar/foo; simple = /pwd/lang/foo; slashes = /foo/bar; surrounded = /pwd/lang/a-foo-b; }

tests/lang/eval-okay-path.exp

@@ -0,0 +1 @@
+"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output"