Merge branch 'master' into debug-exploratory-PR

.github/workflows/backport.yml

@@ -15,7 +15,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v0.0.7
+        uses: zeebe-io/backport-action@v0.0.8
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -100,7 +100,7 @@ jobs:
     - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION
     - run: docker tag nix:$NIX_VERSION nixos/nix:master
     - name: Login to Docker Hub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}

configure.ac

@@ -294,6 +294,17 @@ esac
 AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]),
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)
+if test ${cross_compiling:-no} = no && ! test -z ${sandbox_shell+x}; then
+  AC_MSG_CHECKING([whether sandbox-shell has the standalone feature])
+  # busybox shell sometimes allows executing other busybox applets,
+  # even if they are not in the path, breaking our sandbox
+  if PATH= $sandbox_shell -c "busybox" 2>&1 | grep -qv "not found"; then
+    AC_MSG_RESULT(enabled)
+    AC_MSG_ERROR([Please disable busybox FEATURE_SH_STANDALONE])
+  else
+    AC_MSG_RESULT(disabled)
+  fi
+fi
 
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix

doc/manual/src/contributing/hacking.md

@@ -71,18 +71,6 @@ To install it in `$(pwd)/outputs` and test it:
 nix (Nix) 3.0
 ```
 
-To run a functional test:
-
-```console
-make tests/test-name-should-auto-complete.sh.test
-```
-
-To run the unit-tests for C++ code:
-
-```
-make check
-```
-
 If you have a flakes-enabled Nix you can replace:
 
 ```console
@@ -94,3 +82,29 @@ by:
 ```console
 $ nix develop
 ```
+
+## Testing
+
+Nix comes with three different flavors of tests: unit, functional and integration.
+
+### Unit-tests
+
+The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
+under `src/{library_name}/tests` using the
+[googletest](https://google.github.io/googletest/) framework.
+
+You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option.
+
+### Functional tests
+
+The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
+The whole testsuite can be run with `make install && make installcheck`.
+Individual tests can be run with `make tests/{testName}.sh.test`.
+
+### Integration tests
+
+The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
+These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup.
+Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
+
+You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`

doc/manual/src/release-notes/rl-next.md

@@ -37,3 +37,6 @@
   :c            Go until end of program, exception, or builtins.break().
   :s            Go one step
   ```
+
+* `builtins.fetchTree` (and flake inputs) can now be used to fetch plain files
+  over the `http(s)` and `file` protocols in addition to directory tarballs.

misc/systemd/nix-daemon.service.in

@@ -9,6 +9,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 [Service]
 ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
+LimitNOFILE=4096
 
 [Install]
 WantedBy=multi-user.target

scripts/create-darwin-volume.sh

@@ -442,9 +442,13 @@ add_nix_vol_fstab_line() {
     local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}"
     shift
 
-    # wrap `ex` to work around a problem with vim plugins breaking exit codes;
+    # wrap `ex` to work around a problem with vim plugins breaking exit codes
-    # (see https://github.com/NixOS/nix/issues/5468)
+    # (see github.com/NixOS/nix/issues/5468)
-    # we'd prefer EDITOR="/usr/bin/ex --noplugin" but vifs doesn't word-split
+    #
+    # the first draft used `--noplugin`, but github.com/NixOS/nix/issues/6462
+    # suggests we need the less-semantic `-u NONE`
+    #
+    # we'd prefer EDITOR="/usr/bin/ex -u NONE" but vifs doesn't word-split
     # the EDITOR env.
     #
     # TODO: at some point we should switch to `--clean`, but it wasn't added
@@ -452,7 +456,7 @@ add_nix_vol_fstab_line() {
     # minver 10.12.6 seems to have released with vim 7.4
     cat > "$SCRATCH/ex_cleanroom_wrapper" <<EOF
 #!/bin/sh
-/usr/bin/ex --noplugin "\$@"
+/usr/bin/ex -u NONE "\$@"
 EOF
     chmod 755 "$SCRATCH/ex_cleanroom_wrapper"
 
@@ -646,8 +650,9 @@ EOF
         task "Configuring /etc/synthetic.conf to make a mount-point at $NIX_ROOT" >&2
         # technically /etc/synthetic.d/nix is supported in Big Sur+
         # but handling both takes even more code...
+        # Note: `-u NONE` disables vim plugins/rc; see note on --clean earlier
         _sudo "to add Nix to /etc/synthetic.conf" \
-            /usr/bin/ex --noplugin /etc/synthetic.conf <<EOF
+            /usr/bin/ex -u NONE /etc/synthetic.conf <<EOF
 :a
 ${NIX_ROOT:1}
 .
@@ -815,7 +820,8 @@ setup_volume_daemon() {
     local volume_uuid="$2"
     if ! test_voldaemon; then
         task "Configuring LaunchDaemon to mount '$NIX_VOLUME_LABEL'" >&2
-        _sudo "to install the Nix volume mounter" /usr/bin/ex --noplugin "$NIX_VOLUME_MOUNTD_DEST" <<EOF
+        # Note: `-u NONE` disables vim plugins/rc; see note on --clean earlier
+        _sudo "to install the Nix volume mounter" /usr/bin/ex -u NONE "$NIX_VOLUME_MOUNTD_DEST" <<EOF
 :a
 $(generate_mount_daemon "$cmd_type" "$volume_uuid")
 .

src/libcmd/installables.cc

@@ -621,11 +621,15 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto drvPath = attr->forceDerivation();
 
     std::set<std::string> outputsToInstall;
+    std::optional<NixInt> priority;
 
-    if (auto aMeta = attr->maybeGetAttr(state->sMeta))
+    if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
         if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
             for (auto & s : aOutputsToInstall->getListOfStrings())
                 outputsToInstall.insert(s);
+        if (auto aPriority = aMeta->maybeGetAttr("priority"))
+            priority = aPriority->getInt();
+    }
 
     if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
         outputsToInstall.clear();
@@ -643,6 +647,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto drvInfo = DerivationInfo {
         .drvPath = std::move(drvPath),
         .outputsToInstall = std::move(outputsToInstall),
+        .priority = priority,
     };
 
     return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};

src/libcmd/installables.hh

@@ -142,6 +142,7 @@ struct InstallableValue : Installable
     {
         StorePath drvPath;
         std::set<std::string> outputsToInstall;
+        std::optional<NixInt> priority;
     };
 
     virtual std::vector<DerivationInfo> toDerivations() = 0;

src/libexpr/eval-cache.cc

@@ -47,7 +47,7 @@ struct AttrDb
     {
         auto state(_state->lock());
 
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v4";
         createDirs(cacheDir);
 
         Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
@@ -175,6 +175,24 @@ struct AttrDb
         });
     }
 
+    AttrId setInt(
+        AttrKey key,
+        int n)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (symbols[key.second])
+                (AttrType::Int)
+                (n).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
     AttrId setListOfStrings(
         AttrKey key,
         const std::vector<std::string> & l)
@@ -287,6 +305,8 @@ struct AttrDb
             }
             case AttrType::Bool:
                 return {{rowId, queryAttribute.getInt(2) != 0}};
+            case AttrType::Int:
+                return {{rowId, int_t{queryAttribute.getInt(2)}}};
             case AttrType::ListOfStrings:
                 return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
             case AttrType::Missing:
@@ -426,6 +446,8 @@ Value & AttrCursor::forceValue()
             cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
         else if (v.type() == nBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+        else if (v.type() == nInt)
+            cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}};
         else if (v.type() == nAttrs)
             ; // FIXME: do something?
         else
@@ -621,6 +643,28 @@ bool AttrCursor::getBool()
     return v.boolean;
 }
 
+NixInt AttrCursor::getInt()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey());
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto i = std::get_if<int_t>(&cachedValue->second)) {
+                debug("using cached Integer attribute '%s'", getAttrPathStr());
+                return i->x;
+            } else
+                throw TypeError("'%s' is not an Integer", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nInt)
+        throw TypeError("'%s' is not an Integer", getAttrPathStr());
+
+    return v.integer;
+}
+
 std::vector<std::string> AttrCursor::getListOfStrings()
 {
     if (root->db) {

src/libexpr/eval-cache.hh

@@ -45,12 +45,14 @@ enum AttrType {
     Failed = 5,
     Bool = 6,
     ListOfStrings = 7,
+    Int = 8,
 };
 
 struct placeholder_t {};
 struct missing_t {};
 struct misc_t {};
 struct failed_t {};
+struct int_t { NixInt x; };
 typedef uint64_t AttrId;
 typedef std::pair<AttrId, Symbol> AttrKey;
 typedef std::pair<std::string, NixStringContext> string_t;
@@ -63,6 +65,7 @@ typedef std::variant<
     misc_t,
     failed_t,
     bool,
+    int_t,
     std::vector<std::string>
     > AttrValue;
 
@@ -116,6 +119,8 @@ public:
 
     bool getBool();
 
+    NixInt getInt();
+
     std::vector<std::string> getListOfStrings();
 
     std::vector<Symbol> getAttrs();

src/libexpr/flake/config.cc

@@ -31,7 +31,7 @@ static void writeTrustedList(const TrustedList & trustedList)
 
 void ConfigFile::apply()
 {
-    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"};
+    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry"};
 
     for (auto & [name, value] : settings) {
 

src/libexpr/flake/flake.cc

@@ -723,6 +723,7 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V
         lockFlake(state, flakeRef,
             LockFlags {
                 .updateLockFile = false,
+                .writeLockFile = false,
                 .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries,
                 .allowMutable  = !evalSettings.pureEval,
             }),

src/libexpr/primops.cc

@@ -3560,7 +3560,7 @@ static RegisterPrimOp primop_match({
       builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" "  FOO   "
       ```
 
-      Evaluates to `[ "foo" ]`.
+      Evaluates to `[ "FOO" ]`.
     )s",
     .fun = prim_match,
 });

src/libexpr/tests/local.mk

@@ -10,6 +10,6 @@ libexpr-tests_SOURCES := $(wildcard $(d)/*.cc)
 
 libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests
 
-libexpr-tests_LIBS = libexpr libutil libstore
+libexpr-tests_LIBS = libexpr libutil libstore libfetchers
 
 libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock

src/libfetchers/git.cc

@@ -26,11 +26,6 @@ namespace {
 // old version of git, which will ignore unrecognized `-c` options.
 const std::string gitInitialBranch = "__nix_dummy_branch";
 
-std::string getGitDir()
-{
-    return getEnv("GIT_DIR").value_or(".git");
-}
-
 bool isCacheFileWithinTtl(const time_t now, const struct stat & st)
 {
     return st.st_mtime + settings.tarballTtl > now;
@@ -152,7 +147,7 @@ struct WorkdirInfo
 WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
 {
     const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-    auto gitDir = getGitDir();
+    std::string gitDir(".git");
 
     auto env = getEnv();
     // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
@@ -370,7 +365,7 @@ struct GitInputScheme : InputScheme
     {
         auto sourcePath = getSourcePath(input);
         assert(sourcePath);
-        auto gitDir = getGitDir();
+        auto gitDir = ".git";
 
         runProgram("git", true,
             { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) });
@@ -396,7 +391,7 @@ struct GitInputScheme : InputScheme
     std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
     {
         Input input(_input);
-        auto gitDir = getGitDir();
+        auto gitDir = ".git";
 
         std::string name = input.getName();
 

src/libfetchers/github.cc

@@ -243,7 +243,10 @@ struct GitHubInputScheme : GitArchiveInputScheme
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
+        auto url = fmt(
+            host == "github.com"
+            ? "https://api.%s/repos/%s/%s/commits/%s"
+            : "https://%s/api/v3/repos/%s/%s/commits/%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
         Headers headers = makeHeadersWithAuthTokens(host);
@@ -262,7 +265,10 @@ struct GitHubInputScheme : GitArchiveInputScheme
         // FIXME: use regular /archive URLs instead? api.github.com
         // might have stricter rate limits.
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
+        auto url = fmt(
+            host == "github.com"
+            ? "https://api.%s/repos/%s/%s/tarball/%s"
+            : "https://%s/api/v3/repos/%s/%s/tarball/%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 

src/libfetchers/tarball.cc

@@ -6,6 +6,7 @@
 #include "archive.hh"
 #include "tarfile.hh"
 #include "types.hh"
+#include "split.hh"
 
 namespace nix::fetchers {
 
@@ -168,24 +169,34 @@ std::pair<Tree, time_t> downloadTarball(
     };
 }
 
-struct TarballInputScheme : InputScheme
+// An input scheme corresponding to a curl-downloadable resource.
+struct CurlInputScheme : InputScheme
 {
+    virtual const std::string inputType() const = 0;
+    const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
+
+    const bool hasTarballExtension(std::string_view path) const
+    {
+        return hasSuffix(path, ".zip") || hasSuffix(path, ".tar")
+            || hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz")
+            || hasSuffix(path, ".tar.xz") || hasSuffix(path, ".tar.bz2")
+            || hasSuffix(path, ".tar.zst");
+    }
+
+    virtual bool isValidURL(const ParsedURL & url) const = 0;
+
     std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
-        if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {};
+        if (!isValidURL(url))
-
+            return std::nullopt;
-        if (!hasSuffix(url.path, ".zip")
-            && !hasSuffix(url.path, ".tar")
-            && !hasSuffix(url.path, ".tgz")
-            && !hasSuffix(url.path, ".tar.gz")
-            && !hasSuffix(url.path, ".tar.xz")
-            && !hasSuffix(url.path, ".tar.bz2")
-            && !hasSuffix(url.path, ".tar.zst"))
-            return {};
 
         Input input;
-        input.attrs.insert_or_assign("type", "tarball");
+
-        input.attrs.insert_or_assign("url", url.to_string());
+        auto urlWithoutApplicationScheme = url;
+        urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport;
+
+        input.attrs.insert_or_assign("type", inputType());
+        input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string());
         auto narHash = url.query.find("narHash");
         if (narHash != url.query.end())
             input.attrs.insert_or_assign("narHash", narHash->second);
@@ -194,14 +205,17 @@ struct TarballInputScheme : InputScheme
 
     std::optional<Input> inputFromAttrs(const Attrs & attrs) override
     {
-        if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
+        auto type = maybeGetStrAttr(attrs, "type");
+        if (type != inputType()) return {};
 
+        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack"};
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name")
+            if (!allowedNames.count(name))
-                throw Error("unsupported tarball input attribute '%s'", name);
+                throw Error("unsupported %s input attribute '%s'", *type, name);
 
         Input input;
         input.attrs = attrs;
+
         //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash");
         return input;
     }
@@ -209,14 +223,9 @@ struct TarballInputScheme : InputScheme
     ParsedURL toURL(const Input & input) override
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
-        // NAR hashes are preferred over file hashes since tar/zip files
+        // NAR hashes are preferred over file hashes since tar/zip files        // don't have a canonical representation.
-        // don't have a canonical representation.
         if (auto narHash = input.getNarHash())
             url.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
-        /*
-        else if (auto hash = maybeGetStrAttr(input.attrs, "hash"))
-            url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true));
-        */
         return url;
     }
 
@@ -225,6 +234,42 @@ struct TarballInputScheme : InputScheme
         return true;
     }
 
+};
+
+struct FileInputScheme : CurlInputScheme
+{
+    const std::string inputType() const override { return "file"; }
+
+    bool isValidURL(const ParsedURL & url) const override
+    {
+        auto parsedUrlScheme = parseUrlScheme(url.scheme);
+        return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
+            && (parsedUrlScheme.application
+                    ? parsedUrlScheme.application.value() == inputType()
+                    : !hasTarballExtension(url.path));
+    }
+
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
+    {
+        auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false);
+        return {std::move(file.storePath), input};
+    }
+};
+
+struct TarballInputScheme : CurlInputScheme
+{
+    const std::string inputType() const override { return "tarball"; }
+
+    bool isValidURL(const ParsedURL & url) const override
+    {
+        auto parsedUrlScheme = parseUrlScheme(url.scheme);
+
+        return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
+            && (parsedUrlScheme.application
+                    ? parsedUrlScheme.application.value() == inputType()
+                    : hasTarballExtension(url.path));
+    }
+
     std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
     {
         auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
@@ -233,5 +278,6 @@ struct TarballInputScheme : InputScheme
 };
 
 static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
+static auto rFileInputScheme = OnStartup([] { registerInputScheme(std::make_unique<FileInputScheme>()); });
 
 }

src/libstore/builtins/buildenv.cc

@@ -93,8 +93,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     auto prevPriority = state.priorities[dstFile];
                     if (prevPriority == priority)
                         throw Error(
-                                "packages '%1%' and '%2%' have the same priority %3%; "
+                                "files '%1%' and '%2%' have the same priority %3%; "
                                 "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
+                                "or type 'nix profile install --help' if using 'nix profile' to find out how"
                                 "to change the priority of one of the conflicting packages"
                                 " (0 being the highest priority)",
                                 srcFile, readLink(dstFile), priority);

src/libutil/url.cc

@@ -1,6 +1,7 @@
 #include "url.hh"
 #include "url-parts.hh"
 #include "util.hh"
+#include "split.hh"
 
 namespace nix {
 
@@ -136,4 +137,21 @@ bool ParsedURL::operator ==(const ParsedURL & other) const
         && fragment == other.fragment;
 }
 
+/**
+ * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme'
+ * into a tuple '(applicationScheme, transportScheme)'
+ *
+ * > parseUrlScheme("http") == ParsedUrlScheme{ {}, "http"}
+ * > parseUrlScheme("tarball+http") == ParsedUrlScheme{ {"tarball"}, "http"}
+ */
+ParsedUrlScheme parseUrlScheme(std::string_view scheme)
+{
+    auto application = splitPrefixTo(scheme, '+');
+    auto transport = scheme;
+    return ParsedUrlScheme {
+        .application = application,
+        .transport = transport,
+    };
+}
+
 }

src/libutil/url.hh

@@ -27,4 +27,19 @@ std::map<std::string, std::string> decodeQuery(const std::string & query);
 
 ParsedURL parseURL(const std::string & url);
 
+/*
+ * Although that’s not really standardized anywhere, an number of tools
+ * use a scheme of the form 'x+y' in urls, where y is the “transport layer”
+ * scheme, and x is the “application layer” scheme.
+ *
+ * For example git uses `git+https` to designate remotes using a Git
+ * protocol over http.
+ */
+struct ParsedUrlScheme {
+    std::optional<std::string_view> application;
+    std::string_view transport;
+};
+
+ParsedUrlScheme parseUrlScheme(std::string_view scheme);
+
 }

src/nix-build/nix-build.cc

@@ -543,8 +543,6 @@ static void main_nix_build(int argc, char * * argv)
 
         restoreProcessContext();
 
-        logger->stop();
-
         execvp(shell->c_str(), argPtrs.data());
 
         throw SysError("executing shell '%s'", *shell);
@@ -603,8 +601,6 @@ static void main_nix_build(int argc, char * * argv)
             outPaths.push_back(outputPath);
         }
 
-        logger->stop();
-
         for (auto & path : outPaths)
             std::cout << store->printStorePath(path) << '\n';
     }

src/nix-env/nix-env.cc

@@ -1489,8 +1489,6 @@ static int main_nix_env(int argc, char * * argv)
 
         globals.state->printStats();
 
-        logger->stop();
-
         return 0;
     }
 }

src/nix-store/nix-store.cc

@@ -1095,8 +1095,6 @@ static int main_nix_store(int argc, char * * argv)
 
         op(opFlags, opArgs);
 
-        logger->stop();
-
         return 0;
     }
 }

src/nix/develop.cc

@@ -18,6 +18,9 @@ struct DevelopSettings : Config
     Setting<std::string> bashPrompt{this, "", "bash-prompt",
         "The bash prompt (`PS1`) in `nix develop` shells."};
 
+    Setting<std::string> bashPromptPrefix{this, "", "bash-prompt-prefix",
+        "Prefix prepended to the `PS1` environment variable in `nix develop` shells."};
+
     Setting<std::string> bashPromptSuffix{this, "", "bash-prompt-suffix",
         "Suffix appended to the `PS1` environment variable in `nix develop` shells."};
 };
@@ -482,6 +485,9 @@ struct CmdDevelop : Common, MixEnvironment
             if (developSettings.bashPrompt != "")
                 script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n",
                     shellEscape(developSettings.bashPrompt.get()));
+            if (developSettings.bashPromptPrefix != "")
+                script += fmt("[ -n \"$PS1\" ] && PS1=%s\"$PS1\";\n",
+                    shellEscape(developSettings.bashPromptPrefix.get()));
             if (developSettings.bashPromptSuffix != "")
                 script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n",
                     shellEscape(developSettings.bashPromptSuffix.get()));
@@ -512,9 +518,20 @@ struct CmdDevelop : Common, MixEnvironment
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
                 nixpkgsLockFlags);
 
-            shell = store->printStorePath(
+            bool found = false;
-                Installable::toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable))
+
-                + "/bin/bash";
+            for (auto & path : Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) {
+                auto s = store->printStorePath(path) + "/bin/bash";
+                if (pathExists(s)) {
+                    shell = s;
+                    found = true;
+                    break;
+                }
+            }
+
+            if (!found)
+                throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'");
+
         } catch (Error &) {
             ignoreException();
         }

src/nix/develop.md

@@ -80,8 +80,8 @@ initialised by `stdenv` and exits. This build environment can be
 recorded into a profile using `--profile`.
 
 The prompt used by the `bash` shell can be customised by setting the
-`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in
+`bash-prompt`, `bash-prompt-prefix`, and `bash-prompt-suffix` settings in
-the flake's `nixConfig` attribute.
+`nix.conf` or in the flake's `nixConfig` attribute.
 
 # Flake output attributes
 

src/nix/flake.cc

@@ -509,7 +509,7 @@ struct CmdFlakeCheck : FlakeCommand
 
                         std::string_view replacement =
                             name == "defaultPackage" ? "packages.<system>.default" :
-                            name == "defaultApps" ? "apps.<system>.default" :
+                            name == "defaultApp" ? "apps.<system>.default" :
                             name == "defaultTemplate" ? "templates.default" :
                             name == "defaultBundler" ? "bundlers.<system>.default" :
                             name == "overlay" ? "overlays.default" :
@@ -1076,9 +1076,13 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
                 else if (attrPath.size() > 0 && attrPathS[0] == "legacyPackages") {
                     if (attrPath.size() == 1)
                         recurse();
-                    else if (!showLegacy)
+                    else if (!showLegacy){
-                        logger->warn(fmt("%s: " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix));
+                        if (!json)
+                            logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix));
                         else {
+                            logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS)));
+                        }
+                    } else {
                         if (visitor.isDerivation())
                             showDerivation();
                         else if (attrPath.size() <= 2)

src/nix/flake.md

@@ -181,9 +181,17 @@ Currently the `type` attribute can be one of the following:
 * `tarball`: Tarballs. The location of the tarball is specified by the
   attribute `url`.
 
-  In URL form, the schema must be `http://`, `https://` or `file://`
+  In URL form, the schema must be `tarball+http://`, `tarball+https://` or `tarball+file://`.
-  URLs and the extension must be `.zip`, `.tar`, `.tgz`, `.tar.gz`,
+  If the extension corresponds to a known archive format (`.zip`, `.tar`,
-  `.tar.xz`, `.tar.bz2` or `.tar.zst`.
+  `.tgz`, `.tar.gz`, `.tar.xz`, `.tar.bz2` or `.tar.zst`), then the `tarball+`
+  can be dropped.
+
+* `file`: Plain files or directory tarballs, either over http(s) or from the local
+  disk.
+
+  In URL form, the schema must be `file+http://`, `file+https://` or `file+file://`.
+  If the extension doesn’t correspond to a known archive format (as defined by the
+  `tarball` fetcher), then the `file+` prefix can be dropped.
 
 * `github`: A more efficient way to fetch repositories from
   GitHub. The following attributes are required:
@@ -331,9 +339,10 @@ The following attributes are supported in `flake.nix`:
 
 * `nixConfig`: a set of `nix.conf` options to be set when evaluating any
   part of a flake. In the interests of security, only a small set of
-  whitelisted options (currently `bash-prompt`, `bash-prompt-suffix`,
+  whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`,
-  and `flake-registry`) are allowed to be set without confirmation so long as
+  `bash-prompt-suffix`, and `flake-registry`) are allowed to be set without
-  `accept-flake-config` is not set in the global configuration.
+  confirmation so long as `accept-flake-config` is not set in the global
+  configuration.
 
 ## Flake inputs
 

src/nix/main.cc

@@ -261,6 +261,8 @@ void mainWrapped(int argc, char * * argv)
     }
     #endif
 
+    Finally f([] { logger->stop(); });
+
     programPath = argv[0];
     auto programName = std::string(baseNameOf(programPath));
 
@@ -279,8 +281,6 @@ void mainWrapped(int argc, char * * argv)
         verbosity = lvlInfo;
     }
 
-    Finally f([] { logger->stop(); });
-
     NixArgs args;
 
     if (argc == 2 && std::string(argv[1]) == "__dump-args") {

src/nix/profile.cc

@@ -37,7 +37,7 @@ struct ProfileElement
     StorePathSet storePaths;
     std::optional<ProfileElementSource> source;
     bool active = true;
-    // FIXME: priority
+    int priority = 5;
 
     std::string describe() const
     {
@@ -116,6 +116,9 @@ struct ProfileManifest
                 for (auto & p : e["storePaths"])
                     element.storePaths.insert(state.store->parseStorePath((std::string) p));
                 element.active = e["active"];
+                if(e.contains("priority")) {
+                    element.priority = e["priority"];
+                }
                 if (e.value(sUrl, "") != "") {
                     element.source = ProfileElementSource {
                         parseFlakeRef(e[sOriginalUrl]),
@@ -153,6 +156,7 @@ struct ProfileManifest
             nlohmann::json obj;
             obj["storePaths"] = paths;
             obj["active"] = element.active;
+            obj["priority"] = element.priority;
             if (element.source) {
                 obj["originalUrl"] = element.source->originalRef.to_string();
                 obj["url"] = element.source->resolvedRef.to_string();
@@ -177,7 +181,7 @@ struct ProfileManifest
         for (auto & element : elements) {
             for (auto & path : element.storePaths) {
                 if (element.active)
-                    pkgs.emplace_back(store->printStorePath(path), true, 5);
+                    pkgs.emplace_back(store->printStorePath(path), true, element.priority);
                 references.insert(path);
             }
         }
@@ -259,6 +263,17 @@ builtPathsPerInstallable(
 
 struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
 {
+    std::optional<int64_t> priority;
+
+    CmdProfileInstall() {
+        addFlag({
+            .longName = "priority",
+            .description = "The priority of the package to install.",
+            .labels = {"priority"},
+            .handler = {&priority},
+        });
+    };
+
     std::string description() override
     {
         return "install a package into a profile";
@@ -282,6 +297,8 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
         for (auto & installable : installables) {
             ProfileElement element;
 
+
+
             if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
                 // FIXME: make build() return this?
                 auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
@@ -291,7 +308,15 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                     attrPath,
                     installable2->outputsSpec
                 };
+
+                if(drv.priority) {
+                    element.priority = *drv.priority;
                 }
+            }
+
+            if(priority) { // if --priority was specified we want to override the priority of the installable
+                element.priority = *priority;
+            };
 
             element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
 

tests/fetchTree-file.sh

@@ -0,0 +1,105 @@
+source common.sh
+
+clearStore
+
+cd "$TEST_ROOT"
+
+test_fetch_file () {
+    echo foo > test_input
+
+    input_hash="$(nix hash path test_input)"
+
+    nix eval --impure --file - <<EOF
+    let
+        tree = builtins.fetchTree { type = "file"; url = "file://$PWD/test_input"; };
+    in
+    assert (tree.narHash == "$input_hash");
+    tree
+EOF
+}
+
+# Make sure that `http(s)` and `file` flake inputs are properly extracted when
+# they should be, and treated as opaque files when they should be
+test_file_flake_input () {
+    rm -fr "$TEST_ROOT/testFlake";
+    mkdir "$TEST_ROOT/testFlake";
+    pushd testFlake
+
+    mkdir inputs
+    echo foo > inputs/test_input_file
+    tar cfa test_input.tar.gz inputs
+    cp test_input.tar.gz test_input_no_ext
+    input_tarball_hash="$(nix hash path test_input.tar.gz)"
+    input_directory_hash="$(nix hash path inputs)"
+
+    cat <<EOF > flake.nix
+    {
+        inputs.no_ext_default_no_unpack = {
+            url = "file://$PWD/test_input_no_ext";
+            flake = false;
+        };
+        inputs.no_ext_explicit_unpack = {
+            url = "tarball+file://$PWD/test_input_no_ext";
+            flake = false;
+        };
+        inputs.tarball_default_unpack = {
+            url = "file://$PWD/test_input.tar.gz";
+            flake = false;
+        };
+        inputs.tarball_explicit_no_unpack = {
+            url = "file+file://$PWD/test_input.tar.gz";
+            flake = false;
+        };
+        outputs = { ... }: {};
+    }
+EOF
+
+    nix flake update
+    nix eval --file - <<EOF
+    with (builtins.fromJSON (builtins.readFile ./flake.lock));
+
+    # Url inputs whose extension doesn’t match a know archive format should
+    # not be unpacked by default
+    assert (nodes.no_ext_default_no_unpack.locked.type == "file");
+    assert (nodes.no_ext_default_no_unpack.locked.unpack or false == false);
+    assert (nodes.no_ext_default_no_unpack.locked.narHash == "$input_tarball_hash");
+
+    # For backwards compatibility, flake inputs that correspond to the
+    # old 'tarball' fetcher should still have their type set to 'tarball'
+    assert (nodes.tarball_default_unpack.locked.type == "tarball");
+    # Unless explicitely specified, the 'unpack' parameter shouldn’t appear here
+    # because that would break older Nix versions
+    assert (!nodes.tarball_default_unpack.locked ? unpack);
+    assert (nodes.tarball_default_unpack.locked.narHash == "$input_directory_hash");
+
+    # Explicitely passing the unpack parameter should enforce the desired behavior
+    assert (nodes.no_ext_explicit_unpack.locked.narHash == nodes.tarball_default_unpack.locked.narHash);
+    assert (nodes.tarball_explicit_no_unpack.locked.narHash == nodes.no_ext_default_no_unpack.locked.narHash);
+    true
+EOF
+    popd
+
+    [[ -z "${NIX_DAEMON_PACKAGE}" ]] && return 0
+
+    # Ensure that a lockfile generated by the current Nix for tarball inputs
+    # can still be read by an older Nix
+
+    cat <<EOF > flake.nix
+    {
+        inputs.tarball = {
+            url = "file://$PWD/test_input.tar.gz";
+            flake = false;
+        };
+        outputs = { self, tarball }: {
+            foo = builtins.readFile "${tarball}/test_input_file";
+        };
+    }
+    nix flake update
+
+    clearStore
+    "$NIX_DAEMON_PACKAGE/bin/nix" eval .#foo
+EOF
+}
+
+test_fetch_file
+test_file_flake_input

tests/flakes.sh

@@ -163,6 +163,7 @@ nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Di
 # But should succeed in impure mode.
 (! nix build -o $TEST_ROOT/result flake2#bar --impure)
 nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file
+nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure
 
 # Building a local flake with an unlocked dependency should fail with --no-update-lock-file.
 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes'

tests/local.mk

@@ -23,6 +23,7 @@ nix_tests = \
   fetchGit.sh \
   fetchurl.sh \
   fetchPath.sh \
+  fetchTree-file.sh \
   simple.sh \
   referrers.sh \
   optimise-store.sh \

tests/nix-profile.sh

@@ -120,3 +120,21 @@ nix profile install "$flake1Dir^man"
 (! [ -e $TEST_HOME/.nix-profile/bin/hello ])
 [ -e $TEST_HOME/.nix-profile/share/man ]
 (! [ -e $TEST_HOME/.nix-profile/include ])
+
+# test priority
+nix profile remove 0
+
+# Make another flake.
+flake2Dir=$TEST_ROOT/flake2
+printf World > $flake1Dir/who
+cp -r $flake1Dir $flake2Dir
+printf World2 > $flake2Dir/who
+
+nix profile install $flake1Dir
+[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
+nix profile install $flake2Dir --priority 100
+[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
+nix profile install $flake2Dir --priority 0
+[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]]
+# nix profile install $flake1Dir --priority 100
+# [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]