From 240124f7b1761073c4db24305a5decdeb597f549 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Wed, 4 May 2022 16:55:59 -0500 Subject: [PATCH 01/29] darwin-install: fix break from bad vimrc It looks like the `--noplugin` flag added in #5489 wasn't enough to skirt this class of vim-init error, so this is swing 2 at a full fix. Fixes #6462. --- scripts/create-darwin-volume.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 4bac4b7ba..aee7ff4bf 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -442,9 +442,13 @@ add_nix_vol_fstab_line() { local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}" shift - # wrap `ex` to work around a problem with vim plugins breaking exit codes; - # (see https://github.com/NixOS/nix/issues/5468) - # we'd prefer EDITOR="/usr/bin/ex --noplugin" but vifs doesn't word-split + # wrap `ex` to work around a problem with vim plugins breaking exit codes + # (see github.com/NixOS/nix/issues/5468) + # + # the first draft used `--noplugin`, but github.com/NixOS/nix/issues/6462 + # suggests we need the less-semantic `-u NONE` + # + # we'd prefer EDITOR="/usr/bin/ex -u NONE" but vifs doesn't word-split # the EDITOR env. # # TODO: at some point we should switch to `--clean`, but it wasn't added @@ -452,7 +456,7 @@ add_nix_vol_fstab_line() { # minver 10.12.6 seems to have released with vim 7.4 cat > "$SCRATCH/ex_cleanroom_wrapper" <&2 # technically /etc/synthetic.d/nix is supported in Big Sur+ # but handling both takes even more code... + # Note: `-u NONE` disables vim plugins/rc; see note on --clean earlier _sudo "to add Nix to /etc/synthetic.conf" \ - /usr/bin/ex --noplugin /etc/synthetic.conf <&2 - _sudo "to install the Nix volume mounter" /usr/bin/ex --noplugin "$NIX_VOLUME_MOUNTD_DEST" < Date: Thu, 5 May 2022 10:46:48 -0700 Subject: [PATCH 02/29] Use correct URL for GitHub Enterprise For GitHub Enterprise, the API is accessed through a slightly different URL. See [1], where it says: > Use http(s)://[hostname]/api/v3 to access the API for GitHub > Enterprise Server. Also tested working on a GHE instance. [1] https://docs.github.com/en/enterprise-server@3.3/rest/guides/getting-started-with-the-rest-api --- src/libfetchers/github.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1bdf2759f..50b3150ee 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -243,7 +243,10 @@ struct GitHubInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check + auto url = fmt( + host == "github.com" + ? "https://api.%s/repos/%s/%s/commits/%s" + : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); Headers headers = makeHeadersWithAuthTokens(host); @@ -262,7 +265,10 @@ struct GitHubInputScheme : GitArchiveInputScheme // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances + auto url = fmt( + host == "github.com" + ? "https://api.%s/repos/%s/%s/commits/%s" + : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); From c060e93b3c48c5d6cfde88ee9080a10f73f00fe4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 22:01:15 +0000 Subject: [PATCH 03/29] Bump docker/login-action from 1 to 2 Bumps [docker/login-action](https://github.com/docker/login-action) from 1 to 2. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d01ef4768..aae5b93e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From 7c75f1d52b3078608be29cbe0b009875829cc03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 10 May 2022 11:56:47 +0200 Subject: [PATCH 04/29] Expand the testing section of the hacking docs - Make it clear what the different kind of tests are, where they live and how they can be ran - Ask people to primarily write unit tests --- doc/manual/src/contributing/hacking.md | 41 ++++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 90a8f1f94..7ce8d8de6 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -71,18 +71,6 @@ To install it in `$(pwd)/outputs` and test it: nix (Nix) 3.0 ``` -To run a functional test: - -```console -make tests/test-name-should-auto-complete.sh.test -``` - -To run the unit-tests for C++ code: - -``` -make check -``` - If you have a flakes-enabled Nix you can replace: ```console @@ -94,3 +82,32 @@ by: ```console $ nix develop ``` + +## Testing + +Nix comes with three different flavors of tests: unit, functional and integration. + +Most tests are currently written as functional tests. +**However**, it is preferable (as much as it makes sense) to primarily test new code with unit tests. + +### Unit-tests + +The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined +under `src/{library_name}/tests` using the +[googletest](https://google.github.io/googletest/) framework. + +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. + +### Functional tests + +The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +The whole testsuite can be run with `make install && make installcheck`. +Individual tests can be run with `make tests/{testName}.sh.test`. + +### Integration tests + +The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. +These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. +Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). + +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` From a9cbc2857f4dd3af738b76edea00d692fbcee63c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 May 2022 16:42:35 +0200 Subject: [PATCH 05/29] nix develop: Find bin/bash in the bashInteractive outputs --- src/nix/develop.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1190b8348..3a99fff6f 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -512,9 +512,20 @@ struct CmdDevelop : Common, MixEnvironment Strings{"legacyPackages." + settings.thisSystem.get() + "."}, nixpkgsLockFlags); - shell = store->printStorePath( - Installable::toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable)) - + "/bin/bash"; + bool found = false; + + for (auto & path : Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { + auto s = store->printStorePath(path) + "/bin/bash"; + if (pathExists(s)) { + shell = s; + found = true; + break; + } + } + + if (!found) + throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'"); + } catch (Error &) { ignoreException(); } From 2998527b185a41157be6ead42fd03a66601c4f56 Mon Sep 17 00:00:00 2001 From: Jimmy Reichley Date: Tue, 10 May 2022 16:53:22 -0400 Subject: [PATCH 06/29] Allow setting bash-prompt-prefix nix develop configuration --- src/libexpr/flake/config.cc | 2 +- src/nix/develop.cc | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 92ec27046..3e9d264b4 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -31,7 +31,7 @@ static void writeTrustedList(const TrustedList & trustedList) void ConfigFile::apply() { - std::set whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"}; + std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry"}; for (auto & [name, value] : settings) { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 3a99fff6f..2a3fc0213 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -18,6 +18,9 @@ struct DevelopSettings : Config Setting bashPrompt{this, "", "bash-prompt", "The bash prompt (`PS1`) in `nix develop` shells."}; + Setting bashPromptPrefix{this, "", "bash-prompt-prefix", + "Prefix prepended to the `PS1` environment variable in `nix develop` shells."}; + Setting bashPromptSuffix{this, "", "bash-prompt-suffix", "Suffix appended to the `PS1` environment variable in `nix develop` shells."}; }; @@ -482,6 +485,9 @@ struct CmdDevelop : Common, MixEnvironment if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt.get())); + if (developSettings.bashPromptPrefix != "") + script += fmt("[ -n \"$PS1\" ] && PS1=%s\"$PS1\";\n", + shellEscape(developSettings.bashPromptPrefix.get())); if (developSettings.bashPromptSuffix != "") script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", shellEscape(developSettings.bashPromptSuffix.get())); From 584475acf9f4b8eda2a451901f6f9af35ae976e0 Mon Sep 17 00:00:00 2001 From: Jimmy Reichley Date: Tue, 10 May 2022 16:55:25 -0400 Subject: [PATCH 07/29] Add documentation for bash-prompt-prefix --- src/nix/develop.md | 4 ++-- src/nix/flake.md | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/nix/develop.md b/src/nix/develop.md index 8bcff66c9..e036ec6b9 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -80,8 +80,8 @@ initialised by `stdenv` and exits. This build environment can be recorded into a profile using `--profile`. The prompt used by the `bash` shell can be customised by setting the -`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in -the flake's `nixConfig` attribute. +`bash-prompt`, `bash-prompt-prefix`, and `bash-prompt-suffix` settings in +`nix.conf` or in the flake's `nixConfig` attribute. # Flake output attributes diff --git a/src/nix/flake.md b/src/nix/flake.md index c8251eb74..aa3f9f303 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -331,9 +331,10 @@ The following attributes are supported in `flake.nix`: * `nixConfig`: a set of `nix.conf` options to be set when evaluating any part of a flake. In the interests of security, only a small set of - whitelisted options (currently `bash-prompt`, `bash-prompt-suffix`, - and `flake-registry`) are allowed to be set without confirmation so long as - `accept-flake-config` is not set in the global configuration. + whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`, + `bash-prompt-suffix`, and `flake-registry`) are allowed to be set without + confirmation so long as `accept-flake-config` is not set in the global + configuration. ## Flake inputs From 54457382f948bff30e2879a7d9047616e134ac5b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 May 2022 11:36:56 +0200 Subject: [PATCH 08/29] Fix static build https://hydra.nixos.org/build/176211267 --- src/libexpr/tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index cb1c4a977..b95980cab 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -10,6 +10,6 @@ libexpr-tests_SOURCES := $(wildcard $(d)/*.cc) libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -libexpr-tests_LIBS = libexpr libutil libstore +libexpr-tests_LIBS = libexpr libutil libstore libfetchers libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock From aefc6c4f41bfac0c76807c234fd0a786dd40f140 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Wed, 11 May 2022 12:15:08 +0200 Subject: [PATCH 09/29] Add priority for nix profile install --- src/libstore/builtins/buildenv.cc | 3 ++- src/nix/profile.cc | 29 +++++++++++++++++++++++++++-- tests/nix-profile.sh | 16 ++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 6f6ad57cb..c17c76e71 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -93,8 +93,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) throw Error( - "packages '%1%' and '%2%' have the same priority %3%; " + "files '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " + "or 'nix profile --priority NUMBER INSTALLED_PKGNAME' " "to change the priority of one of the conflicting packages" " (0 being the highest priority)", srcFile, readLink(dstFile), priority); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 685776bec..fb8bef670 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -37,7 +37,7 @@ struct ProfileElement StorePathSet storePaths; std::optional source; bool active = true; - // FIXME: priority + int priority = 5; std::string describe() const { @@ -116,6 +116,9 @@ struct ProfileManifest for (auto & p : e["storePaths"]) element.storePaths.insert(state.store->parseStorePath((std::string) p)); element.active = e["active"]; + if(e.contains("priority")) { + element.priority = e["priority"]; + } if (e.value(sUrl, "") != "") { element.source = ProfileElementSource { parseFlakeRef(e[sOriginalUrl]), @@ -153,6 +156,7 @@ struct ProfileManifest nlohmann::json obj; obj["storePaths"] = paths; obj["active"] = element.active; + obj["priority"] = element.priority; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); obj["url"] = element.source->resolvedRef.to_string(); @@ -177,7 +181,7 @@ struct ProfileManifest for (auto & element : elements) { for (auto & path : element.storePaths) { if (element.active) - pkgs.emplace_back(store->printStorePath(path), true, 5); + pkgs.emplace_back(store->printStorePath(path), true, element.priority); references.insert(path); } } @@ -259,6 +263,23 @@ builtPathsPerInstallable( struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { + std::optional priority; + CmdProfileInstall() { + addFlag({ + .longName = "priority", + .description = "The priority of the package to install.", + .labels = {"priority"}, + .handler = {[&](std::string s) { + try{ + priority = std::stoi(s); + } catch (std::invalid_argument & e) { + throw ParseError("invalid priority '%s'", s); + } + }}, + // .completer = // no completer since number + }); + }; + std::string description() override { return "install a package into a profile"; @@ -282,6 +303,10 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; + if(priority) { + element.priority = *priority; + }; + if (auto installable2 = std::dynamic_pointer_cast(installable)) { // FIXME: make build() return this? auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index f8da3d929..1cc724483 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -120,3 +120,19 @@ nix profile install "$flake1Dir^man" (! [ -e $TEST_HOME/.nix-profile/bin/hello ]) [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) + +# test priority +nix profile remove 0 + +# Make another flake. +flake2Dir=$TEST_ROOT/flake2 +printf World > $flake1Dir/who +cp -r $flake1Dir $flake2Dir +printf World2 > $flake2Dir/who + +nix profile install $flake1Dir +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +nix profile install $flake2Dir --priority 100 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +nix profile install $flake2Dir --priority 0 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]] From 1461e6cdda06f7f461114cce5b415f6d50381311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 11 May 2022 12:55:31 +0200 Subject: [PATCH 10/29] Stop the logger properly in legacy commands Ensures the logger is stopped on exit in legacy commands. Without this, when using `nix-build --log-format bar` and stopping nix with CTRL+C, the bar is not cleared from the screen. --- src/nix-build/nix-build.cc | 4 ---- src/nix-env/nix-env.cc | 2 -- src/nix-store/nix-store.cc | 2 -- src/nix/main.cc | 4 ++-- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 519855ea3..426f23905 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -543,8 +543,6 @@ static void main_nix_build(int argc, char * * argv) restoreProcessContext(); - logger->stop(); - execvp(shell->c_str(), argPtrs.data()); throw SysError("executing shell '%s'", *shell); @@ -603,8 +601,6 @@ static void main_nix_build(int argc, char * * argv) outPaths.push_back(outputPath); } - logger->stop(); - for (auto & path : outPaths) std::cout << store->printStorePath(path) << '\n'; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 96f3c3b26..c412bb814 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1489,8 +1489,6 @@ static int main_nix_env(int argc, char * * argv) globals.state->printStats(); - logger->stop(); - return 0; } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 153b84137..9163eefd0 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1095,8 +1095,6 @@ static int main_nix_store(int argc, char * * argv) op(opFlags, opArgs); - logger->stop(); - return 0; } } diff --git a/src/nix/main.cc b/src/nix/main.cc index 6d0f6ce6e..dadb54306 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -261,6 +261,8 @@ void mainWrapped(int argc, char * * argv) } #endif + Finally f([] { logger->stop(); }); + programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); @@ -279,8 +281,6 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlInfo; } - Finally f([] { logger->stop(); }); - NixArgs args; if (argc == 2 && std::string(argv[1]) == "__dump-args") { From 831e2743ea0bac57ebccb73f532667a194b938d7 Mon Sep 17 00:00:00 2001 From: Norbert Melzer Date: Thu, 12 May 2022 00:56:39 +0200 Subject: [PATCH 11/29] fix GitHub URL template --- src/libfetchers/github.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 0721a13f2..0631fb6e8 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -267,8 +267,8 @@ struct GitHubInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto url = fmt( host == "github.com" - ? "https://api.%s/repos/%s/%s/commits/%s" - : "https://%s/api/v3/repos/%s/%s/commits/%s", + ? "https://api.%s/repos/%s/%s/tarball/%s" + : "https://%s/api/v3/repos/%s/%s/tarball/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); From 65a913d29be7305b2c743fb92c93f0e6bb12d610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 12 May 2022 12:02:31 +0200 Subject: [PATCH 12/29] =?UTF-8?q?Don=E2=80=99t=20recommend=20writing=20uni?= =?UTF-8?q?t=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As asked in --- doc/manual/src/contributing/hacking.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 7ce8d8de6..59ce5cac7 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -87,9 +87,6 @@ $ nix develop Nix comes with three different flavors of tests: unit, functional and integration. -Most tests are currently written as functional tests. -**However**, it is preferable (as much as it makes sense) to primarily test new code with unit tests. - ### Unit-tests The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined From 8150b93968c648c6d273aaffaffba94096ec3faf Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 13 May 2022 11:12:11 -0400 Subject: [PATCH 13/29] fix: alignment during flake show of legacyPackages Fixes: https://github.com/NixOS/nix/issues/6240 https://github.com/NixOS/nix/issues/6045 --- src/nix/flake.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1938ce4e6..f55929751 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1076,9 +1076,13 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else if (attrPath.size() > 0 && attrPathS[0] == "legacyPackages") { if (attrPath.size() == 1) recurse(); - else if (!showLegacy) - logger->warn(fmt("%s: " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); - else { + else if (!showLegacy){ + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); + } + } else { if (visitor.isDerivation()) showDerivation(); else if (attrPath.size() <= 2) From be2b19041eeec53fba24f7c2494f3f700a4ec595 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Fri, 13 May 2022 22:02:28 +0200 Subject: [PATCH 14/29] Integrate review changes --- src/libcmd/installables.cc | 11 ++++++++--- src/libcmd/installables.hh | 3 ++- src/libexpr/eval-cache.cc | 22 ++++++++++++++++++++++ src/libexpr/eval-cache.hh | 3 +++ src/libstore/builtins/buildenv.cc | 2 +- src/nix/profile.cc | 25 ++++++++++++------------- tests/nix-profile.sh | 2 ++ 7 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index a94e60aca..3f6dfd592 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -609,7 +609,7 @@ InstallableFlake::InstallableFlake( throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); } -std::tuple InstallableFlake::toDerivation() +std::tuple> InstallableFlake::toDerivation() { auto attr = getCursor(*state); @@ -621,11 +621,15 @@ std::tuple InstallableF auto drvPath = attr->forceDerivation(); std::set outputsToInstall; + std::optional priority; - if (auto aMeta = attr->maybeGetAttr(state->sMeta)) + if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) for (auto & s : aOutputsToInstall->getListOfStrings()) outputsToInstall.insert(s); + if (auto aPriority = aMeta->maybeGetAttr("priority")) + priority = aPriority->getInt(); + } if (outputsToInstall.empty() || std::get_if(&outputsSpec)) { outputsToInstall.clear(); @@ -643,9 +647,10 @@ std::tuple InstallableF auto drvInfo = DerivationInfo { .drvPath = std::move(drvPath), .outputsToInstall = std::move(outputsToInstall), + .priority = priority, }; - return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; + return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo), priority}; } std::vector InstallableFlake::toDerivations() diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 1a5a96153..d7b61f1b8 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -142,6 +142,7 @@ struct InstallableValue : Installable { StorePath drvPath; std::set outputsToInstall; + std::optional priority; }; virtual std::vector toDerivations() = 0; @@ -176,7 +177,7 @@ struct InstallableFlake : InstallableValue Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - std::tuple toDerivation(); + std::tuple> toDerivation(); std::vector toDerivations() override; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 0eb4bc79e..b4bce512b 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -621,6 +621,28 @@ bool AttrCursor::getBool() return v.boolean; } +NixInt AttrCursor::getInt() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey()); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto i = std::get_if(&cachedValue->second)) { + debug("using cached Integer attribute '%s'", getAttrPathStr()); + return *i; + } else + throw TypeError("'%s' is not an Integer", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type() != nInt) + throw TypeError("'%s' is not an Integer", getAttrPathStr()); + + return v.integer; +} + std::vector AttrCursor::getListOfStrings() { if (root->db) { diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 636e293ad..105e9217b 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -63,6 +63,7 @@ typedef std::variant< misc_t, failed_t, bool, + NixInt, std::vector > AttrValue; @@ -116,6 +117,8 @@ public: bool getBool(); + NixInt getInt(); + std::vector getListOfStrings(); std::vector getAttrs(); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c17c76e71..58ef89a1c 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -95,7 +95,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw Error( "files '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "or 'nix profile --priority NUMBER INSTALLED_PKGNAME' " + "or 'nix profile install --priority NUMBER INSTALLED_PKGNAME' " "to change the priority of one of the conflicting packages" " (0 being the highest priority)", srcFile, readLink(dstFile), priority); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index fb8bef670..ca5041873 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -269,14 +269,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile .longName = "priority", .description = "The priority of the package to install.", .labels = {"priority"}, - .handler = {[&](std::string s) { - try{ - priority = std::stoi(s); - } catch (std::invalid_argument & e) { - throw ParseError("invalid priority '%s'", s); - } - }}, - // .completer = // no completer since number + .handler = {&priority}, }); }; @@ -303,21 +296,27 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; - if(priority) { - element.priority = *priority; - }; + if (auto installable2 = std::dynamic_pointer_cast(installable)) { // FIXME: make build() return this? - auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + auto [attrPath, resolvedRef, drv, priority] = installable2->toDerivation(); element.source = ProfileElementSource { installable2->flakeRef, resolvedRef, attrPath, installable2->outputsSpec }; + + if(drv.priority) { + element.priority = *drv.priority; + } } + if(priority) { // if --priority was specified we want to override the priority of the installable + element.priority = *priority; + }; + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); manifest.elements.push_back(std::move(element)); @@ -476,7 +475,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Strings{}, lockFlags); - auto [attrPath, resolvedRef, drv] = installable->toDerivation(); + auto [attrPath, resolvedRef, drv, priority] = installable->toDerivation(); if (element.source->resolvedRef == resolvedRef) continue; diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 1cc724483..7ba3235fa 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -136,3 +136,5 @@ nix profile install $flake2Dir --priority 100 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] nix profile install $flake2Dir --priority 0 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]] +# nix profile install $flake1Dir --priority 100 +# [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] From c81d24f1c70cc454c9a88cea70048d8563f60784 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Mon, 16 May 2022 02:28:21 +0200 Subject: [PATCH 15/29] Add int to eval-cache, bump eval cache schema version --- src/libexpr/eval-cache.cc | 24 +++++++++++++++++++++++- src/libexpr/eval-cache.hh | 1 + 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index b4bce512b..bf811c8ed 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -47,7 +47,7 @@ struct AttrDb { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v3"; + Path cacheDir = getCacheDir() + "/nix/eval-cache-v4"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; @@ -175,6 +175,24 @@ struct AttrDb }); } + AttrId setInt( + AttrKey key, + int n) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (symbols[key.second]) + (AttrType::Int) + (n).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + AttrId setListOfStrings( AttrKey key, const std::vector & l) @@ -287,6 +305,8 @@ struct AttrDb } case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; + case AttrType::Int: + return {{rowId, queryAttribute.getInt(2)}}; case AttrType::ListOfStrings: return {{rowId, tokenizeString>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: @@ -426,6 +446,8 @@ Value & AttrCursor::forceValue() cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; + else if (v.type() == nInt) + cachedValue = {root->db->setInt(getKey(), v.integer), v.integer}; else if (v.type() == nAttrs) ; // FIXME: do something? else diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 105e9217b..ec255c60d 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -45,6 +45,7 @@ enum AttrType { Failed = 5, Bool = 6, ListOfStrings = 7, + Int = 8, }; struct placeholder_t {}; From 27d0f6747d7e70be4b9ade28ce77444e6135cadb Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Mon, 16 May 2022 15:17:35 +0200 Subject: [PATCH 16/29] resolve redundant priority passing, wrap NixInt in eval-cache variant --- src/libcmd/installables.cc | 4 ++-- src/libcmd/installables.hh | 2 +- src/libexpr/eval-cache.cc | 6 +++--- src/libexpr/eval-cache.hh | 3 ++- src/nix/profile.cc | 4 ++-- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 3f6dfd592..635ce19b6 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -609,7 +609,7 @@ InstallableFlake::InstallableFlake( throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); } -std::tuple> InstallableFlake::toDerivation() +std::tuple InstallableFlake::toDerivation() { auto attr = getCursor(*state); @@ -650,7 +650,7 @@ std::tupleflake.lockedRef, std::move(drvInfo), priority}; + return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; } std::vector InstallableFlake::toDerivations() diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index d7b61f1b8..5d715210e 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -177,7 +177,7 @@ struct InstallableFlake : InstallableValue Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - std::tuple> toDerivation(); + std::tuple toDerivation(); std::vector toDerivations() override; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index bf811c8ed..6b3c27fd5 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -306,7 +306,7 @@ struct AttrDb case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; case AttrType::Int: - return {{rowId, queryAttribute.getInt(2)}}; + return {{rowId, (int_t) queryAttribute.getInt(2)}}; case AttrType::ListOfStrings: return {{rowId, tokenizeString>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: @@ -649,9 +649,9 @@ NixInt AttrCursor::getInt() if (!cachedValue) cachedValue = root->db->getAttr(getKey()); if (cachedValue && !std::get_if(&cachedValue->second)) { - if (auto i = std::get_if(&cachedValue->second)) { + if (auto i = std::get_if(&cachedValue->second)) { debug("using cached Integer attribute '%s'", getAttrPathStr()); - return *i; + return (*i).x; } else throw TypeError("'%s' is not an Integer", getAttrPathStr()); } diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index ec255c60d..68b5952eb 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -52,6 +52,7 @@ struct placeholder_t {}; struct missing_t {}; struct misc_t {}; struct failed_t {}; +struct int_t { NixInt x; int_t(NixInt x) : x(x) {}; }; typedef uint64_t AttrId; typedef std::pair AttrKey; typedef std::pair string_t; @@ -64,7 +65,7 @@ typedef std::variant< misc_t, failed_t, bool, - NixInt, + int_t, std::vector > AttrValue; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ca5041873..1aae347df 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -300,7 +300,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile if (auto installable2 = std::dynamic_pointer_cast(installable)) { // FIXME: make build() return this? - auto [attrPath, resolvedRef, drv, priority] = installable2->toDerivation(); + auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); element.source = ProfileElementSource { installable2->flakeRef, resolvedRef, @@ -475,7 +475,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Strings{}, lockFlags); - auto [attrPath, resolvedRef, drv, priority] = installable->toDerivation(); + auto [attrPath, resolvedRef, drv] = installable->toDerivation(); if (element.source->resolvedRef == resolvedRef) continue; From e53349dd6e4a710eb7abff78722853cad418e9d2 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Mon, 16 May 2022 16:16:06 +0200 Subject: [PATCH 17/29] change priority conflict message --- src/libstore/builtins/buildenv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 58ef89a1c..47458a388 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -95,7 +95,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw Error( "files '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "or 'nix profile install --priority NUMBER INSTALLED_PKGNAME' " + "or type 'nix profile install --help' if using 'nix profile' to find out how" "to change the priority of one of the conflicting packages" " (0 being the highest priority)", srcFile, readLink(dstFile), priority); From 43a2c1367292733d3e0aa2e57137c897fb66d8f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 16 May 2022 16:36:21 +0200 Subject: [PATCH 18/29] Make nix::eval_cache::int_t more idiomatic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don’t explicitely give it a constructor, but use aggregate initialization instead (also prevents having an implicit coertion, which is probably good here) --- src/libexpr/eval-cache.cc | 6 +++--- src/libexpr/eval-cache.hh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 6b3c27fd5..6a2e775d0 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -306,7 +306,7 @@ struct AttrDb case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; case AttrType::Int: - return {{rowId, (int_t) queryAttribute.getInt(2)}}; + return {{rowId, int_t{queryAttribute.getInt(2)}}}; case AttrType::ListOfStrings: return {{rowId, tokenizeString>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: @@ -447,7 +447,7 @@ Value & AttrCursor::forceValue() else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type() == nInt) - cachedValue = {root->db->setInt(getKey(), v.integer), v.integer}; + cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}}; else if (v.type() == nAttrs) ; // FIXME: do something? else @@ -651,7 +651,7 @@ NixInt AttrCursor::getInt() if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto i = std::get_if(&cachedValue->second)) { debug("using cached Integer attribute '%s'", getAttrPathStr()); - return (*i).x; + return i->x; } else throw TypeError("'%s' is not an Integer", getAttrPathStr()); } diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 68b5952eb..c93e55b93 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -52,7 +52,7 @@ struct placeholder_t {}; struct missing_t {}; struct misc_t {}; struct failed_t {}; -struct int_t { NixInt x; int_t(NixInt x) : x(x) {}; }; +struct int_t { NixInt x; }; typedef uint64_t AttrId; typedef std::pair AttrKey; typedef std::pair string_t; From b8e44dc62bb884b5c887852733819a909129d850 Mon Sep 17 00:00:00 2001 From: zhujun Date: Wed, 18 May 2022 14:05:26 +0800 Subject: [PATCH 19/29] primop_match: fix example letter case in document --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 28fea276e..fe4d0fc5f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3529,7 +3529,7 @@ static RegisterPrimOp primop_match({ builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " ``` - Evaluates to `[ "foo" ]`. + Evaluates to `[ "FOO" ]`. )s", .fun = prim_match, }); From 169384abb2bcfb687c8ad5959896738a76f3452e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 18 May 2022 11:33:04 +0200 Subject: [PATCH 20/29] Do not attempt to write a lock file in builtins.getFlake Fixes https://github.com/NixOS/nix/issues/6541 --- src/libexpr/flake/flake.cc | 1 + tests/flakes.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index cbf4f0a6f..35c841897 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -723,6 +723,7 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V lockFlake(state, flakeRef, LockFlags { .updateLockFile = false, + .writeLockFile = false, .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, .allowMutable = !evalSettings.pureEval, }), diff --git a/tests/flakes.sh b/tests/flakes.sh index 24601784f..9a1f0ab6a 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -163,6 +163,7 @@ nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Di # But should succeed in impure mode. (! nix build -o $TEST_ROOT/result flake2#bar --impure) nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file +nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure # Building a local flake with an unlocked dependency should fail with --no-update-lock-file. nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' From 5b8c1deb18e0e6fc7a83fb8101cf5fc8dba38843 Mon Sep 17 00:00:00 2001 From: Tony Olagbaiye Date: Fri, 16 Oct 2020 00:35:24 +0100 Subject: [PATCH 21/29] fetchTree: Allow fetching plain files Add a new `file` fetcher type, which will fetch a plain file over http(s), or from the local file. Because plain `http(s)://` or `file://` urls can already correspond to `tarball` inputs (if the path ends-up with a know archive extension), the URL parsing logic is a bit convuluted in that: - {http,https,file}:// urls will be interpreted as either a tarball or a file input, depending on the extensions of the path part (so `https://foo.com/bar` will be a `file` input and `https://foo.com/bar.tar.gz` as a `tarball` input) - `file+{something}://` urls will be interpreted as `file` urls (with the `file+` part removed) - `tarball+{something}://` urls will be interpreted as `tarball` urls (with the `tarball+` part removed) Fix #3785 Co-Authored-By: Tony Olagbaiye --- doc/manual/src/release-notes/rl-next.md | 3 + src/libfetchers/tarball.cc | 90 +++++++++++++++----- src/libutil/url.cc | 18 ++++ src/libutil/url.hh | 15 ++++ src/nix/flake.md | 14 +++- tests/fetchTree-file.sh | 105 ++++++++++++++++++++++++ tests/local.mk | 1 + 7 files changed, 221 insertions(+), 25 deletions(-) create mode 100644 tests/fetchTree-file.sh diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index efd893662..9f00893fd 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -24,3 +24,6 @@ Selecting derivation outputs using the attribute selection syntax (e.g. `nixpkgs#glibc.dev`) no longer works. + +* `builtins.fetchTree` (and flake inputs) can now be used to fetch plain files + over the `http(s)` and `file` protocols in addition to directory tarballs. diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index dde0ad761..09acb74d3 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "tarfile.hh" #include "types.hh" +#include "split.hh" namespace nix::fetchers { @@ -168,24 +169,34 @@ std::pair downloadTarball( }; } -struct TarballInputScheme : InputScheme +// An input scheme corresponding to a curable ressource +struct CurlInputScheme : InputScheme { + virtual const std::string inputType() const = 0; + const std::set transportUrlSchemes = {"file", "http", "https"}; + + const bool hasTarballExtension(std::string_view path) const + { + return hasSuffix(path, ".zip") || hasSuffix(path, ".tar") + || hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz") + || hasSuffix(path, ".tar.xz") || hasSuffix(path, ".tar.bz2") + || hasSuffix(path, ".tar.zst"); + } + + virtual bool isValidURL(const ParsedURL & url) const = 0; + std::optional inputFromURL(const ParsedURL & url) override { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; - - if (!hasSuffix(url.path, ".zip") - && !hasSuffix(url.path, ".tar") - && !hasSuffix(url.path, ".tgz") - && !hasSuffix(url.path, ".tar.gz") - && !hasSuffix(url.path, ".tar.xz") - && !hasSuffix(url.path, ".tar.bz2") - && !hasSuffix(url.path, ".tar.zst")) - return {}; + if (!isValidURL(url)) + return std::nullopt; Input input; - input.attrs.insert_or_assign("type", "tarball"); - input.attrs.insert_or_assign("url", url.to_string()); + + auto urlWithoutApplicationScheme = url; + urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport; + + input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string()); auto narHash = url.query.find("narHash"); if (narHash != url.query.end()) input.attrs.insert_or_assign("narHash", narHash->second); @@ -194,14 +205,17 @@ struct TarballInputScheme : InputScheme std::optional inputFromAttrs(const Attrs & attrs) override { - if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + auto type = maybeGetStrAttr(attrs, "type"); + if (type != inputType()) return {}; + std::set allowedNames = {"type", "url", "narHash", "name", "unpack"}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name") - throw Error("unsupported tarball input attribute '%s'", name); + if (!allowedNames.count(name)) + throw Error("unsupported %s input attribute '%s'", *type, name); Input input; input.attrs = attrs; + //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } @@ -209,14 +223,9 @@ struct TarballInputScheme : InputScheme ParsedURL toURL(const Input & input) override { auto url = parseURL(getStrAttr(input.attrs, "url")); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. + // NAR hashes are preferred over file hashes since tar/zip files // don't have a canonical representation. if (auto narHash = input.getNarHash()) url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); - /* - else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) - url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true)); - */ return url; } @@ -225,6 +234,42 @@ struct TarballInputScheme : InputScheme return true; } +}; + +struct FileInputScheme : CurlInputScheme +{ + const std::string inputType() const override { return "file"; } + + bool isValidURL(const ParsedURL & url) const override + { + auto parsedUrlScheme = parseUrlScheme(url.scheme); + return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) + && (parsedUrlScheme.application + ? parsedUrlScheme.application.value() == inputType() + : !hasTarballExtension(url.path)); + } + + std::pair fetch(ref store, const Input & input) override + { + auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); + return {std::move(file.storePath), input}; + } +}; + +struct TarballInputScheme : CurlInputScheme +{ + const std::string inputType() const override { return "tarball"; } + + bool isValidURL(const ParsedURL & url) const override + { + auto parsedUrlScheme = parseUrlScheme(url.scheme); + + return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) + && (parsedUrlScheme.application + ? parsedUrlScheme.application.value() == inputType() + : hasTarballExtension(url.path)); + } + std::pair fetch(ref store, const Input & input) override { auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; @@ -233,5 +278,6 @@ struct TarballInputScheme : InputScheme }; static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rFileInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index f6232d255..5b7abeb49 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -1,6 +1,7 @@ #include "url.hh" #include "url-parts.hh" #include "util.hh" +#include "split.hh" namespace nix { @@ -136,4 +137,21 @@ bool ParsedURL::operator ==(const ParsedURL & other) const && fragment == other.fragment; } +/** + * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme' + * into a tuple '(applicationScheme, transportScheme)' + * + * > parseUrlScheme("http") == ParsedUrlScheme{ {}, "http"} + * > parseUrlScheme("tarball+http") == ParsedUrlScheme{ {"tarball"}, "http"} + */ +ParsedUrlScheme parseUrlScheme(std::string_view scheme) +{ + auto application = splitPrefixTo(scheme, '+'); + auto transport = scheme; + return ParsedUrlScheme { + .application = application, + .transport = transport, + }; +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 6e77142e3..2a9fb34c1 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -27,4 +27,19 @@ std::map decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); +/* + * Although that’s not really standardized anywhere, an number of tools + * use a scheme of the form 'x+y' in urls, where y is the “transport layer” + * scheme, and x is the “application layer” scheme. + * + * For example git uses `git+https` to designate remotes using a Git + * protocol over http. + */ +struct ParsedUrlScheme { + std::optional application; + std::string_view transport; +}; + +ParsedUrlScheme parseUrlScheme(std::string_view scheme); + } diff --git a/src/nix/flake.md b/src/nix/flake.md index aa3f9f303..a1ab43281 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -181,9 +181,17 @@ Currently the `type` attribute can be one of the following: * `tarball`: Tarballs. The location of the tarball is specified by the attribute `url`. - In URL form, the schema must be `http://`, `https://` or `file://` - URLs and the extension must be `.zip`, `.tar`, `.tgz`, `.tar.gz`, - `.tar.xz`, `.tar.bz2` or `.tar.zst`. + In URL form, the schema must be `tarball+http://`, `tarball+https://` or `tarball+file://`. + If the extension corresponds to a known archive format (`.zip`, `.tar`, + `.tgz`, `.tar.gz`, `.tar.xz`, `.tar.bz2` or `.tar.zst`), then the `tarball+` + can be dropped. + +* `file`: Plain files or directory tarballs, either over http(s) or from the local + disk. + + In URL form, the schema must be `file+http://`, `file+https://` or `file+file://`. + If the extension doesn’t correspond to a known archive format (as defined by the + `tarball` fetcher), then the `file+` prefix can be dropped. * `github`: A more efficient way to fetch repositories from GitHub. The following attributes are required: diff --git a/tests/fetchTree-file.sh b/tests/fetchTree-file.sh new file mode 100644 index 000000000..1c0ce39ce --- /dev/null +++ b/tests/fetchTree-file.sh @@ -0,0 +1,105 @@ +source common.sh + +clearStore + +cd "$TEST_ROOT" + +test_fetch_file () { + echo foo > test_input + + input_hash="$(nix hash path test_input)" + + nix eval --impure --file - < inputs/test_input_file + tar cfa test_input.tar.gz inputs + cp test_input.tar.gz test_input_no_ext + input_tarball_hash="$(nix hash path test_input.tar.gz)" + input_directory_hash="$(nix hash path inputs)" + + cat < flake.nix + { + inputs.no_ext_default_no_unpack = { + url = "file://$PWD/test_input_no_ext"; + flake = false; + }; + inputs.no_ext_explicit_unpack = { + url = "tarball+file://$PWD/test_input_no_ext"; + flake = false; + }; + inputs.tarball_default_unpack = { + url = "file://$PWD/test_input.tar.gz"; + flake = false; + }; + inputs.tarball_explicit_no_unpack = { + url = "file+file://$PWD/test_input.tar.gz"; + flake = false; + }; + outputs = { ... }: {}; + } +EOF + + nix flake update + nix eval --file - < flake.nix + { + inputs.tarball = { + url = "file://$PWD/test_input.tar.gz"; + flake = false; + }; + outputs = { self, tarball }: { + foo = builtins.readFile "${tarball}/test_input_file"; + }; + } + nix flake update + + clearStore + "$NIX_DAEMON_PACKAGE/bin/nix" eval .#foo +EOF +} + +test_fetch_file +test_file_flake_input diff --git a/tests/local.mk b/tests/local.mk index e3c4ff4eb..2932d2b13 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -23,6 +23,7 @@ nix_tests = \ fetchGit.sh \ fetchurl.sh \ fetchPath.sh \ + fetchTree-file.sh \ simple.sh \ referrers.sh \ optimise-store.sh \ From cebef6a25031e984a30d823f19b4cdc414ee9b48 Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 19 May 2022 21:16:07 +0300 Subject: [PATCH 22/29] nix-daemon.service: sync LimitNOFILE with the nixos service https://github.com/NixOS/nixpkgs/blob/5628480acda2985cc334d0c5ec85512858ed60f9/nixos/modules/services/misc/nix-daemon.nix#L737 Closes https://github.com/NixOS/nix/issues/6007 --- misc/systemd/nix-daemon.service.in | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 24d894898..e3ac42beb 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -9,6 +9,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process +LimitNOFILE=4096 [Install] WantedBy=multi-user.target From 81a9bf0ad2ff3244096ed14299c65c0b32c0aca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Camille=20Favier?= Date: Sat, 21 May 2022 14:41:24 +0200 Subject: [PATCH 23/29] =?UTF-8?q?typo:=20defaultApps=20=E2=86=92=20default?= =?UTF-8?q?App?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/nix/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1938ce4e6..500116eaf 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -509,7 +509,7 @@ struct CmdFlakeCheck : FlakeCommand std::string_view replacement = name == "defaultPackage" ? "packages..default" : - name == "defaultApps" ? "apps..default" : + name == "defaultApp" ? "apps..default" : name == "defaultTemplate" ? "templates.default" : name == "defaultBundler" ? "bundlers..default" : name == "overlay" ? "overlays.default" : From d1c270431ab017c48c28801bb93091c091e9d49d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 May 2022 22:01:52 +0000 Subject: [PATCH 24/29] Bump zeebe-io/backport-action from 0.0.7 to 0.0.8 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 0.0.7 to 0.0.8. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v0.0.7...v0.0.8) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index dd481160f..3a2d4de0e 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -15,7 +15,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v0.0.7 + uses: zeebe-io/backport-action@v0.0.8 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From b916c08feba5173c3455890cff615fd46464409a Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 24 May 2022 14:20:48 +0200 Subject: [PATCH 25/29] libfetchers: drop `getGitDir` and hardcode `.git` As discussed[1] this is most likely not desirable. [1] https://github.com/NixOS/nix/pull/6440#issuecomment-1120876248 --- src/libfetchers/git.cc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d23a820a4..a71bff76f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -26,11 +26,6 @@ namespace { // old version of git, which will ignore unrecognized `-c` options. const std::string gitInitialBranch = "__nix_dummy_branch"; -std::string getGitDir() -{ - return getEnv("GIT_DIR").value_or(".git"); -} - bool isCacheFileWithinTtl(const time_t now, const struct stat & st) { return st.st_mtime + settings.tarballTtl > now; @@ -152,7 +147,7 @@ struct WorkdirInfo WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) { const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - auto gitDir = getGitDir(); + std::string gitDir(".git"); auto env = getEnv(); // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong @@ -370,7 +365,7 @@ struct GitInputScheme : InputScheme { auto sourcePath = getSourcePath(input); assert(sourcePath); - auto gitDir = getGitDir(); + auto gitDir = ".git"; runProgram("git", true, { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) }); @@ -396,7 +391,7 @@ struct GitInputScheme : InputScheme std::pair fetch(ref store, const Input & _input) override { Input input(_input); - auto gitDir = getGitDir(); + auto gitDir = ".git"; std::string name = input.getName(); From cbf60bec6ff900e6759b439b782c8cef163b3046 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 24 May 2022 16:26:40 +0200 Subject: [PATCH 26/29] configure.ac: check for sandbox-shell's FEATURE_SH_STANDALONE See also: https://bugs.archlinux.org/task/73998. Busybox's FEATURE_SH_STANDALONE feature causes other busybox applets to leak into the sandbox, where system() calls will start preferring them over tools in $PATH. On arch, this even includes `ar`. Let's check for this evil feature and disallow using this as a sandbox shell. --- configure.ac | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/configure.ac b/configure.ac index 8a01c33ec..715c70de1 100644 --- a/configure.ac +++ b/configure.ac @@ -294,6 +294,17 @@ esac AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) +if ! test -z ${sandbox_shell+x}; then + AC_MSG_CHECKING([whether sandbox-shell has the standalone feature]) + # busybox shell sometimes allows executing other busybox applets, + # even if they are not in the path, breaking our sandbox + if PATH= $sandbox_shell -c "busybox" 2>&1 | grep -qv "not found"; then + AC_MSG_RESULT(enabled) + AC_MSG_ERROR([Please disable busybox FEATURE_SH_STANDALONE]) + else + AC_MSG_RESULT(disabled) + fi +fi # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix From 7e52472759bfecbbfc9146fd0992361ea930f195 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 24 May 2022 17:00:27 +0200 Subject: [PATCH 27/29] configure.ac: don't run sandbox-shell test when cross compiling --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 715c70de1..789dfdb3c 100644 --- a/configure.ac +++ b/configure.ac @@ -294,7 +294,7 @@ esac AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) -if ! test -z ${sandbox_shell+x}; then +if test ${cross_compiling:-no} = no && ! test -z ${sandbox_shell+x}; then AC_MSG_CHECKING([whether sandbox-shell has the standalone feature]) # busybox shell sometimes allows executing other busybox applets, # even if they are not in the path, breaking our sandbox From 2f8a34cddcdd738afebde38e83b2315d3e305152 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 May 2022 15:05:39 +0200 Subject: [PATCH 28/29] Fix warning --- src/nix/profile.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 1aae347df..3814e7d5a 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -263,7 +263,8 @@ builtPathsPerInstallable( struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { - std::optional priority; + std::optional priority; + CmdProfileInstall() { addFlag({ .longName = "priority", From d8398d33c9a09e1f5599127ae6d477e7e0868b55 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 May 2022 15:29:27 +0200 Subject: [PATCH 29/29] Typo --- src/libfetchers/tarball.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 09acb74d3..6c551bd93 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -169,7 +169,7 @@ std::pair downloadTarball( }; } -// An input scheme corresponding to a curable ressource +// An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { virtual const std::string inputType() const = 0;