From f58604ac32d0ab0718a0e68b3c1b050e4ab121cb Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Fri, 25 Oct 2019 17:45:14 +0200 Subject: [PATCH 001/118] Add fetchSubmodules to builtins.fetchGit There are some downsides to this features: - Submodules are not cached (unlike the root repo), - Full checkouts are created in a temporary directory. --- src/libexpr/primops/fetchGit.cc | 45 +++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..b08e48404 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -22,13 +22,14 @@ struct GitInfo std::string rev; std::string shortRev; uint64_t revCount = 0; + bool submodules = false; }; std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, - const std::string & name) + const std::string & name, bool fetchSubmodules) { if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); @@ -145,7 +146,8 @@ GitInfo exportGit(ref store, const std::string & uri, printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); + std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev + + (fetchSubmodules ? "submodules" : "")).to_string(Base32, false); Path storeLink = cacheDir + "/" + storeLinkName + ".link"; PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken @@ -165,16 +167,35 @@ GitInfo exportGit(ref store, const std::string & uri, if (e.errNo != ENOENT) throw; } - auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); - }); - Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - unpackTarfile(*source, tmpDir); + // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root + // repo is cached. + if (fetchSubmodules) { + Path tmpGitDir = createTempDir(); + AutoDelete delTmpGitDir(tmpGitDir, true); + + runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--", cacheDir, fmt("%s:%s", *ref, *ref) }); + + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); + runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" }); + + deletePath(tmpDir + "/.git"); + + gitInfo.submodules = true; + } else { + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + unpackTarfile(*source, tmpDir); + } gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); @@ -198,6 +219,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va std::optional ref; std::string rev; std::string name = "source"; + bool fetchSubmodules = false; PathSet context; state.forceValue(*args[0]); @@ -216,6 +238,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va rev = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "fetchSubmodules") + fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); } @@ -230,13 +254,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); - auto gitInfo = exportGit(state.store, url, ref, rev, name); + auto gitInfo = exportGit(state.store, url, ref, rev, name, fetchSubmodules); state.mkAttrs(v, 8); mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), gitInfo.submodules); v.attrs->sort(); if (state.allowedPaths) From c2a24c2b8876a0862ede39f63e684cb875cc7aab Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sat, 26 Oct 2019 11:09:50 +0200 Subject: [PATCH 002/118] Add test for fetchGit submodule support --- tests/fetchGitSubmodules.sh | 50 +++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 2 files changed, 51 insertions(+) create mode 100644 tests/fetchGitSubmodules.sh diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh new file mode 100644 index 000000000..c821d2caa --- /dev/null +++ b/tests/fetchGitSubmodules.sh @@ -0,0 +1,50 @@ +source common.sh + +set -u + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git submodule tests" + exit 99 +fi + +clearStore + +rootRepo=$TEST_ROOT/gitSubmodulesRoot +subRepo=$TEST_ROOT/gitSubmodulesSub + +rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix/gitv2 + +initGitRepo() { + git init $1 + git -C $1 config user.email "foobar@example.com" + git -C $1 config user.name "Foobar" +} + +addGitContent() { + echo "lorem ipsum" > $1/content + git -C $1 add content + git -C $1 commit -m "Initial commit" +} + +initGitRepo $subRepo +addGitContent $subRepo + +initGitRepo $rootRepo + +git -C $rootRepo submodule init +git -C $rootRepo submodule add $subRepo sub +git -C $rootRepo add sub +git -C $rootRepo commit -m "Add submodule" + +rev=$(git -C $rootRepo rev-parse HEAD) + +pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") + +# The resulting store path cannot be the same. +[[ $pathWithoutSubmodules != $pathWithSubmodules ]] + +[[ ! -e $pathWithoutSubmodules/sub/content ]] +[[ -e $pathWithSubmodules/sub/content ]] + + diff --git a/tests/local.mk b/tests/local.mk index dab3a23b6..01fac4fcd 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ nar-access.sh \ structured-attrs.sh \ fetchGit.sh \ + fetchGitSubmodules.sh \ fetchMercurial.sh \ signing.sh \ run.sh \ From ea861be292d790db9835c08abee734baa77b9cf7 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sat, 26 Oct 2019 11:13:08 +0200 Subject: [PATCH 003/118] Add documentation for submodule support in fetchGit --- doc/manual/expressions/builtins.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 394e1fc32..838edae3c 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -422,6 +422,16 @@ stdenv.mkDerivation { … } + + fetchSubmodules + + + A boolean parameter that specifies whether submodules + should be checked out. Defaults to + false. + + + From c8d33de77781fe7850d04a7374c6c22a2e995b70 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:42:19 +0100 Subject: [PATCH 004/118] Add git submodule fixes from @bjornfor This fixes fetching repositories with no submodules and also cleans up .git files in checkouts. --- src/libexpr/local.mk | 1 + src/libexpr/primops/fetchGit.cc | 9 +++++++-- tests/fetchGitSubmodules.sh | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 8a9b3c2ea..9b5fcc561 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -9,6 +9,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust libexpr_LDFLAGS = + ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b08e48404..c58d28bbc 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -6,6 +6,7 @@ #include "hash.hh" #include "tarfile.hh" +#include #include #include @@ -182,9 +183,13 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); - deletePath(tmpDir + "/.git"); + for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) { + if (p.path().filename() == ".git") { + std::filesystem::remove_all(p.path()); + } + } gitInfo.submodules = true; } else { diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index c821d2caa..987bfd69e 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -47,4 +47,5 @@ pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo [[ ! -e $pathWithoutSubmodules/sub/content ]] [[ -e $pathWithSubmodules/sub/content ]] - +# No .git directory or submodule reference files must be left +test "$(find "$pathWithSubmodules" -name .git)" = "" From c846abb5cc900d5fcb5aeb48427a1eac646cc0f3 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:46:19 +0100 Subject: [PATCH 005/118] Add more test for git submodule functionality --- tests/fetchGitSubmodules.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 987bfd69e..c252d42f7 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -40,12 +40,23 @@ rev=$(git -C $rootRepo rev-parse HEAD) pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] + +# The submodules flag is actually honored. [[ ! -e $pathWithoutSubmodules/sub/content ]] [[ -e $pathWithSubmodules/sub/content ]] # No .git directory or submodule reference files must be left test "$(find "$pathWithSubmodules" -name .git)" = "" + +# Git repos without submodules can be fetched with submodules = true. +noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") + +[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] From 435366ed3c292756bb74fd12f47ff96e93cd48ac Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:50:13 +0100 Subject: [PATCH 006/118] Rename fetchGit fetchSubmodules to just submodules --- src/libexpr/primops/fetchGit.cc | 2 +- tests/fetchGitSubmodules.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index c58d28bbc..e8905b548 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -243,7 +243,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va rev = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "fetchSubmodules") + else if (n == "submodules") fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index c252d42f7..6e4ef270b 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -39,8 +39,8 @@ git -C $rootRepo commit -m "Add submodule" rev=$(git -C $rootRepo rev-parse HEAD) pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") -pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] @@ -57,6 +57,6 @@ test "$(find "$pathWithSubmodules" -name .git)" = "" # Git repos without submodules can be fetched with submodules = true. noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath") -noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] From 6c00a9545f2162496032c490aadbadd85856662c Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 15:50:39 +0100 Subject: [PATCH 007/118] Fix typo in submodule test --- tests/fetchGitSubmodules.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 6e4ef270b..9a21421cc 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -56,7 +56,8 @@ pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$roo test "$(find "$pathWithSubmodules" -name .git)" = "" # Git repos without submodules can be fetched with submodules = true. -noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath") -noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; submodules = true; }).outPath") +subRev=$(git -C $subRepo rev-parse HEAD) +noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] From cc4fe977e5eedf00d8e3d267ccbc3676b3fac1a0 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 9 Mar 2020 14:07:26 +0100 Subject: [PATCH 008/118] Link to stdc++fs Some platforms seem to still require linking with stdc++fs to enable STL std::filesystem support. --- src/libexpr/local.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 9b5fcc561..b6ee82424 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,7 +8,11 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust +ifeq ($(CXX), g++) +libexpr_LDFLAGS = -lstdc++fs +else libexpr_LDFLAGS = +endif ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl From 002a3a95dcb7391a8372ba7a74f4e7e9bd48b59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Mon, 16 Mar 2020 14:47:21 +0100 Subject: [PATCH 009/118] fetchGit: fix "fatal: couldn't find remote ref refs/heads/master" issue with submodules --- src/libexpr/primops/fetchGit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index e8905b548..66259d778 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -179,7 +179,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--", cacheDir, fmt("%s:%s", *ref, *ref) }); + "--", cacheDir, fmt("%s", *ref) }); runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); From 587e259bfdda248fb2d81389faca816b8efdb5f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Mon, 16 Mar 2020 14:47:55 +0100 Subject: [PATCH 010/118] tests/fetchGitSubmodules: add more tests --- tests/fetchGitSubmodules.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 9a21421cc..8ee1bb067 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -42,6 +42,25 @@ pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootR pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") +r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r2 ]] +[[ $r2 != $r3 ]] + +r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; }).outPath") +r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = false; }).outPath") +r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") +r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") +r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r4 ]] +[[ $r4 == $r5 ]] +[[ $r3 == $r6 ]] +[[ $r6 == $r7 ]] +[[ $r7 == $r8 ]] + # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] From 6864ad7cf510d96f0cfc0cdbeffc1188a4eab44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 14:27:14 +0100 Subject: [PATCH 011/118] fetchGit: fix submodule output attribute Before this change it would be false for all evaluations but the first. Now it follows the input argument (as it should). --- src/libexpr/primops/fetchGit.cc | 7 +++---- tests/fetchGitSubmodules.sh | 9 +++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 66259d778..653d99b53 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -32,6 +32,9 @@ GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name, bool fetchSubmodules) { + GitInfo gitInfo; + gitInfo.submodules = fetchSubmodules; + if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); @@ -49,7 +52,6 @@ GitInfo exportGit(ref store, const std::string & uri, if (!clean) { /* This is an unclean working tree. So copy all tracked files. */ - GitInfo gitInfo; gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); @@ -141,7 +143,6 @@ GitInfo exportGit(ref store, const std::string & uri, } // FIXME: check whether rev is an ancestor of ref. - GitInfo gitInfo; gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); @@ -190,8 +191,6 @@ GitInfo exportGit(ref store, const std::string & uri, std::filesystem::remove_all(p.path()); } } - - gitInfo.submodules = true; } else { auto source = sinkToSource([&](Sink & sink) { RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 8ee1bb067..2d625c376 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -61,6 +61,15 @@ r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submo [[ $r6 == $r7 ]] [[ $r7 == $r8 ]] +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") +[[ $have_submodules == true ]] + # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] From 369fffd6f1f7d939e528b3591dca49989c51ae35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 14:27:58 +0100 Subject: [PATCH 012/118] fetchGit: add submodules attribute to the .link file The .link file is used as a lock, so I think we should put the "submodule" attribute in there since turning on submodules creates a new .link file path. --- src/libexpr/primops/fetchGit.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 653d99b53..969b1a4ed 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -211,6 +211,7 @@ GitInfo exportGit(ref store, const std::string & uri, json["name"] = name; json["rev"] = gitInfo.rev; json["revCount"] = gitInfo.revCount; + json["submodules"] = gitInfo.submodules; writeFile(storeLink, json.dump()); From be84049baf047fd2eee08def5c8013797c0832a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 20:14:19 +0100 Subject: [PATCH 013/118] tests/fetchGitSubmodules.sh: more checks --- tests/fetchGitSubmodules.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 2d625c376..30a8737bd 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -38,10 +38,6 @@ git -C $rootRepo commit -m "Add submodule" rev=$(git -C $rootRepo rev-parse HEAD) -pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") - r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") @@ -70,16 +66,26 @@ have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\" have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") [[ $have_submodules == true ]] +pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") + # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] # Checking out the same repo with submodules returns in the same store path. [[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]] + # The submodules flag is actually honored. [[ ! -e $pathWithoutSubmodules/sub/content ]] [[ -e $pathWithSubmodules/sub/content ]] +[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]] + # No .git directory or submodule reference files must be left test "$(find "$pathWithSubmodules" -name .git)" = "" From b306b7039ef3f2b76dbcc8020dcf67ed27442e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 18 Mar 2020 08:44:37 +0100 Subject: [PATCH 014/118] fetchGit: checkout rev instead of latest ref Major bugfix for the submodules = true code path. TODO: Add tests. --- src/libexpr/primops/fetchGit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 969b1a4ed..0c6bc6dc0 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -182,7 +182,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--", cacheDir, fmt("%s", *ref) }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); From cc522d0d23b71faeac95998d264abe14ddceeed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 18 Mar 2020 08:45:31 +0100 Subject: [PATCH 015/118] fetchGit: fix submodules = true for dirty trees --- src/libexpr/primops/fetchGit.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 0c6bc6dc0..66e1d7b98 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -55,8 +55,12 @@ GitInfo exportGit(ref store, const std::string & uri, gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + auto gitOpts = Strings({ "-C", uri, "ls-files", "-z" }); + if (fetchSubmodules) { + gitOpts.emplace_back("--recurse-submodules"); + } auto files = tokenizeString>( - runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); + runProgram("git", true, gitOpts), "\0"s); PathFilter filter = [&](const Path & p) -> bool { assert(hasPrefix(p, uri)); From f686efeed4d6305101c56136855e2d6b87e649e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Fri, 20 Mar 2020 14:36:10 +0100 Subject: [PATCH 016/118] fetchGit: fix submodule corner case by fetching all refs from cacheDir Due to fetchGit not checking if rev is an ancestor of ref (there is even a FIXME comment about it in the code), the cache repo might not have the ref even though it has the rev. This doesn't matter when submodule = false, but the submodule = true code blows up because it tries to fetch the (missing) ref from the cache repo. Fix this in the simplest way possible: fetch all refs from the local cache repo when submodules = true. TODO: Add tests. --- src/libexpr/primops/fetchGit.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 66e1d7b98..f138b755c 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -183,8 +183,11 @@ GitInfo exportGit(ref store, const std::string & uri, AutoDelete delTmpGitDir(tmpGitDir, true); runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + // TODO: the cacheDir repo might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch everything to + // ensure we get the rev. runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--", cacheDir, fmt("%s", *ref) }); + "--update-head-ok", "--", cacheDir, "refs/*:refs/*" }); runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); From 40c023ecfe49fea6e66db34c5f841fcf7001cbeb Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sun, 29 Mar 2020 23:47:48 +0200 Subject: [PATCH 017/118] fetchGit: don't use std::filesystem to filter git repos Using std::filesystem means also having to link with -lstdc++fs on some platforms and it's hard to discover for what platforms this is needed. As all the functionality is already implemented as utilities, use those instead. --- src/libexpr/local.mk | 4 ---- src/libexpr/primops/fetchGit.cc | 17 ++++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index b6ee82424..9b5fcc561 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,11 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust -ifeq ($(CXX), g++) -libexpr_LDFLAGS = -lstdc++fs -else libexpr_LDFLAGS = -endif ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index f138b755c..d60e6b803 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -6,7 +6,6 @@ #include "hash.hh" #include "tarfile.hh" -#include #include #include @@ -28,6 +27,13 @@ struct GitInfo std::regex revRegex("^[0-9a-fA-F]{40}$"); +static bool isNotDotGitDirectory(const Path & path) +{ + static const std::regex gitDirRegex("^(?:.*/)?\\.git$"); + + return not std::regex_match(path, gitDirRegex); +} + GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name, bool fetchSubmodules) @@ -175,6 +181,7 @@ GitInfo exportGit(ref store, const std::string & uri, Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); + PathFilter filter = defaultPathFilter; // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root // repo is cached. @@ -193,11 +200,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); - for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) { - if (p.path().filename() == ".git") { - std::filesystem::remove_all(p.path()); - } - } + filter = isNotDotGitDirectory; } else { auto source = sinkToSource([&](Sink & sink) { RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); @@ -208,7 +211,7 @@ GitInfo exportGit(ref store, const std::string & uri, unpackTarfile(*source, tmpDir); } - gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); + gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir, true, htSHA256, filter)); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); From c34e96f7e0d53894f302134b2f90f83b20ffd22a Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 2 Apr 2020 05:03:58 +0200 Subject: [PATCH 018/118] Make function arguments retain position info This allows querying the location of function arguments. E.g. builtins.unsafeGetAttrPos "x" (builtins.functionArgs ({ x }: null)) => { column = 57; file = "/home/infinisil/src/nix/inst/test.nix"; line = 1; } --- src/libexpr/nixexpr.hh | 3 ++- src/libexpr/parser.y | 4 ++-- src/libexpr/primops.cc | 7 +++++-- tests/lang/eval-okay-getattrpos-functionargs.exp | 1 + tests/lang/eval-okay-getattrpos-functionargs.nix | 4 ++++ 5 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 tests/lang/eval-okay-getattrpos-functionargs.exp create mode 100644 tests/lang/eval-okay-getattrpos-functionargs.nix diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index f7e9105a4..8c96de37c 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -209,9 +209,10 @@ struct ExprList : Expr struct Formal { + Pos pos; Symbol name; Expr * def; - Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; + Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { }; }; struct Formals diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9c769e803..08cec96dc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -531,8 +531,8 @@ formals ; formal - : ID { $$ = new Formal(data->symbols.create($1), 0); } - | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); } + : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); } + | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); } ; %% diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..81c378dff 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1354,9 +1354,12 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args } state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); - for (auto & i : args[0]->lambda.fun->formals->formals) + for (auto & i : args[0]->lambda.fun->formals->formals) { // !!! should optimise booleans (allocate only once) - mkBool(*state.allocAttr(v, i.name), i.def); + Value * value = state.allocValue(); + v.attrs->push_back(Attr(i.name, value, &i.pos)); + mkBool(*value, i.def); + } v.attrs->sort(); } diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/lang/eval-okay-getattrpos-functionargs.exp new file mode 100644 index 000000000..7f9ac40e8 --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-functionargs.exp @@ -0,0 +1 @@ +{ column = 11; file = "eval-okay-getattrpos-functionargs.nix"; line = 2; } diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/lang/eval-okay-getattrpos-functionargs.nix new file mode 100644 index 000000000..11d6bb0e3 --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-functionargs.nix @@ -0,0 +1,4 @@ +let + fun = { foo }: {}; + pos = builtins.unsafeGetAttrPos "foo" (builtins.functionArgs fun); +in { inherit (pos) column line; file = baseNameOf pos.file; } From 462421d34588c227eb736b07f7b40a38aa0972a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 16:04:18 +0200 Subject: [PATCH 019/118] Backport libfetchers from the flakes branch This provides a pluggable mechanism for defining new fetchers. It adds a builtin function 'fetchTree' that generalizes existing fetchers like 'fetchGit', 'fetchMercurial' and 'fetchTarball'. 'fetchTree' takes a set of attributes, e.g. fetchTree { type = "git"; url = "https://example.org/repo.git"; ref = "some-branch"; rev = "abcdef..."; } The existing fetchers are just wrappers around this. Note that the input attributes to fetchTree are the same as flake input specifications and flake lock file entries. All fetchers share a common cache stored in ~/.cache/nix/fetcher-cache-v1.sqlite. This replaces the ad hoc caching mechanisms in fetchGit and download.cc (e.g. ~/.cache/nix/{tarballs,git-revs*}). This also adds support for Git worktrees (c169ea59049f861aaba429f48b828d0820b74d1d). --- Makefile | 1 + src/libexpr/common-eval-args.cc | 8 +- src/libexpr/local.mk | 4 +- src/libexpr/parser.y | 6 +- src/libexpr/primops.cc | 67 ----- src/libexpr/primops.hh | 1 + src/libexpr/primops/fetchGit.cc | 223 ++------------ src/libexpr/primops/fetchMercurial.cc | 207 +++---------- src/libexpr/primops/fetchTree.cc | 165 +++++++++++ src/libfetchers/attrs.cc | 92 ++++++ src/libfetchers/attrs.hh | 37 +++ src/libfetchers/cache.cc | 121 ++++++++ src/libfetchers/cache.hh | 34 +++ src/libfetchers/fetchers.cc | 73 +++++ src/libfetchers/fetchers.hh | 103 +++++++ src/libfetchers/git.cc | 401 ++++++++++++++++++++++++++ src/libfetchers/github.cc | 195 +++++++++++++ src/libfetchers/local.mk | 11 + src/libfetchers/mercurial.cc | 303 +++++++++++++++++++ src/libfetchers/tarball.cc | 277 ++++++++++++++++++ src/libfetchers/tree-info.cc | 14 + src/libfetchers/tree-info.hh | 29 ++ src/libstore/derivations.cc | 2 +- src/libstore/download.cc | 137 --------- src/libstore/download.hh | 31 +- src/libstore/globals.hh | 9 + src/libstore/store-api.cc | 25 +- src/libstore/store-api.hh | 1 + src/libutil/types.hh | 8 + src/libutil/url.cc | 137 +++++++++ src/libutil/url.hh | 62 ++++ src/nix-channel/nix-channel.cc | 17 +- src/nix/local.mk | 4 +- tests/fetchGit.sh | 32 +- tests/init.sh | 2 +- tests/tarball.sh | 7 + 36 files changed, 2199 insertions(+), 647 deletions(-) create mode 100644 src/libexpr/primops/fetchTree.cc create mode 100644 src/libfetchers/attrs.cc create mode 100644 src/libfetchers/attrs.hh create mode 100644 src/libfetchers/cache.cc create mode 100644 src/libfetchers/cache.hh create mode 100644 src/libfetchers/fetchers.cc create mode 100644 src/libfetchers/fetchers.hh create mode 100644 src/libfetchers/git.cc create mode 100644 src/libfetchers/github.cc create mode 100644 src/libfetchers/local.mk create mode 100644 src/libfetchers/mercurial.cc create mode 100644 src/libfetchers/tarball.cc create mode 100644 src/libfetchers/tree-info.cc create mode 100644 src/libfetchers/tree-info.hh create mode 100644 src/libutil/url.cc create mode 100644 src/libutil/url.hh diff --git a/Makefile b/Makefile index 469070533..e3057c36c 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ makefiles = \ nix-rust/local.mk \ src/libutil/local.mk \ src/libstore/local.mk \ + src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ src/nix/local.mk \ diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 13950ab8d..82bfeac36 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -3,6 +3,8 @@ #include "download.hh" #include "util.hh" #include "eval.hh" +#include "fetchers.hh" +#include "store-api.hh" namespace nix { @@ -46,9 +48,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) { - CachedDownloadRequest request(s); - request.unpack = true; - return getDownloader()->downloadCached(state.store, request).path; + return state.store->toRealPath( + fetchers::downloadTarball( + state.store, resolveUri(s), "source", false).storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index a4ccab376..917e8a1c7 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,9 +6,9 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc -libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr +libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr -libexpr_LIBS = libutil libstore libnixrust +libexpr_LIBS = libutil libstore libfetchers libnixrust libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9c769e803..a30fb44b5 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -545,6 +545,7 @@ formal #include "eval.hh" #include "download.hh" +#include "fetchers.hh" #include "store-api.hh" @@ -687,9 +688,8 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - CachedDownloadRequest request(elem.second); - request.unpack = true; - res = { true, getDownloader()->downloadCached(store, request).path }; + res = { true, store->toRealPath(fetchers::downloadTarball( + store, resolveUri(elem.second), "source", false).storePath) }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..629f3da15 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,6 +1,5 @@ #include "archive.hh" #include "derivations.hh" -#include "download.hh" #include "eval-inline.hh" #include "eval.hh" #include "globals.hh" @@ -2045,68 +2044,6 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args } -/************************************************************* - * Networking - *************************************************************/ - - -void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, - const string & who, bool unpack, const std::string & defaultName) -{ - CachedDownloadRequest request(""); - request.unpack = unpack; - request.name = defaultName; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "sha256") - request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); - else if (n == "name") - request.name = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); - } - - if (request.uri.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); - - } else - request.uri = state.forceStringNoCtx(*args[0], pos); - - state.checkURI(request.uri); - - if (evalSettings.pureEval && !request.expectedHash) - throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - - auto res = getDownloader()->downloadCached(state.store, request); - - if (state.allowedPaths) - state.allowedPaths->insert(res.path); - - mkString(v, res.storePath, PathSet({res.storePath})); -} - - -static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchurl", false, ""); -} - - -static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchTarball", true, "source"); -} - - /************************************************************* * Primop registration *************************************************************/ @@ -2289,10 +2226,6 @@ void EvalState::createBaseEnv() addPrimOp("derivationStrict", 1, prim_derivationStrict); addPrimOp("placeholder", 1, prim_placeholder); - // Networking - addPrimOp("__fetchurl", 1, prim_fetchurl); - addPrimOp("fetchTarball", 1, prim_fetchTarball); - /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index c790b30f6..05d0792ef 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -20,6 +20,7 @@ struct RegisterPrimOp them. */ /* Load a ValueInitializer from a DSO and return whatever it initializes */ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); + /* Execute a program and parse its output */ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..812de9d91 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -1,202 +1,17 @@ #include "primops.hh" #include "eval-inline.hh" -#include "download.hh" #include "store-api.hh" -#include "pathlocks.hh" #include "hash.hh" -#include "tarfile.hh" - -#include - -#include - -#include - -using namespace std::string_literals; +#include "fetchers.hh" +#include "url.hh" namespace nix { -struct GitInfo -{ - Path storePath; - std::string rev; - std::string shortRev; - uint64_t revCount = 0; -}; - -std::regex revRegex("^[0-9a-fA-F]{40}$"); - -GitInfo exportGit(ref store, const std::string & uri, - std::optional ref, std::string rev, - const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { - - bool clean = true; - - try { - runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - clean = false; - } - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked files. */ - GitInfo gitInfo; - gitInfo.rev = "0000000000000000000000000000000000000000"; - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - - auto files = tokenizeString>( - runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); - - return gitInfo; - } - - // clean working tree, but no ref or rev specified. Use 'HEAD'. - rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); - ref = "HEAD"s; - } - - if (!ref) ref = "HEAD"s; - - if (rev != "" && !std::regex_match(rev, revRegex)) - throw Error("invalid Git revision '%s'", rev); - - deletePath(getCacheDir() + "/nix/git"); - - Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); - - if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", cacheDir }); - } - - Path localRefFile; - if (ref->compare(0, 5, "refs/") == 0) - localRefFile = cacheDir + "/" + *ref; - else - localRefFile = cacheDir + "/refs/heads/" + *ref; - - bool doFetch; - time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (rev != "") { - try { - runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } - } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; - } - if (doFetch) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); - - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); - - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; - - utimes(localRefFile.c_str(), times); - } - - // FIXME: check whether rev is an ancestor of ref. - GitInfo gitInfo; - gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - - printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); - Path storeLink = cacheDir + "/" + storeLinkName + ".link"; - PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == gitInfo.rev); - - gitInfo.storePath = json["storePath"]; - - if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) { - gitInfo.revCount = json["revCount"]; - return gitInfo; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); - }); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - unpackTarfile(*source, tmpDir); - - gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); - - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); - - nlohmann::json json; - json["storePath"] = gitInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["rev"] = gitInfo.rev; - json["revCount"] = gitInfo.revCount; - - writeFile(storeLink, json.dump()); - - return gitInfo; -} - static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; std::optional ref; - std::string rev; + std::optional rev; std::string name = "source"; PathSet context; @@ -213,7 +28,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va else if (n == "ref") ref = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else @@ -230,17 +45,35 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); - auto gitInfo = exportGit(state.store, url, ref, rev, name); + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + + auto parsedUrl = parseURL( + url.find("://") != std::string::npos + ? "git+" + url + : "git+file://" + url); + if (ref) parsedUrl.query.insert_or_assign("ref", *ref); + if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); + // FIXME: use name + auto input = fetchers::inputFromURL(parsedUrl); + + auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); + // Backward compatibility: set 'revCount' to 0 for a dirty tree. + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), + tree.info.revCount.value_or(0)); v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); + state.allowedPaths->insert(tree.actualPath); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index db274fa4f..f18351646 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,174 +1,18 @@ #include "primops.hh" #include "eval-inline.hh" -#include "download.hh" #include "store-api.hh" -#include "pathlocks.hh" - -#include +#include "fetchers.hh" +#include "url.hh" #include -#include - -using namespace std::string_literals; - namespace nix { -struct HgInfo -{ - Path storePath; - std::string branch; - std::string rev; - uint64_t revCount = 0; -}; - -std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); - -HgInfo exportMercurial(ref store, const std::string & uri, - std::string rev, const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - - if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { - - bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked - files. */ - - printTalkative("copying unclean Mercurial working tree '%s'", uri); - - HgInfo hgInfo; - hgInfo.rev = "0000000000000000000000000000000000000000"; - hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri })); - - auto files = tokenizeString>( - runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); - - return hgInfo; - } - } - - if (rev == "") rev = "default"; - - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false)); - - Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false)); - - /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, - do so now. */ - time_t now = time(0); - struct stat st; - if (stat(stampFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) - { - /* Except that if this is a commit hash that we already have, - we don't have to pull again. */ - if (!(std::regex_match(rev, commitHashRegex) - && pathExists(cacheDir) - && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" }) - .killStderr(true)).second == "1")) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); - - if (pathExists(cacheDir)) { - try { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - catch (ExecError & e) { - string transJournal = cacheDir + "/.hg/store/journal"; - /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); - } - } - } else { - createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); - } - } - - writeFile(stampFile, ""); - } - - auto tokens = tokenizeString>( - runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" })); - assert(tokens.size() == 3); - - HgInfo hgInfo; - hgInfo.rev = tokens[0]; - hgInfo.revCount = std::stoull(tokens[1]); - hgInfo.branch = tokens[2]; - - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false); - Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == hgInfo.rev); - - hgInfo.storePath = json["storePath"]; - - if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) { - printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); - return hgInfo; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir }); - - deletePath(tmpDir + "/.hg_archival.txt"); - - hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); - - nlohmann::json json; - json["storePath"] = hgInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["branch"] = hgInfo.branch; - json["rev"] = hgInfo.rev; - json["revCount"] = hgInfo.revCount; - - writeFile(storeLink, json.dump()); - - return hgInfo; -} - static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::string rev; + std::optional rev; + std::optional ref; std::string name = "source"; PathSet context; @@ -182,8 +26,15 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar string n(attr.name); if (n == "url") url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "rev") { + // Ugly: unlike fetchGit, here the "rev" attribute can + // be both a revision or a branch/tag name. + auto value = state.forceStringNoCtx(*attr.value, *attr.pos); + if (std::regex_match(value, revRegex)) + rev = Hash(value, htSHA1); + else + ref = value; + } else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else @@ -200,18 +51,36 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // whitelist. Ah well. state.checkURI(url); - auto hgInfo = exportMercurial(state.store, url, rev, name); + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); + + auto parsedUrl = parseURL( + url.find("://") != std::string::npos + ? "hg+" + url + : "hg+file://" + url); + if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); + if (ref) parsedUrl.query.insert_or_assign("ref", *ref); + // FIXME: use name + auto input = fetchers::inputFromURL(parsedUrl); + + auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); - mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + if (input2->getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); + state.allowedPaths->insert(tree.actualPath); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc new file mode 100644 index 000000000..9586f71ed --- /dev/null +++ b/src/libexpr/primops/fetchTree.cc @@ -0,0 +1,165 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "download.hh" + +#include +#include + +namespace nix { + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + std::shared_ptr input, + Value & v) +{ + state.mkAttrs(v, 8); + + auto storePath = state.store->printStorePath(tree.storePath); + + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + + assert(tree.info.narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + tree.info.narHash.to_string(SRI)); + + if (input->getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + } + + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + + if (tree.info.lastModified) + mkString(*state.allocAttr(v, state.symbols.create("lastModified")), + fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + + v.attrs->sort(); +} + +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + + std::shared_ptr input; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + fetchers::Attrs attrs; + + for (auto & attr : *args[0]->attrs) { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else if (attr.value->type == tBool) + attrs.emplace(attr.name, attr.value->boolean); + else + throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + attr.name, showType(*attr.value)); + } + + if (!attrs.count("type")) + throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos); + + input = fetchers::inputFromAttrs(attrs); + } else + input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + + if (evalSettings.pureEval && !input->isImmutable()) + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + + // FIXME: use fetchOrSubstituteTree + auto [tree, input2] = input->fetchTree(state.store); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + emitTreeAttrs(state, tree, input2, v); +} + +static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); + +static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, + const string & who, bool unpack, std::string name) +{ + std::optional url; + std::optional expectedHash; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to '%s', at %s", + attr.name, who, attr.pos); + } + + if (!url) + throw EvalError("'url' argument required, at %s", pos); + + } else + url = state.forceStringNoCtx(*args[0], pos); + + url = resolveUri(*url); + + state.checkURI(*url); + + if (name == "") + name = baseNameOf(*url); + + if (evalSettings.pureEval && !expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + + auto storePath = + unpack + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + + auto path = state.store->toRealPath(storePath); + + if (expectedHash) { + auto hash = unpack + ? state.store->queryPathInfo(storePath)->narHash + : hashFile(htSHA256, path); + if (hash != *expectedHash) + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + *url, expectedHash->to_string(), hash.to_string()); + } + + if (state.allowedPaths) + state.allowedPaths->insert(path); + + mkString(v, path, PathSet({path})); +} + +static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchurl", false, ""); +} + +static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + +static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); +static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); + +} diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc new file mode 100644 index 000000000..40c02de42 --- /dev/null +++ b/src/libfetchers/attrs.cc @@ -0,0 +1,92 @@ +#include "attrs.hh" +#include "fetchers.hh" + +#include + +namespace nix::fetchers { + +Attrs jsonToAttrs(const nlohmann::json & json) +{ + Attrs attrs; + + for (auto & i : json.items()) { + if (i.value().is_number()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_string()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_boolean()) + attrs.emplace(i.key(), i.value().get()); + else + throw Error("unsupported input attribute type in lock file"); + } + + return attrs; +} + +nlohmann::json attrsToJson(const Attrs & attrs) +{ + nlohmann::json json; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if>(&attr.second)) { + json[attr.first] = v->t; + } else abort(); + } + return json; +} + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump()); +} + +std::string getStrAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetStrAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not an integer", name); +} + +int64_t getIntAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetIntAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a Boolean", name); +} + +bool getBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetBoolAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +} diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh new file mode 100644 index 000000000..2c9e772d2 --- /dev/null +++ b/src/libfetchers/attrs.hh @@ -0,0 +1,37 @@ +#pragma once + +#include "types.hh" + +#include + +#include + +namespace nix::fetchers { + +/* Wrap bools to prevent string literals (i.e. 'char *') from being + cast to a bool in Attr. */ +template +struct Explicit { + T t; +}; + +typedef std::variant> Attr; +typedef std::map Attrs; + +Attrs jsonToAttrs(const nlohmann::json & json); + +nlohmann::json attrsToJson(const Attrs & attrs); + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name); + +std::string getStrAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); + +int64_t getIntAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name); + +bool getBoolAttr(const Attrs & attrs, const std::string & name); + +} diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc new file mode 100644 index 000000000..e1c7f3dee --- /dev/null +++ b/src/libfetchers/cache.cc @@ -0,0 +1,121 @@ +#include "cache.hh" +#include "sqlite.hh" +#include "sync.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +static const char * schema = R"sql( + +create table if not exists Cache ( + input text not null, + info text not null, + path text not null, + immutable integer not null, + timestamp integer not null, + primary key (input) +); +)sql"; + +struct CacheImpl : Cache +{ + struct State + { + SQLite db; + SQLiteStmt add, lookup; + }; + + Sync _state; + + CacheImpl() + { + auto state(_state.lock()); + + auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->add.create(state->db, + "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)"); + + state->lookup.create(state->db, + "select info, path, immutable, timestamp from Cache where input = ?"); + } + + void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) override + { + _state.lock()->add.use() + (attrsToJson(inAttrs).dump()) + (attrsToJson(infoAttrs).dump()) + (store->printStorePath(storePath)) + (immutable) + (time(0)).exec(); + } + + std::optional> lookup( + ref store, + const Attrs & inAttrs) override + { + if (auto res = lookupExpired(store, inAttrs)) { + if (!res->expired) + return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); + debug("ignoring expired cache entry '%s'", + attrsToJson(inAttrs).dump()); + } + return {}; + } + + std::optional lookupExpired( + ref store, + const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJson = attrsToJson(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJson)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJson); + return {}; + } + + auto infoJson = stmt.getStr(0); + auto storePath = store->parseStorePath(stmt.getStr(1)); + auto immutable = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + store->addTempRoot(storePath); + if (!store->isValidPath(storePath)) { + // FIXME: we could try to substitute 'storePath'. + debug("ignoring disappeared cache entry '%s'", inAttrsJson); + return {}; + } + + debug("using cache entry '%s' -> '%s', '%s'", + inAttrsJson, infoJson, store->printStorePath(storePath)); + + return Result { + .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)), + .storePath = std::move(storePath) + }; + } +}; + +ref getCache() +{ + static auto cache = std::make_shared(); + return ref(cache); +} + +} diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh new file mode 100644 index 000000000..d76ab1233 --- /dev/null +++ b/src/libfetchers/cache.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "fetchers.hh" + +namespace nix::fetchers { + +struct Cache +{ + virtual void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) = 0; + + virtual std::optional> lookup( + ref store, + const Attrs & inAttrs) = 0; + + struct Result + { + bool expired = false; + Attrs infoAttrs; + StorePath storePath; + }; + + virtual std::optional lookupExpired( + ref store, + const Attrs & inAttrs) = 0; +}; + +ref getCache(); + +} diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc new file mode 100644 index 000000000..25d3da431 --- /dev/null +++ b/src/libfetchers/fetchers.cc @@ -0,0 +1,73 @@ +#include "fetchers.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::unique_ptr>> inputSchemes = nullptr; + +void registerInputScheme(std::unique_ptr && inputScheme) +{ + if (!inputSchemes) inputSchemes = std::make_unique>>(); + inputSchemes->push_back(std::move(inputScheme)); +} + +std::unique_ptr inputFromURL(const ParsedURL & url) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromURL(url); + if (res) return res; + } + throw Error("input '%s' is unsupported", url.url); +} + +std::unique_ptr inputFromURL(const std::string & url) +{ + return inputFromURL(parseURL(url)); +} + +std::unique_ptr inputFromAttrs(const Attrs & attrs) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromAttrs(attrs); + if (res) { + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + res->narHash = Hash(*narHash); + return res; + } + } + throw Error("input '%s' is unsupported", attrsToJson(attrs)); +} + +Attrs Input::toAttrs() const +{ + auto attrs = toAttrsInternal(); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); + attrs.emplace("type", type()); + return attrs; +} + +std::pair> Input::fetchTree(ref store) const +{ + auto [tree, input] = fetchTreeInternal(store); + + if (tree.actualPath == "") + tree.actualPath = store->toRealPath(tree.storePath); + + if (!tree.info.narHash) + tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + + if (input->narHash) + assert(input->narHash == tree.info.narHash); + + if (narHash && narHash != input->narHash) + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + + return {std::move(tree), input}; +} + +} diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh new file mode 100644 index 000000000..59a58ae67 --- /dev/null +++ b/src/libfetchers/fetchers.hh @@ -0,0 +1,103 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "path.hh" +#include "tree-info.hh" +#include "attrs.hh" +#include "url.hh" + +#include + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Input; + +struct Tree +{ + Path actualPath; + StorePath storePath; + TreeInfo info; +}; + +struct Input : std::enable_shared_from_this +{ + std::optional narHash; // FIXME: implement + + virtual std::string type() const = 0; + + virtual ~Input() { } + + virtual bool operator ==(const Input & other) const { return false; } + + /* Check whether this is a "direct" input, that is, not + one that goes through a registry. */ + virtual bool isDirect() const { return true; } + + /* Check whether this is an "immutable" input, that is, + one that contains a commit hash or content hash. */ + virtual bool isImmutable() const { return (bool) narHash; } + + virtual bool contains(const Input & other) const { return false; } + + virtual std::optional getRef() const { return {}; } + + virtual std::optional getRev() const { return {}; } + + virtual ParsedURL toURL() const = 0; + + std::string to_string() const + { + return toURL().to_string(); + } + + Attrs toAttrs() const; + + std::pair> fetchTree(ref store) const; + +private: + + virtual std::pair> fetchTreeInternal(ref store) const = 0; + + virtual Attrs toAttrsInternal() const = 0; +}; + +struct InputScheme +{ + virtual ~InputScheme() { } + + virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; + + virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; +}; + +std::unique_ptr inputFromURL(const ParsedURL & url); + +std::unique_ptr inputFromURL(const std::string & url); + +std::unique_ptr inputFromAttrs(const Attrs & attrs); + +void registerInputScheme(std::unique_ptr && fetcher); + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc new file mode 100644 index 000000000..cee0713f4 --- /dev/null +++ b/src/libfetchers/git.cc @@ -0,0 +1,401 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +static std::string readHead(const Path & path) +{ + return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); +} + +struct GitInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + bool shallow = false; + + GitInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "git"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + if (shallow) url2.query.insert_or_assign("shallow", "1"); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (shallow) + attrs.emplace("shallow", true); + return attrs; + } + + std::pair getActualUrl() const + { + // Don't clone file:// URIs (but otherwise treat them the + // same as remote URIs, i.e. don't use the working tree or + // HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + bool isLocal = url.scheme == "file" && !forceHttp; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + assert(!rev || rev->type == htSHA1); + + auto cacheType = shallow ? "git-shallow" : "git"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", cacheType}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree { + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), + .lastModified = getIntAttr(infoAttrs, "lastModified"), + }, + }, + input + }; + }; + + if (rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input->ref && !input->rev && isLocal) { + bool clean = false; + + /* Check whether this repo has any commits. There are + probably better ways to do this. */ + auto gitDir = actualUrl + "/.git"; + auto commonGitDir = chomp(runProgram( + "git", + true, + { "-C", actualUrl, "rev-parse", "--git-common-dir" } + )); + if (commonGitDir != ".git") + gitDir = commonGitDir; + + bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty(); + + try { + if (haveCommits) { + runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked files. */ + + if (!settings.allowDirty) + throw Error("Git tree '%s' is dirty", actualUrl); + + if (settings.warnDirty) + warn("Git tree '%s' is dirty", actualUrl); + + auto files = tokenizeString>( + runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + auto tree = Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + } + }; + + return {std::move(tree), input}; + } + } + + if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + + Attrs mutableAttrs({ + {"type", cacheType}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + Path repoDir; + + if (isLocal) { + + if (!input->rev) + input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + + repoDir = actualUrl; + + } else { + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + repoDir = cacheDir; + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } + + Path localRefFile = + input->ref->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input->ref + : cacheDir + "/refs/heads/" + *input->ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (input->rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + try { + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + } + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!input->rev) + input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + } + + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + + if (isShallow && !shallow) + throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + + // FIXME: check whether rev is an ancestor of ref. + + printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + + /* Now that we know the ref, check again whether we have it in + the store. */ + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + unpackTarfile(*source, tmpDir); + + auto storePath = store->addToStore(name, tmpDir); + + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"lastModified", lastModified}, + }); + + if (!shallow) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct GitInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return nullptr; + + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "git"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow") + throw Error("unsupported Git input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + + input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc new file mode 100644 index 000000000..3fc95ff51 --- /dev/null +++ b/src/libfetchers/github.cc @@ -0,0 +1,195 @@ +#include "download.hh" +#include "cache.hh" +#include "fetchers.hh" +#include "globals.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); +std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct GitHubInput : Input +{ + std::string owner; + std::string repo; + std::optional ref; + std::optional rev; + + std::string type() const override { return "github"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && owner == other2->owner + && repo == other2->repo + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + auto path = owner + "/" + repo; + assert(!(ref && rev)); + if (ref) path += "/" + *ref; + if (rev) path += "/" + rev->to_string(Base16, false); + return ParsedURL { + .scheme = "github", + .path = path, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("owner", owner); + attrs.emplace("repo", repo); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto rev = this->rev; + auto ref = this->ref.value_or("master"); + + if (!rev) { + auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", + owner, repo, ref); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + rev = Hash(json["sha"], htSHA1); + debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + } + + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = *rev; + + Attrs immutableAttrs({ + {"type", "git-tarball"}, + {"rev", rev->gitRev()}, + }); + + if (auto res = getCache()->lookup(store, immutableAttrs)) { + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + owner, repo, rev->to_string(Base16, false)); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + auto tree = downloadTarball(store, url, "source", true); + + getCache()->add( + store, + immutableAttrs, + { + {"rev", rev->gitRev()}, + {"lastModified", *tree.info.lastModified} + }, + tree.storePath, + true); + + return {std::move(tree), input}; + } +}; + +struct GitHubInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "github") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + input->rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + input->ref = path[2]; + else + throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (input->rev) + throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); + if (input->ref) + throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); + input->ref = value; + } + } + + if (input->ref && input->rev) + throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); + + input->owner = path[0]; + input->repo = path[1]; + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "github") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") + throw Error("unsupported GitHub input attribute '%s'", name); + + auto input = std::make_unique(); + input->owner = getStrAttr(attrs, "owner"); + input->repo = getStrAttr(attrs, "repo"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk new file mode 100644 index 000000000..d7143d8a6 --- /dev/null +++ b/src/libfetchers/local.mk @@ -0,0 +1,11 @@ +libraries += libfetchers + +libfetchers_NAME = libnixfetchers + +libfetchers_DIR := $(d) + +libfetchers_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers_CXXFLAGS += -I src/libutil -I src/libstore + +libfetchers_LIBS = libutil libstore libnixrust diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc new file mode 100644 index 000000000..790f7c753 --- /dev/null +++ b/src/libfetchers/mercurial.cc @@ -0,0 +1,303 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +struct MercurialInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + + MercurialInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "hg"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + url2.scheme = "hg+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + return url; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::pair getActualUrl() const + { + bool isLocal = url.scheme == "file"; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // FIXME: return lastModified. + + // FIXME: don't clone local repositories. + + if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + + bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked + files. */ + + if (!settings.allowDirty) + throw Error("Mercurial tree '%s' is unclean", actualUrl); + + if (settings.warnDirty) + warn("Mercurial tree '%s' is unclean", actualUrl); + + input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + + auto files = tokenizeString>( + runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + return {Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input}; + } + } + + if (!input->ref) input->ref = "default"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", "hg"}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree{ + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = getIntAttr(infoAttrs, "revCount"), + }, + }, + input + }; + }; + + if (input->rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + assert(input->rev || input->ref); + auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + + Attrs mutableAttrs({ + {"type", "hg"}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + + /* If this is a commit hash that we already have, we don't + have to pull again. */ + if (!(input->rev + && pathExists(cacheDir) + && runProgram( + RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + .killStderr(true)).second == "1")) + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); + } + } + + auto tokens = tokenizeString>( + runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); + assert(tokens.size() == 3); + + input->rev = Hash(tokens[0], htSHA1); + auto revCount = std::stoull(tokens[1]); + input->ref = tokens[2]; + + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + + deletePath(tmpDir + "/.hg_archival.txt"); + + auto storePath = store->addToStore(name, tmpDir); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"revCount", (int64_t) revCount}, + }); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct MercurialInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return nullptr; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev") + throw Error("unsupported Mercurial input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc new file mode 100644 index 000000000..b0a83001e --- /dev/null +++ b/src/libfetchers/tarball.cc @@ -0,0 +1,277 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "download.hh" +#include "globals.hh" +#include "store-api.hh" +#include "archive.hh" +#include "tarfile.hh" + +namespace nix::fetchers { + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + // FIXME: check store + + Attrs inAttrs({ + {"type", "file"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + auto useCached = [&]() -> DownloadFileResult + { + return { + .storePath = std::move(cached->storePath), + .etag = getStrAttr(cached->infoAttrs, "etag"), + .effectiveUrl = getStrAttr(cached->infoAttrs, "url") + }; + }; + + if (cached && !cached->expired) + return useCached(); + + DownloadRequest request(url); + if (cached) + request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); + DownloadResult res; + try { + res = getDownloader()->download(request); + } catch (DownloadError & e) { + if (cached) { + warn("%s; using cached version", e.msg()); + return useCached(); + } else + throw; + } + + // FIXME: write to temporary file. + + Attrs infoAttrs({ + {"etag", res.etag}, + {"url", res.effectiveUri}, + }); + + std::optional storePath; + + if (res.cached) { + assert(cached); + assert(request.expectedETag == res.etag); + storePath = std::move(cached->storePath); + } else { + StringSink sink; + dumpString(*res.data, sink); + auto hash = hashString(htSHA256, *res.data); + ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(false, hash); + store->addToStore(info, sink.s, NoRepair, NoCheckSigs); + storePath = std::move(info.path); + } + + getCache()->add( + store, + inAttrs, + infoAttrs, + *storePath, + immutable); + + if (url != res.effectiveUri) + getCache()->add( + store, + { + {"type", "file"}, + {"url", res.effectiveUri}, + {"name", name}, + }, + infoAttrs, + *storePath, + immutable); + + return { + .storePath = std::move(*storePath), + .etag = res.etag, + .effectiveUrl = res.effectiveUri, + }; +} + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + Attrs inAttrs({ + {"type", "tarball"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + if (cached && !cached->expired) + return Tree { + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), + .info = TreeInfo { + .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), + }, + }; + + auto res = downloadFile(store, url, name, immutable); + + std::optional unpackedStorePath; + time_t lastModified; + + if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { + unpackedStorePath = std::move(cached->storePath); + lastModified = getIntAttr(cached->infoAttrs, "lastModified"); + } else { + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + unpackTarfile(store->toRealPath(res.storePath), tmpDir); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); + } + + Attrs infoAttrs({ + {"lastModified", lastModified}, + {"etag", res.etag}, + }); + + getCache()->add( + store, + inAttrs, + infoAttrs, + *unpackedStorePath, + immutable); + + return Tree { + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), + .info = TreeInfo { + .lastModified = lastModified, + }, + }; +} + +struct TarballInput : Input +{ + ParsedURL url; + std::optional hash; + + TarballInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "tarball"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && to_string() == other2->to_string() + && hash == other2->hash; + } + + bool isImmutable() const override + { + return hash || narHash; + } + + ParsedURL toURL() const override + { + auto url2(url); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (narHash) + url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); + else if (hash) + url2.query.insert_or_assign("hash", hash->to_string(SRI)); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); + else if (hash) + attrs.emplace("hash", hash->to_string(SRI)); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto tree = downloadTarball(store, url.to_string(), "source", false); + + auto input = std::make_shared(*this); + input->narHash = store->queryPathInfo(tree.storePath)->narHash; + + return {std::move(tree), input}; + } +}; + +struct TarballInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + + if (!hasSuffix(url.path, ".zip") + && !hasSuffix(url.path, ".tar") + && !hasSuffix(url.path, ".tar.gz") + && !hasSuffix(url.path, ".tar.xz") + && !hasSuffix(url.path, ".tar.bz2")) + return nullptr; + + auto input = std::make_unique(url); + + auto hash = input->url.query.find("hash"); + if (hash != input->url.query.end()) { + // FIXME: require SRI hash. + input->hash = Hash(hash->second); + input->url.query.erase(hash); + } + + auto narHash = input->url.query.find("narHash"); + if (narHash != input->url.query.end()) { + // FIXME: require SRI hash. + input->narHash = Hash(narHash->second); + input->url.query.erase(narHash); + } + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "hash" && name != "narHash") + throw Error("unsupported tarball input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto hash = maybeGetStrAttr(attrs, "hash")) + // FIXME: require SRI hash. + input->hash = Hash(*hash); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc new file mode 100644 index 000000000..5788e94a1 --- /dev/null +++ b/src/libfetchers/tree-info.cc @@ -0,0 +1,14 @@ +#include "tree-info.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +StorePath TreeInfo::computeStorePath(Store & store) const +{ + assert(narHash); + return store.makeFixedOutputPath(true, narHash, "source"); +} + +} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh new file mode 100644 index 000000000..2c7347281 --- /dev/null +++ b/src/libfetchers/tree-info.hh @@ -0,0 +1,29 @@ +#pragma once + +#include "path.hh" +#include "hash.hh" + +#include + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct TreeInfo +{ + Hash narHash; + std::optional revCount; + std::optional lastModified; + + bool operator ==(const TreeInfo & other) const + { + return + narHash == other.narHash + && revCount == other.revCount + && lastModified == other.lastModified; + } + + StorePath computeStorePath(Store & store) const; +}; + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 973ddc86a..0067032af 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -378,7 +378,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, - readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first; + readDerivation(store, store.toRealPath(i.first)), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 5967d0425..215046b72 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -35,10 +35,6 @@ DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); -CachedDownloadRequest::CachedDownloadRequest(const std::string & uri) - : uri(uri), ttl(settings.tarballTtl) -{ } - std::string resolveUri(const std::string & uri) { if (uri.compare(0, 8, "channel:") == 0) @@ -802,139 +798,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached( - ref store, const CachedDownloadRequest & request) -{ - auto url = resolveUri(request.uri); - - auto name = request.name; - if (name == "") { - auto p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); - } - - std::optional expectedStorePath; - if (request.expectedHash) { - expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); - if (store->isValidPath(*expectedStorePath)) { - CachedDownloadResult result; - result.storePath = store->printStorePath(*expectedStorePath); - result.path = store->toRealPath(result.storePath); - return result; - } - } - - Path cacheDir = getCacheDir() + "/nix/tarballs"; - createDirs(cacheDir); - - string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); - - Path dataFile = cacheDir + "/" + urlHash + ".info"; - Path fileLink = cacheDir + "/" + urlHash + "-file"; - - PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); - - std::optional storePath; - - string expectedETag; - - bool skip = false; - - CachedDownloadResult result; - - if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = store->parseStorePath(readLink(fileLink)); - // FIXME - store->addTempRoot(*storePath); - if (store->isValidPath(*storePath)) { - auto ss = tokenizeString>(readFile(dataFile), "\n"); - if (ss.size() >= 3 && ss[0] == url) { - time_t lastChecked; - if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { - skip = true; - result.effectiveUri = request.uri; - result.etag = ss[1]; - } else if (!ss[1].empty()) { - debug(format("verifying previous ETag '%1%'") % ss[1]); - expectedETag = ss[1]; - } - } - } else - storePath.reset(); - } - - if (!skip) { - - try { - DownloadRequest request2(url); - request2.expectedETag = expectedETag; - auto res = download(request2); - result.effectiveUri = res.effectiveUri; - result.etag = res.etag; - - if (!res.cached) { - StringSink sink; - dumpString(*res.data, sink); - Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = info.path.clone(); - } - - assert(storePath); - replaceSymlink(store->printStorePath(*storePath), fileLink); - - writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); - } catch (DownloadError & e) { - if (!storePath) throw; - warn("warning: %s; using cached result", e.msg()); - result.etag = expectedETag; - } - } - - if (request.unpack) { - Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked"; - PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); - std::optional unpackedStorePath; - if (pathExists(unpackedLink)) { - unpackedStorePath = store->parseStorePath(readLink(unpackedLink)); - // FIXME - store->addTempRoot(*unpackedStorePath); - if (!store->isValidPath(*unpackedStorePath)) - unpackedStorePath.reset(); - } - if (!unpackedStorePath) { - printInfo("unpacking '%s'...", url); - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); - } - replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink); - storePath = std::move(*unpackedStorePath); - } - - if (expectedStorePath && *storePath != *expectedStorePath) { - unsigned int statusCode = 102; - Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first - : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))); - throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, request.expectedHash.to_string(), gotHash.to_string()); - } - - result.storePath = store->printStorePath(*storePath); - result.path = store->toRealPath(result.storePath); - return result; -} - bool isUri(const string & s) { diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 5a131c704..28c8a9162 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -65,28 +65,6 @@ struct DownloadResult uint64_t bodySize = 0; }; -struct CachedDownloadRequest -{ - std::string uri; - bool unpack = false; - std::string name; - Hash expectedHash; - unsigned int ttl; - - CachedDownloadRequest(const std::string & uri); - CachedDownloadRequest() = delete; -}; - -struct CachedDownloadResult -{ - // Note: 'storePath' may be different from 'path' when using a - // chroot store. - Path storePath; - Path path; - std::optional etag; - std::string effectiveUri; -}; - class Store; struct Downloader @@ -108,12 +86,6 @@ struct Downloader invoked on the thread of the caller. */ void download(DownloadRequest && request, Sink & sink); - /* Check if the specified file is already in ~/.cache/nix/tarballs - and is more recent than ‘tarball-ttl’ seconds. Otherwise, - use the recorded ETag to verify if the server has a more - recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached(ref store, const CachedDownloadRequest & request); - enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; @@ -135,4 +107,7 @@ public: bool isUri(const string & s); +/* Resolve deprecated 'channel:' URLs. */ +std::string resolveUri(const std::string & uri); + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 3aa3653f3..40f350f0b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -351,12 +351,21 @@ public: Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + Setting githubAccessToken{this, "", "github-access-token", + "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; bool isExperimentalFeatureEnabled(const std::string & name); void requireExperimentalFeature(const std::string & name); + + Setting allowDirty{this, true, "allow-dirty", + "Whether to allow dirty Git/Mercurial trees."}; + + Setting warnDirty{this, true, "warn-dirty", + "Whether to warn about dirty Git/Mercurial trees."}; }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b9e894a9a..e5282bb30 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,6 +6,7 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "url.hh" #include @@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw NotInStore("path '%1%' is not in the Nix store", path); return path; } @@ -866,27 +867,7 @@ std::pair splitUriAndParams(const std::string & uri_ Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - for (auto s : tokenizeString(uri.substr(q + 1), "&")) { - auto e = s.find('='); - if (e != std::string::npos) { - auto value = s.substr(e + 1); - std::string decoded; - for (size_t i = 0; i < value.size(); ) { - if (value[i] == '%') { - if (i + 2 >= value.size()) - throw Error("invalid URI parameter '%s'", value); - try { - decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); - i += 3; - } catch (...) { - throw Error("invalid URI parameter '%s'", value); - } - } else - decoded += value[i++]; - } - params[s.substr(0, e)] = decoded; - } - } + params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } return {uri, params}; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0fa59be6a..81014763d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -28,6 +28,7 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); +MakeError(NotInStore, Error); struct BasicDerivation; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 20b96a85c..a1ce7b372 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -157,4 +157,12 @@ typedef list Paths; typedef set PathSet; +/* Helper class to run code at startup. */ +template +struct OnStartup +{ + OnStartup(T && t) { t(); } +}; + + } diff --git a/src/libutil/url.cc b/src/libutil/url.cc new file mode 100644 index 000000000..5d5328e5d --- /dev/null +++ b/src/libutil/url.cc @@ -0,0 +1,137 @@ +#include "url.hh" +#include "util.hh" + +namespace nix { + +std::regex refRegex(refRegexS, std::regex::ECMAScript); +std::regex revRegex(revRegexS, std::regex::ECMAScript); +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); + +ParsedURL parseURL(const std::string & url) +{ + static std::regex uriRegex( + "((" + schemeRegex + "):" + + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + std::smatch match; + + if (std::regex_match(url, match, uriRegex)) { + auto & base = match[1]; + std::string scheme = match[2]; + auto authority = match[3].matched + ? std::optional(match[3]) : std::nullopt; + std::string path = match[4].matched ? match[4] : match[5]; + auto & query = match[6]; + auto & fragment = match[7]; + + auto isFile = scheme.find("file") != std::string::npos; + + if (authority && *authority != "" && isFile) + throw Error("file:// URL '%s' has unexpected authority '%s'", + url, *authority); + + if (isFile && path.empty()) + path = "/"; + + return ParsedURL{ + .url = url, + .base = base, + .scheme = scheme, + .authority = authority, + .path = path, + .query = decodeQuery(query), + .fragment = percentDecode(std::string(fragment)) + }; + } + + else + throw BadURL("'%s' is not a valid URL", url); +} + +std::string percentDecode(std::string_view in) +{ + std::string decoded; + for (size_t i = 0; i < in.size(); ) { + if (in[i] == '%') { + if (i + 2 >= in.size()) + throw BadURL("invalid URI parameter '%s'", in); + try { + decoded += std::stoul(std::string(in, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw BadURL("invalid URI parameter '%s'", in); + } + } else + decoded += in[i++]; + } + return decoded; +} + +std::map decodeQuery(const std::string & query) +{ + std::map result; + + for (auto s : tokenizeString(query, "&")) { + auto e = s.find('='); + if (e != std::string::npos) + result.emplace( + s.substr(0, e), + percentDecode(std::string_view(s).substr(e + 1))); + } + + return result; +} + +std::string percentEncode(std::string_view s) +{ + std::string res; + for (auto & c : s) + if ((c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || strchr("-._~!$&'()*+,;=:@", c)) + res += c; + else + res += fmt("%%%02x", (unsigned int) c); + return res; +} + +std::string encodeQuery(const std::map & ss) +{ + std::string res; + bool first = true; + for (auto & [name, value] : ss) { + if (!first) res += '&'; + first = false; + res += percentEncode(name); + res += '='; + res += percentEncode(value); + } + return res; +} + +std::string ParsedURL::to_string() const +{ + return + scheme + + ":" + + (authority ? "//" + *authority : "") + + path + + (query.empty() ? "" : "?" + encodeQuery(query)) + + (fragment.empty() ? "" : "#" + percentEncode(fragment)); +} + +bool ParsedURL::operator ==(const ParsedURL & other) const +{ + return + scheme == other.scheme + && authority == other.authority + && path == other.path + && query == other.query + && fragment == other.fragment; +} + +} diff --git a/src/libutil/url.hh b/src/libutil/url.hh new file mode 100644 index 000000000..1503023a2 --- /dev/null +++ b/src/libutil/url.hh @@ -0,0 +1,62 @@ +#pragma once + +#include "types.hh" + +#include + +namespace nix { + +struct ParsedURL +{ + std::string url; + std::string base; // URL without query/fragment + std::string scheme; + std::optional authority; + std::string path; + std::map query; + std::string fragment; + + std::string to_string() const; + + bool operator ==(const ParsedURL & other) const; +}; + +MakeError(BadURL, Error); + +std::string percentDecode(std::string_view in); + +std::map decodeQuery(const std::string & query); + +ParsedURL parseURL(const std::string & url); + +// URI stuff. +const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; +const static std::string schemeRegex = "(?:[a-z+]+)"; +const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; +const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; +const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; +const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; +const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; +const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; +const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; + +} diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index a2639579d..2a9defb4e 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -3,6 +3,7 @@ #include "download.hh" #include "store-api.hh" #include "../nix/legacy.hh" +#include "fetchers.hh" #include #include @@ -86,12 +87,9 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - CachedDownloadRequest request(url); - request.ttl = 0; - auto dl = getDownloader(); - auto result = dl->downloadCached(store, request); - auto filename = result.path; - url = chomp(result.effectiveUri); + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto filename = store->toRealPath(result.storePath); + url = result.effectiveUrl; // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -114,11 +112,10 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); } catch (DownloadError & e) { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } - chomp(filename); } // Regardless of where it came from, add the expression representing this channel to accumulated expression @@ -185,6 +182,8 @@ static int _main(int argc, char ** argv) } else if (*arg == "--rollback") { cmd = cRollback; } else { + if (hasPrefix(*arg, "-")) + throw UsageError("unsupported argument '%s'", *arg); args.push_back(std::move(*arg)); } return true; diff --git a/src/nix/local.mk b/src/nix/local.mk index 50a18efd7..033675e89 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,9 +15,9 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -nix_LIBS = libexpr libmain libstore libutil libnixrust +nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index ed8fa14d6..d9c9874f5 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -11,7 +11,7 @@ repo=$TEST_ROOT/git export _NIX_FORCE_HTTP=1 -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow git init $repo git -C $repo config user.email "foobar@example.com" @@ -25,8 +25,16 @@ rev1=$(git -C $repo rev-parse HEAD) echo world > $repo/hello git -C $repo commit -m 'Bla2' -a +git -C $repo worktree add $TEST_ROOT/worktree +echo hello >> $TEST_ROOT/worktree/hello rev2=$(git -C $repo rev-parse HEAD) +# Fetch a worktree +unset _NIX_FORCE_HTTP +path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") +export _NIX_FORCE_HTTP=1 +[[ $(tail -n 1 $path0/hello) = "hello" ]] + # Fetch the default branch. path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] @@ -50,9 +58,6 @@ path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") [[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]] [[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] -# But with TTL 0, it should fail. -(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv) - # Fetching with a explicit hash should succeed. path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] @@ -74,6 +79,7 @@ echo bar > $repo/dir2/bar git -C $repo add dir1/foo git -C $repo rm hello +unset _NIX_FORCE_HTTP path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") [ ! -e $path2/hello ] [ ! -e $path2/bar ] @@ -110,9 +116,9 @@ path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") git -C $repo checkout $rev2 -b dev echo dev > $repo/hello -# File URI uses 'master' unless specified otherwise +# File URI uses dirty tree unless specified otherwise path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") -[[ $path = $path2 ]] +[ $(cat $path2/hello) = dev ] # Using local path with branch other than 'master' should work when clean or dirty path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath") @@ -131,9 +137,9 @@ path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outP # Nuke the cache -rm -rf $TEST_HOME/.cache/nix/gitv2 +rm -rf $TEST_HOME/.cache/nix -# Try again, but without 'git' on PATH +# Try again, but without 'git' on PATH. This should fail. NIX=$(command -v nix) # This should fail (! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) @@ -141,3 +147,13 @@ NIX=$(command -v nix) # Try again, with 'git' available. This should work. path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] + +# Fetching a shallow repo shouldn't work by default, because we can't +# return a revCount. +git clone --depth 1 file://$repo $TEST_ROOT/shallow +(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath") + +# But you can request a shallow clone, which won't return a revCount. +path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath") +[[ $path3 = $path6 ]] +[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] diff --git a/tests/init.sh b/tests/init.sh index 6a119aad0..c62c4856a 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -17,7 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <&1 | grep 'NAR hash mismatch in input' + nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext (! nix-instantiate --eval -E ' 1' -I fnord=file://no-such-tarball$ext) From 670feb000a9fac76f0996711f061ec466a53dc97 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 14:56:20 +0200 Subject: [PATCH 020/118] Add 'path' fetcher This fetchers copies a plain directory (i.e. not a Git/Mercurial repository) to the store (or does nothing if the path is already a store path). One use case is to pin the 'nixpkgs' flake used to build the current NixOS system, and prevent it from being garbage-collected, via a system registry entry like this: { "from": { "id": "nixpkgs", "type": "indirect" }, "to": { "type": "path", "path": "/nix/store/rralhl3wj4rdwzjn16g7d93mibvlr521-source", "lastModified": 1585388205, "rev": "b0c285807d6a9f1b7562ec417c24fa1a30ecc31a" }, "exact": true } Note the fake "lastModified" and "rev" attributes that ensure that the flake gives the same evaluation results as the corresponding Git/GitHub inputs. (cherry picked from commit 12f9379123eba828f2ae06f7978a37b7045c2b23) --- src/libfetchers/attrs.cc | 15 +++++ src/libfetchers/attrs.hh | 2 + src/libfetchers/path.cc | 130 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 src/libfetchers/path.cc diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 40c02de42..feb0a6085 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -89,4 +89,19 @@ bool getBoolAttr(const Attrs & attrs, const std::string & name) return *s; } +std::map attrsToQuery(const Attrs & attrs) +{ + std::map query; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + query.insert_or_assign(attr.first, fmt("%d", *v)); + } else if (auto v = std::get_if(&attr.second)) { + query.insert_or_assign(attr.first, *v); + } else if (auto v = std::get_if>(&attr.second)) { + query.insert_or_assign(attr.first, v->t ? "1" : "0"); + } else abort(); + } + return query; +} + } diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 2c9e772d2..d6e0ae000 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -34,4 +34,6 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na bool getBoolAttr(const Attrs & attrs, const std::string & name); +std::map attrsToQuery(const Attrs & attrs); + } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc new file mode 100644 index 000000000..9801d9b86 --- /dev/null +++ b/src/libfetchers/path.cc @@ -0,0 +1,130 @@ +#include "fetchers.hh" +#include "store-api.hh" + +namespace nix::fetchers { + +struct PathInput : Input +{ + Path path; + + /* Allow the user to pass in "fake" tree info attributes. This is + useful for making a pinned tree work the same as the repository + from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + std::optional rev; + std::optional revCount; + std::optional lastModified; + + std::string type() const override { return "path"; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + auto query = attrsToQuery(toAttrsInternal()); + query.erase("path"); + return ParsedURL { + .scheme = "path", + .path = path, + .query = query, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("path", path); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (revCount) + attrs.emplace("revCount", *revCount); + if (lastModified) + attrs.emplace("lastModified", *lastModified); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto input = std::make_shared(*this); + + auto storePath = store->maybeParseStorePath(path); + + if (storePath) + store->addTempRoot(*storePath); + + if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) + // FIXME: try to substitute storePath. + storePath = store->addToStore("name", path); + + return + { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + .info = TreeInfo { + .revCount = revCount, + .lastModified = lastModified + } + }, + input + }; + } + +}; + +struct PathInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "path") return nullptr; + + auto input = std::make_unique(); + input->path = url.path; + + for (auto & [name, value] : url.query) + if (name == "rev") + input->rev = Hash(value, htSHA1); + else if (name == "revCount") { + uint64_t revCount; + if (!string2Int(value, revCount)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->revCount = revCount; + } + else if (name == "lastModified") { + time_t lastModified; + if (!string2Int(value, lastModified)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->lastModified = lastModified; + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + auto input = std::make_unique(); + input->path = getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + if (name == "rev") + input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); + else if (name == "revCount") + input->revCount = getIntAttr(attrs, "revCount"); + else if (name == "lastModified") + input->lastModified = getIntAttr(attrs, "lastModified"); + else if (name == "type" || name == "path") + ; + else + throw Error("unsupported path input attribute '%s'", name); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} From f58a9b0e62879e28cc7cac59b489a52c924bbc36 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 19:04:33 +0200 Subject: [PATCH 021/118] Respect the narHash attribute in more input types (cherry picked from commit a6ff66b658b61aef80d936f0183447fe4cb46000) --- src/libfetchers/fetchers.cc | 4 +++- src/libfetchers/git.cc | 2 +- src/libfetchers/github.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/tarball.cc | 6 ++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 25d3da431..94ac30e38 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -29,8 +29,10 @@ std::unique_ptr inputFromURL(const std::string & url) std::unique_ptr inputFromAttrs(const Attrs & attrs) { + auto attrs2(attrs); + attrs2.erase("narHash"); for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); + auto res = inputScheme->inputFromAttrs(attrs2); if (res) { if (auto narHash = maybeGetStrAttr(attrs, "narHash")) // FIXME: require SRI hash. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index cee0713f4..3f94d9bdd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -39,7 +39,7 @@ struct GitInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 3fc95ff51..ef27eaa76 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -33,7 +33,7 @@ struct GitHubInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 790f7c753..1d6571571 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -33,7 +33,7 @@ struct MercurialInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b0a83001e..4c4e5828e 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -205,9 +205,7 @@ struct TarballInput : Input { Attrs attrs; attrs.emplace("url", url.to_string()); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - else if (hash) + if (hash) attrs.emplace("hash", hash->to_string(SRI)); return attrs; } @@ -260,7 +258,7 @@ struct TarballInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash" && name != "narHash") + if (name != "type" && name != "url" && name != "hash") throw Error("unsupported tarball input attribute '%s'", name); auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); From a6dfa3cb85b59a0979de7fe02c9d67939c4ac217 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 19:04:27 +0200 Subject: [PATCH 022/118] PathInput: Add some methods (cherry picked from commit 78ad5b3d91507427fa563f3474dc52da608ad224) --- src/libfetchers/path.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 9801d9b86..037404726 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -19,6 +19,22 @@ struct PathInput : Input std::optional getRev() const override { return rev; } + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && path == other2->path + && rev == other2->rev + && revCount == other2->revCount + && lastModified == other2->lastModified; + } + + bool isImmutable() const override + { + return (bool) narHash; + } + ParsedURL toURL() const override { auto query = attrsToQuery(toAttrsInternal()); From 26aeeb7653fa051ddec913bdc2c578b9066bc08f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 6 Apr 2020 14:28:37 +0200 Subject: [PATCH 023/118] Add FIXME (cherry picked from commit 2f494531b7811b45f6b76787f225495a14d28a7f) --- src/libfetchers/path.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 037404726..7c7e20f4e 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -63,6 +63,8 @@ struct PathInput : Input { auto input = std::make_shared(*this); + // FIXME: check whether access to 'path' is allowed. + auto storePath = store->maybeParseStorePath(path); if (storePath) From cd39709003eee4b85a31799b8bc2de59721930ce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Apr 2020 09:27:17 +0200 Subject: [PATCH 024/118] Cleanup --- src/libstore/download.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 215046b72..af69699a8 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -1,14 +1,10 @@ #include "download.hh" #include "util.hh" #include "globals.hh" -#include "hash.hh" #include "store-api.hh" -#include "archive.hh" #include "s3.hh" #include "compression.hh" -#include "pathlocks.hh" #include "finally.hh" -#include "tarfile.hh" #ifdef ENABLE_S3 #include From 9d04b5da17898fd564308423bc7d10fc929efe18 Mon Sep 17 00:00:00 2001 From: mlatus Date: Tue, 7 Apr 2020 20:29:40 +0800 Subject: [PATCH 025/118] `nix run` using $SHELL as default command --- src/nix/run.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/run.cc b/src/nix/run.cc index 8e30264c0..5334531fd 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -59,14 +59,14 @@ struct RunCommon : virtual Command struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment { - std::vector command = { "bash" }; + std::vector command = { getEnv("SHELL").value_or("bash") }; CmdRun() { mkFlag() .longName("command") .shortName('c') - .description("command and arguments to be executed; defaults to 'bash'") + .description("command and arguments to be executed; defaults to '$SHELL'") .labels({"command", "args"}) .arity(ArityAny) .handler([&](std::vector ss) { From 1ab8d6ac1861e9405ae34af3deb681020c03e82d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Apr 2020 15:18:41 +0200 Subject: [PATCH 026/118] Downloader: Only write data to the sink on a 200 response Hopefully fixes #3278. --- src/libstore/download.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index af69699a8..bc875f653 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -83,8 +83,15 @@ struct CurlDownloader : public Downloader , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { if (this->request.dataCallback) { - writtenToSink += len; - this->request.dataCallback((char *) data, len); + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + + /* Only write data to the sink if this is a + successful response. */ + if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + writtenToSink += len; + this->request.dataCallback((char *) data, len); + } } else this->result.data->append((char *) data, len); }) From bf81b3155998152582abdad1550277cf8ca32e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 8 Apr 2020 18:27:10 +0200 Subject: [PATCH 027/118] build.cc: improve message if home directory exists --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 527d7ac42..632982ecd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2161,7 +2161,7 @@ void DerivationGoal::startBuilder() if (needsHashRewrite()) { if (pathExists(homeDir)) - throw Error(format("directory '%1%' exists; please remove it") % homeDir); + throw Error(format("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing") % homeDir); /* We're not doing a chroot build, but we have some valid output paths. Since we can't just overwrite or delete From 65ef57e0cbded52652510355304303a7b71586af Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 13:30:45 +0200 Subject: [PATCH 028/118] DownloadRequest -> DataTransferRequest --- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/download.cc | 13 ++++++------- src/libstore/download.hh | 12 ++++++------ src/libstore/http-binary-cache-store.cc | 8 ++++---- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index f6ae5d2e6..a939f040f 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -36,7 +36,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) /* No need to do TLS verification, because we check the hash of the result anyway. */ - DownloadRequest request(url); + DataTransferRequest request(url); request.verifyTLS = false; request.decompress = false; diff --git a/src/libstore/download.cc b/src/libstore/download.cc index bc875f653..77f3c527a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -49,7 +49,7 @@ struct CurlDownloader : public Downloader struct DownloadItem : public std::enable_shared_from_this { CurlDownloader & downloader; - DownloadRequest request; + DataTransferRequest request; DownloadResult result; Activity act; bool done = false; // whether either the success or failure function has been called @@ -73,7 +73,7 @@ struct CurlDownloader : public Downloader curl_off_t writtenToSink = 0; DownloadItem(CurlDownloader & downloader, - const DownloadRequest & request, + const DataTransferRequest & request, Callback && callback) : downloader(downloader) , request(request) @@ -641,7 +641,7 @@ struct CurlDownloader : public Downloader } #endif - void enqueueDownload(const DownloadRequest & request, + void enqueueDownload(const DataTransferRequest & request, Callback callback) override { /* Ugly hack to support s3:// URIs. */ @@ -687,7 +687,7 @@ ref makeDownloader() return make_ref(); } -std::future Downloader::enqueueDownload(const DownloadRequest & request) +std::future Downloader::enqueueDownload(const DataTransferRequest & request) { auto promise = std::make_shared>(); enqueueDownload(request, @@ -701,12 +701,12 @@ std::future Downloader::enqueueDownload(const DownloadRequest & return promise->get_future(); } -DownloadResult Downloader::download(const DownloadRequest & request) +DownloadResult Downloader::download(const DataTransferRequest & request) { return enqueueDownload(request).get(); } -void Downloader::download(DownloadRequest && request, Sink & sink) +void Downloader::download(DataTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because that would cause the sink to execute on the downloader @@ -801,7 +801,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } - bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 28c8a9162..89d3fce7f 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -33,7 +33,7 @@ struct DownloadSettings : Config extern DownloadSettings downloadSettings; -struct DownloadRequest +struct DataTransferRequest { std::string uri; std::string expectedETag; @@ -47,7 +47,7 @@ struct DownloadRequest std::string mimeType; std::function dataCallback; - DownloadRequest(const std::string & uri) + DataTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() @@ -74,17 +74,17 @@ struct Downloader /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ - virtual void enqueueDownload(const DownloadRequest & request, + virtual void enqueueDownload(const DataTransferRequest & request, Callback callback) = 0; - std::future enqueueDownload(const DownloadRequest & request); + std::future enqueueDownload(const DataTransferRequest & request); /* Synchronously download a file. */ - DownloadResult download(const DownloadRequest & request); + DownloadResult download(const DataTransferRequest & request); /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ - void download(DownloadRequest && request, Sink & sink); + void download(DataTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 011794c62..e152e7524 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -85,7 +85,7 @@ protected: checkEnabled(); try { - DownloadRequest request(cacheUri + "/" + path); + DataTransferRequest request(cacheUri + "/" + path); request.head = true; getDownloader()->download(request); return true; @@ -103,7 +103,7 @@ protected: const std::string & data, const std::string & mimeType) override { - auto req = DownloadRequest(cacheUri + "/" + path); + auto req = DataTransferRequest(cacheUri + "/" + path); req.data = std::make_shared(data); // FIXME: inefficient req.mimeType = mimeType; try { @@ -113,9 +113,9 @@ protected: } } - DownloadRequest makeRequest(const std::string & path) + DataTransferRequest makeRequest(const std::string & path) { - DownloadRequest request(cacheUri + "/" + path); + DataTransferRequest request(cacheUri + "/" + path); return request; } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 2b9254659..bfd65da98 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -180,7 +180,7 @@ static int _main(int argc, char * * argv) FdSink sink(fd.get()); - DownloadRequest req(actualUri); + DataTransferRequest req(actualUri); req.decompress = false; getDownloader()->download(std::move(req), sink); } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index c05c29517..aeaf7b09c 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -138,7 +138,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = DownloadRequest(storePathsUrl); + auto req = DataTransferRequest(storePathsUrl); auto res = getDownloader()->download(req); auto state = std::make_unique(Strings(), store); From 741e9012d3fa6f0fb41a4a5662c6b8b38ecfaa1f Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 15:10:58 +0200 Subject: [PATCH 029/118] Rename src/lib/download.* to src/lib/datatransfer.* --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/parser.y | 2 +- src/libstore/build.cc | 2 +- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/{download.cc => datatransfer.cc} | 2 +- src/libstore/{download.hh => datatransfer.hh} | 0 src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/main.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 13 files changed, 12 insertions(+), 12 deletions(-) rename src/libstore/{download.cc => datatransfer.cc} (99%) rename src/libstore/{download.hh => datatransfer.hh} (100%) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 82bfeac36..26460601d 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -1,6 +1,6 @@ #include "common-eval-args.hh" #include "shared.hh" -#include "download.hh" +#include "datatransfer.hh" #include "util.hh" #include "eval.hh" #include "fetchers.hh" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dac32b6f5..7a20c6fa6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,7 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" -#include "download.hh" +#include "datatransfer.hh" #include "json.hh" #include "function-trace.hh" diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 6f25f5cf0..96b6f0ecd 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -544,7 +544,7 @@ formal #include #include "eval.hh" -#include "download.hh" +#include "datatransfer.hh" #include "fetchers.hh" #include "store-api.hh" diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 632982ecd..db01d9510 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -7,7 +7,7 @@ #include "affinity.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "download.hh" +#include "datatransfer.hh" #include "finally.hh" #include "compression.hh" #include "json.hh" diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index a939f040f..01d21fa98 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,5 +1,5 @@ #include "builtins.hh" -#include "download.hh" +#include "datatransfer.hh" #include "store-api.hh" #include "archive.hh" #include "compression.hh" diff --git a/src/libstore/download.cc b/src/libstore/datatransfer.cc similarity index 99% rename from src/libstore/download.cc rename to src/libstore/datatransfer.cc index 77f3c527a..30ad48577 100644 --- a/src/libstore/download.cc +++ b/src/libstore/datatransfer.cc @@ -1,4 +1,4 @@ -#include "download.hh" +#include "datatransfer.hh" #include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/download.hh b/src/libstore/datatransfer.hh similarity index 100% rename from src/libstore/download.hh rename to src/libstore/datatransfer.hh diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index e152e7524..1fccee518 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,5 +1,5 @@ #include "binary-cache-store.hh" -#include "download.hh" +#include "datatransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f2e4b63e0..ed3c690b6 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -6,7 +6,7 @@ #include "nar-info-disk-cache.hh" #include "globals.hh" #include "compression.hh" -#include "download.hh" +#include "datatransfer.hh" #include "istringstream_nocopy.hh" #include diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 2a9defb4e..d0719194d 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "globals.hh" -#include "download.hh" +#include "datatransfer.hh" #include "store-api.hh" #include "../nix/legacy.hh" #include "fetchers.hh" diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index bfd65da98..58a180e72 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -1,6 +1,6 @@ #include "hash.hh" #include "shared.hh" -#include "download.hh" +#include "datatransfer.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 3b5f5516f..ad4102201 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,7 +8,7 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "download.hh" +#include "datatransfer.hh" #include "finally.hh" #include diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index aeaf7b09c..414689ec5 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "store-api.hh" -#include "download.hh" +#include "datatransfer.hh" #include "eval.hh" #include "attr-path.hh" #include "names.hh" From e5cc53beec62a36ebdc219e26b87f501ddb84a4f Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 15:14:49 +0200 Subject: [PATCH 030/118] DownloadSettings -> DataTransferSettings --- src/libstore/datatransfer.cc | 14 +++++++------- src/libstore/datatransfer.hh | 6 +++--- src/nix/main.cc | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 30ad48577..62f546cfa 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -27,9 +27,9 @@ using namespace std::string_literals; namespace nix { -DownloadSettings downloadSettings; +DataTransferSettings dataTransferSettings; -static GlobalConfig::Register r1(&downloadSettings); +static GlobalConfig::Register r1(&dataTransferSettings); std::string resolveUri(const std::string & uri) { @@ -257,12 +257,12 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); + (dataTransferSettings.userAgentSuffix != "" ? " " + dataTransferSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif #if LIBCURL_VERSION_NUM >= 0x072f00 - if (downloadSettings.enableHttp2) + if (dataTransferSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); @@ -297,10 +297,10 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, dataTransferSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, dataTransferSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ @@ -469,7 +469,7 @@ struct CurlDownloader : public Downloader #endif #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - downloadSettings.httpConnections.get()); + dataTransferSettings.httpConnections.get()); #endif wakeupPipe.create(); diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index 89d3fce7f..16955ff06 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -9,7 +9,7 @@ namespace nix { -struct DownloadSettings : Config +struct DataTransferSettings : Config { Setting enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; @@ -31,7 +31,7 @@ struct DownloadSettings : Config "How often Nix will attempt to download a file before giving up."}; }; -extern DownloadSettings downloadSettings; +extern DataTransferSettings dataTransferSettings; struct DataTransferRequest { @@ -39,7 +39,7 @@ struct DataTransferRequest std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = downloadSettings.tries; + size_t tries = dataTransferSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; diff --git a/src/nix/main.cc b/src/nix/main.cc index ad4102201..6bb04d8b6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -176,10 +176,10 @@ void mainWrapped(int argc, char * * argv) settings.useSubstitutes = false; if (!settings.tarballTtl.overriden) settings.tarballTtl = std::numeric_limits::max(); - if (!downloadSettings.tries.overriden) - downloadSettings.tries = 0; - if (!downloadSettings.connectTimeout.overriden) - downloadSettings.connectTimeout = 1; + if (!dataTransferSettings.tries.overriden) + dataTransferSettings.tries = 0; + if (!dataTransferSettings.connectTimeout.overriden) + dataTransferSettings.connectTimeout = 1; } if (args.refresh) From 142ed7fe45a2b1d566e4401b3974cb29b8944fc6 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 15:18:51 +0200 Subject: [PATCH 031/118] DownloadResult -> DataTransferResult --- src/libstore/datatransfer.cc | 20 ++++++++++---------- src/libstore/datatransfer.hh | 8 ++++---- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 4 ++-- src/libstore/s3.hh | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 62f546cfa..06aa86d97 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -50,10 +50,10 @@ struct CurlDownloader : public Downloader { CurlDownloader & downloader; DataTransferRequest request; - DownloadResult result; + DataTransferResult result; Activity act; bool done = false; // whether either the success or failure function has been called - Callback callback; + Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object std::string status; @@ -74,7 +74,7 @@ struct CurlDownloader : public Downloader DownloadItem(CurlDownloader & downloader, const DataTransferRequest & request, - Callback && callback) + Callback && callback) : downloader(downloader) , request(request) , act(*logger, lvlTalkative, actDownload, @@ -642,7 +642,7 @@ struct CurlDownloader : public Downloader #endif void enqueueDownload(const DataTransferRequest & request, - Callback callback) override + Callback callback) override { /* Ugly hack to support s3:// URIs. */ if (hasPrefix(request.uri, "s3://")) { @@ -660,7 +660,7 @@ struct CurlDownloader : public Downloader // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); - DownloadResult res; + DataTransferResult res; if (!s3Res.data) throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; @@ -687,11 +687,11 @@ ref makeDownloader() return make_ref(); } -std::future Downloader::enqueueDownload(const DataTransferRequest & request) +std::future Downloader::enqueueDownload(const DataTransferRequest & request) { - auto promise = std::make_shared>(); + auto promise = std::make_shared>(); enqueueDownload(request, - {[promise](std::future fut) { + {[promise](std::future fut) { try { promise->set_value(fut.get()); } catch (...) { @@ -701,7 +701,7 @@ std::future Downloader::enqueueDownload(const DataTransferReques return promise->get_future(); } -DownloadResult Downloader::download(const DataTransferRequest & request) +DataTransferResult Downloader::download(const DataTransferRequest & request) { return enqueueDownload(request).get(); } @@ -756,7 +756,7 @@ void Downloader::download(DataTransferRequest && request, Sink & sink) }; enqueueDownload(request, - {[_state](std::future fut) { + {[_state](std::future fut) { auto state(_state->lock()); state->quit = true; try { diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index 16955ff06..b9e55655f 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -56,7 +56,7 @@ struct DataTransferRequest } }; -struct DownloadResult +struct DataTransferResult { bool cached = false; std::string etag; @@ -75,12 +75,12 @@ struct Downloader the download. The future may throw a DownloadError exception. */ virtual void enqueueDownload(const DataTransferRequest & request, - Callback callback) = 0; + Callback callback) = 0; - std::future enqueueDownload(const DataTransferRequest & request); + std::future enqueueDownload(const DataTransferRequest & request); /* Synchronously download a file. */ - DownloadResult download(const DataTransferRequest & request); + DataTransferResult download(const DataTransferRequest & request); /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 1fccee518..24a98d073 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -143,7 +143,7 @@ protected: auto callbackPtr = std::make_shared(std::move(callback)); getDownloader()->enqueueDownload(request, - {[callbackPtr, this](std::future result) { + {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); } catch (DownloadError & e) { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ed3c690b6..478cb9f84 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -132,7 +132,7 @@ ref S3Helper::makeConfig(const string & region return res; } -S3Helper::DownloadResult S3Helper::getObject( +S3Helper::DataTransferResult S3Helper::getObject( const std::string & bucketName, const std::string & key) { debug("fetching 's3://%s/%s'...", bucketName, key); @@ -146,7 +146,7 @@ S3Helper::DownloadResult S3Helper::getObject( return Aws::New("STRINGSTREAM"); }); - DownloadResult res; + DataTransferResult res; auto now1 = std::chrono::steady_clock::now(); diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index ef5f23d0f..d7d309243 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -18,13 +18,13 @@ struct S3Helper ref makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); - struct DownloadResult + struct DataTransferResult { std::shared_ptr data; unsigned int durationMs; }; - DownloadResult getObject( + DataTransferResult getObject( const std::string & bucketName, const std::string & key); }; From 2df2741ec6e9411739b2049d7487b7b8a26d4547 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 15:37:33 +0200 Subject: [PATCH 032/118] enqueueDownload -> enqueueDataTransfer --- src/libstore/datatransfer.cc | 10 +++++----- src/libstore/datatransfer.hh | 4 ++-- src/libstore/http-binary-cache-store.cc | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 06aa86d97..9475c24ec 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -641,7 +641,7 @@ struct CurlDownloader : public Downloader } #endif - void enqueueDownload(const DataTransferRequest & request, + void enqueueDataTransfer(const DataTransferRequest & request, Callback callback) override { /* Ugly hack to support s3:// URIs. */ @@ -687,10 +687,10 @@ ref makeDownloader() return make_ref(); } -std::future Downloader::enqueueDownload(const DataTransferRequest & request) +std::future Downloader::enqueueDataTransfer(const DataTransferRequest & request) { auto promise = std::make_shared>(); - enqueueDownload(request, + enqueueDataTransfer(request, {[promise](std::future fut) { try { promise->set_value(fut.get()); @@ -703,7 +703,7 @@ std::future Downloader::enqueueDownload(const DataTransferRe DataTransferResult Downloader::download(const DataTransferRequest & request) { - return enqueueDownload(request).get(); + return enqueueDataTransfer(request).get(); } void Downloader::download(DataTransferRequest && request, Sink & sink) @@ -755,7 +755,7 @@ void Downloader::download(DataTransferRequest && request, Sink & sink) state->avail.notify_one(); }; - enqueueDownload(request, + enqueueDataTransfer(request, {[_state](std::future fut) { auto state(_state->lock()); state->quit = true; diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index b9e55655f..53159bdb4 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -74,10 +74,10 @@ struct Downloader /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ - virtual void enqueueDownload(const DataTransferRequest & request, + virtual void enqueueDataTransfer(const DataTransferRequest & request, Callback callback) = 0; - std::future enqueueDownload(const DataTransferRequest & request); + std::future enqueueDataTransfer(const DataTransferRequest & request); /* Synchronously download a file. */ DataTransferResult download(const DataTransferRequest & request); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 24a98d073..861287a33 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -142,7 +142,7 @@ protected: auto callbackPtr = std::make_shared(std::move(callback)); - getDownloader()->enqueueDownload(request, + getDownloader()->enqueueDataTransfer(request, {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); From cd391206e6d29608bc3e25301b9921f1ac189086 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 22:57:09 +0200 Subject: [PATCH 033/118] {get,make,new}Downloader -> DataTransfer --- src/libstore/builtins/fetchurl.cc | 6 ++-- src/libstore/datatransfer.cc | 36 ++++++++++++------------ src/libstore/datatransfer.hh | 18 ++++++------ src/libstore/http-binary-cache-store.cc | 14 ++++----- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 6 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 01d21fa98..0b6db8f0c 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto mainUrl = getAttr("url"); bool unpack = get(drv.env, "unpack").value_or("") == "1"; - /* Note: have to use a fresh downloader here because we're in + /* Note: have to use a fresh dataTransfer here because we're in a forked process. */ - auto downloader = makeDownloader(); + auto dataTransfer = makeDataTransfer(); auto fetch = [&](const std::string & url) { @@ -42,7 +42,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto decompressor = makeDecompressionSink( unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); - downloader->download(std::move(request), *decompressor); + dataTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 9475c24ec..6ea9f9324 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -39,7 +39,7 @@ std::string resolveUri(const std::string & uri) return uri; } -struct CurlDownloader : public Downloader +struct curlDataTransfer : public DataTransfer { CURLM * curlm = 0; @@ -48,7 +48,7 @@ struct CurlDownloader : public Downloader struct DownloadItem : public std::enable_shared_from_this { - CurlDownloader & downloader; + curlDataTransfer & dataTransfer; DataTransferRequest request; DataTransferResult result; Activity act; @@ -72,10 +72,10 @@ struct CurlDownloader : public Downloader curl_off_t writtenToSink = 0; - DownloadItem(CurlDownloader & downloader, + DownloadItem(curlDataTransfer & dataTransfer, const DataTransferRequest & request, Callback && callback) - : downloader(downloader) + : dataTransfer(dataTransfer) , request(request) , act(*logger, lvlTalkative, actDownload, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), @@ -106,7 +106,7 @@ struct CurlDownloader : public Downloader { if (req) { if (active) - curl_multi_remove_handle(downloader.curlm, req); + curl_multi_remove_handle(dataTransfer.curlm, req); curl_easy_cleanup(req); } if (requestHeaders) curl_slist_free_all(requestHeaders); @@ -422,13 +422,13 @@ struct CurlDownloader : public Downloader || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(dataTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); + dataTransfer.enqueueItem(shared_from_this()); } else fail(exc); @@ -456,7 +456,7 @@ struct CurlDownloader : public Downloader std::thread workerThread; - CurlDownloader() + curlDataTransfer() : mt19937(rd()) { static std::once_flag globalInit; @@ -478,7 +478,7 @@ struct CurlDownloader : public Downloader workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~CurlDownloader() + ~curlDataTransfer() { stopWorkerThread(); @@ -676,18 +676,18 @@ struct CurlDownloader : public Downloader } }; -ref getDownloader() +ref getDataTransfer() { - static ref downloader = makeDownloader(); - return downloader; + static ref dataTransfer = makeDataTransfer(); + return dataTransfer; } -ref makeDownloader() +ref makeDataTransfer() { - return make_ref(); + return make_ref(); } -std::future Downloader::enqueueDataTransfer(const DataTransferRequest & request) +std::future DataTransfer::enqueueDataTransfer(const DataTransferRequest & request) { auto promise = std::make_shared>(); enqueueDataTransfer(request, @@ -701,15 +701,15 @@ std::future Downloader::enqueueDataTransfer(const DataTransf return promise->get_future(); } -DataTransferResult Downloader::download(const DataTransferRequest & request) +DataTransferResult DataTransfer::download(const DataTransferRequest & request) { return enqueueDataTransfer(request).get(); } -void Downloader::download(DataTransferRequest && request, Sink & sink) +void DataTransfer::download(DataTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because - that would cause the sink to execute on the downloader + that would cause the sink to execute on the dataTransfer thread. If 'sink' is a coroutine, this will fail. Also, if the sink is expensive (e.g. one that does decompression and writing to the Nix store), it would stall the download thread too much. diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index 53159bdb4..e1c5196e0 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -67,11 +67,11 @@ struct DataTransferResult class Store; -struct Downloader +struct DataTransfer { - virtual ~Downloader() { } + virtual ~DataTransfer() { } - /* Enqueue a download request, returning a future to the result of + /* Enqueue a data transfer request, returning a future to the result of the download. The future may throw a DownloadError exception. */ virtual void enqueueDataTransfer(const DataTransferRequest & request, @@ -89,18 +89,18 @@ struct Downloader enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared Downloader object. Using this object is preferred +/* Return a shared DataTransfer object. Using this object is preferred because it enables connection reuse and HTTP/2 multiplexing. */ -ref getDownloader(); +ref getDataTransfer(); -/* Return a new Downloader object. */ -ref makeDownloader(); +/* Return a new DataTransfer object. */ +ref makeDataTransfer(); class DownloadError : public Error { public: - Downloader::Error error; - DownloadError(Downloader::Error error, const FormatOrString & fs) + DataTransfer::Error error; + DownloadError(DataTransfer::Error error, const FormatOrString & fs) : Error(fs), error(error) { } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 861287a33..9a19af5e0 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -87,12 +87,12 @@ protected: try { DataTransferRequest request(cacheUri + "/" + path); request.head = true; - getDownloader()->download(request); + getDataTransfer()->download(request); return true; } catch (DownloadError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) return false; maybeDisable(); throw; @@ -107,7 +107,7 @@ protected: req.data = std::make_shared(data); // FIXME: inefficient req.mimeType = mimeType; try { - getDownloader()->download(req); + getDataTransfer()->download(req); } catch (DownloadError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } @@ -124,9 +124,9 @@ protected: checkEnabled(); auto request(makeRequest(path)); try { - getDownloader()->download(std::move(request), sink); + getDataTransfer()->download(std::move(request), sink); } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); throw; @@ -142,12 +142,12 @@ protected: auto callbackPtr = std::make_shared(std::move(callback)); - getDownloader()->enqueueDataTransfer(request, + getDataTransfer()->enqueueDataTransfer(request, {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) return (*callbackPtr)(std::shared_ptr()); maybeDisable(); callbackPtr->rethrow(); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 58a180e72..4c81e7d82 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -182,7 +182,7 @@ static int _main(int argc, char * * argv) DataTransferRequest req(actualUri); req.decompress = false; - getDownloader()->download(std::move(req), sink); + getDataTransfer()->download(std::move(req), sink); } /* Optionally unpack the file. */ diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 414689ec5..a4eda90b4 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -139,7 +139,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand // FIXME: use nixos.org? auto req = DataTransferRequest(storePathsUrl); - auto res = getDownloader()->download(req); + auto res = getDataTransfer()->download(req); auto state = std::make_unique(Strings(), store); auto v = state->allocValue(); From 213d124277460b316cfa03d43e4ec279bba86219 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 22:59:36 +0200 Subject: [PATCH 034/118] DownloadItem -> TransferItem --- src/libstore/datatransfer.cc | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 6ea9f9324..452af107d 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -46,7 +46,7 @@ struct curlDataTransfer : public DataTransfer std::random_device rd; std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this + struct TransferItem : public std::enable_shared_from_this { curlDataTransfer & dataTransfer; DataTransferRequest request; @@ -72,7 +72,7 @@ struct curlDataTransfer : public DataTransfer curl_off_t writtenToSink = 0; - DownloadItem(curlDataTransfer & dataTransfer, + TransferItem(curlDataTransfer & dataTransfer, const DataTransferRequest & request, Callback && callback) : dataTransfer(dataTransfer) @@ -102,7 +102,7 @@ struct curlDataTransfer : public DataTransfer requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); } - ~DownloadItem() + ~TransferItem() { if (req) { if (active) @@ -156,7 +156,7 @@ struct curlDataTransfer : public DataTransfer static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb); + return ((TransferItem *) userp)->writeCallback(contents, size, nmemb); } size_t headerCallback(void * contents, size_t size, size_t nmemb) @@ -198,7 +198,7 @@ struct curlDataTransfer : public DataTransfer static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb); + return ((TransferItem *) userp)->headerCallback(contents, size, nmemb); } int progressCallback(double dltotal, double dlnow) @@ -213,7 +213,7 @@ struct curlDataTransfer : public DataTransfer static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) { - return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); + return ((TransferItem *) userp)->progressCallback(dltotal, dlnow); } static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) @@ -237,7 +237,7 @@ struct curlDataTransfer : public DataTransfer static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) { - return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); + return ((TransferItem *) userp)->readCallback(buffer, size, nitems); } void init() @@ -248,7 +248,7 @@ struct curlDataTransfer : public DataTransfer if (verbosity >= lvlVomit) { curl_easy_setopt(req, CURLOPT_VERBOSE, 1); - curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); } curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); @@ -267,9 +267,9 @@ struct curlDataTransfer : public DataTransfer else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif - curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); + curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); - curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); + curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper); curl_easy_setopt(req, CURLOPT_HEADERDATA, this); curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); @@ -439,12 +439,12 @@ struct curlDataTransfer : public DataTransfer struct State { struct EmbargoComparator { - bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { + bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { return i1->embargo > i2->embargo; } }; bool quit = false; - std::priority_queue, std::vector>, EmbargoComparator> incoming; + std::priority_queue, std::vector>, EmbargoComparator> incoming; }; Sync state_; @@ -504,7 +504,7 @@ struct curlDataTransfer : public DataTransfer stopWorkerThread(); }); - std::map> items; + std::map> items; bool quit = false; @@ -561,7 +561,7 @@ struct curlDataTransfer : public DataTransfer throw SysError("reading curl wakeup socket"); } - std::vector> incoming; + std::vector> incoming; auto now = std::chrono::steady_clock::now(); { @@ -609,7 +609,7 @@ struct curlDataTransfer : public DataTransfer } } - void enqueueItem(std::shared_ptr item) + void enqueueItem(std::shared_ptr item) { if (item->request.data && !hasPrefix(item->request.uri, "http://") @@ -672,7 +672,7 @@ struct curlDataTransfer : public DataTransfer return; } - enqueueItem(std::make_shared(*this, request, std::move(callback))); + enqueueItem(std::make_shared(*this, request, std::move(callback))); } }; From c4c1ae0a00e8dd3258af58c15b664828b592133e Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:00:43 +0200 Subject: [PATCH 035/118] DownloadError -> DataTransferError --- src/libexpr/parser.y | 2 +- src/libstore/datatransfer.cc | 10 +++++----- src/libstore/datatransfer.hh | 6 +++--- src/libstore/http-binary-cache-store.cc | 8 ++++---- src/nix-channel/nix-channel.cc | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 96b6f0ecd..8fbe59301 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -690,7 +690,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl try { res = { true, store->toRealPath(fetchers::downloadTarball( store, resolveUri(elem.second), "source", false).storePath) }; - } catch (DownloadError & e) { + } catch (DataTransferError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; } diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 452af107d..be5dd27cd 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -112,7 +112,7 @@ struct curlDataTransfer : public DataTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + fail(DataTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri)); } catch (...) { ignoreException(); } @@ -401,14 +401,14 @@ struct curlDataTransfer : public DataTransfer auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? DataTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? DownloadError(err, + ? DataTransferError(err, fmt("unable to %s '%s': HTTP error %d", request.verb(), request.uri, httpStatus) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) - : DownloadError(err, + : DataTransferError(err, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -662,7 +662,7 @@ struct curlDataTransfer : public DataTransfer auto s3Res = s3Helper.getObject(bucketName, key); DataTransferResult res; if (!s3Res.data) - throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw DataTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; callback(std::move(res)); #else diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index e1c5196e0..f2ea53ae3 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -72,7 +72,7 @@ struct DataTransfer virtual ~DataTransfer() { } /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a DownloadError + the download. The future may throw a DataTransferError exception. */ virtual void enqueueDataTransfer(const DataTransferRequest & request, Callback callback) = 0; @@ -96,11 +96,11 @@ ref getDataTransfer(); /* Return a new DataTransfer object. */ ref makeDataTransfer(); -class DownloadError : public Error +class DataTransferError : public Error { public: DataTransfer::Error error; - DownloadError(DataTransfer::Error error, const FormatOrString & fs) + DataTransferError(DataTransfer::Error error, const FormatOrString & fs) : Error(fs), error(error) { } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9a19af5e0..5e446ddf2 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -89,7 +89,7 @@ protected: request.head = true; getDataTransfer()->download(request); return true; - } catch (DownloadError & e) { + } catch (DataTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) @@ -108,7 +108,7 @@ protected: req.mimeType = mimeType; try { getDataTransfer()->download(req); - } catch (DownloadError & e) { + } catch (DataTransferError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } } @@ -125,7 +125,7 @@ protected: auto request(makeRequest(path)); try { getDataTransfer()->download(std::move(request), sink); - } catch (DownloadError & e) { + } catch (DataTransferError & e) { if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); @@ -146,7 +146,7 @@ protected: {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); - } catch (DownloadError & e) { + } catch (DataTransferError & e) { if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) return (*callbackPtr)(std::shared_ptr()); maybeDisable(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index d0719194d..0f8024d7b 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -113,7 +113,7 @@ static void update(const StringSet & channelNames) // Download the channel tarball. try { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); - } catch (DownloadError & e) { + } catch (DataTransferError & e) { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } } From a0c5931208042da39bb6a5e80a4b27cf50f665d6 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:08:39 +0200 Subject: [PATCH 036/118] actDownload -> actDataTransfer --- src/libstore/build.cc | 2 +- src/libstore/datatransfer.cc | 2 +- src/libutil/logging.cc | 2 +- src/libutil/logging.hh | 2 +- src/nix/progress-bar.cc | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index db01d9510..f091a000d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -361,7 +361,7 @@ public: { actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize); + act.setExpected(actDataTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } }; diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index be5dd27cd..9ed06b5c1 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -77,7 +77,7 @@ struct curlDataTransfer : public DataTransfer Callback && callback) : dataTransfer(dataTransfer) , request(request) - , act(*logger, lvlTalkative, actDownload, + , act(*logger, lvlTalkative, actDataTransfer, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index fa5c84a27..001a3fb28 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -198,7 +198,7 @@ bool handleJSONLogMessage(const std::string & msg, if (action == "start") { auto type = (ActivityType) json["type"]; - if (trusted || type == actDownload) + if (trusted || type == actDataTransfer) activities.emplace(std::piecewise_construct, std::forward_as_tuple(json["id"]), std::forward_as_tuple(*logger, (Verbosity) json["level"], type, diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index beb5e6b64..423ed95da 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -17,7 +17,7 @@ typedef enum { typedef enum { actUnknown = 0, actCopyPath = 100, - actDownload = 101, + actDataTransfer = 101, actRealise = 102, actCopyPaths = 103, actBuilds = 104, diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 26631416c..d108896ad 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -190,8 +190,8 @@ public: i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); } - if ((type == actDownload && hasAncestor(*state, actCopyPath, parent)) - || (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent)) + if ((type == actDataTransfer && hasAncestor(*state, actCopyPath, parent)) + || (type == actDataTransfer && hasAncestor(*state, actQueryPathInfo, parent)) || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; @@ -416,7 +416,7 @@ public: if (!s2.empty()) { res += " ("; res += s2; res += ')'; } } - showActivity(actDownload, "%s MiB DL", "%.1f", MiB); + showActivity(actDataTransfer, "%s MiB DL", "%.1f", MiB); { auto s = renderActivity(actOptimiseStore, "%s paths optimised"); From 7848372b0f683a5a4db5f86fd998e8df5fa22715 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:34:31 +0200 Subject: [PATCH 037/118] Add upload method --- src/libstore/datatransfer.cc | 6 ++++++ src/libstore/datatransfer.hh | 3 +++ src/libstore/http-binary-cache-store.cc | 3 +-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 9ed06b5c1..dbdf07ad4 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -706,6 +706,12 @@ DataTransferResult DataTransfer::download(const DataTransferRequest & request) return enqueueDataTransfer(request).get(); } +DataTransferResult DataTransfer::upload(const DataTransferRequest & request) +{ + /* Note: this method is the same as download, but helps in readability */ + return enqueueDataTransfer(request).get(); +} + void DataTransfer::download(DataTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index f2ea53ae3..68d97ceb1 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -82,6 +82,9 @@ struct DataTransfer /* Synchronously download a file. */ DataTransferResult download(const DataTransferRequest & request); + /* Synchronously upload a file. */ + DataTransferResult upload(const DataTransferRequest & request); + /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ void download(DataTransferRequest && request, Sink & sink); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 5e446ddf2..c75846bc3 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -107,7 +107,7 @@ protected: req.data = std::make_shared(data); // FIXME: inefficient req.mimeType = mimeType; try { - getDataTransfer()->download(req); + getDataTransfer()->upload(req); } catch (DataTransferError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } @@ -174,4 +174,3 @@ static RegisterStoreImplementation regStore([]( }); } - From c330109bfa38370f7cef6449efb88309a78e1684 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:43:43 +0200 Subject: [PATCH 038/118] DataTransfer -> FileTransfer --- src/libstore/build.cc | 2 +- src/libstore/builtins/fetchurl.cc | 8 +- src/libstore/datatransfer.cc | 94 ++++++++++++------------ src/libstore/datatransfer.hh | 44 +++++------ src/libstore/http-binary-cache-store.cc | 32 ++++---- src/libstore/s3-binary-cache-store.cc | 4 +- src/libstore/s3.hh | 4 +- src/libutil/logging.cc | 2 +- src/libutil/logging.hh | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 4 +- src/nix/main.cc | 8 +- src/nix/progress-bar.cc | 6 +- src/nix/upgrade-nix.cc | 4 +- 13 files changed, 107 insertions(+), 107 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f091a000d..9baa2cf1a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -361,7 +361,7 @@ public: { actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actDataTransfer, expectedDownloadSize + doneDownloadSize); + act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } }; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 0b6db8f0c..1432de9d7 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto mainUrl = getAttr("url"); bool unpack = get(drv.env, "unpack").value_or("") == "1"; - /* Note: have to use a fresh dataTransfer here because we're in + /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ - auto dataTransfer = makeDataTransfer(); + auto fileTransfer = makeFileTransfer(); auto fetch = [&](const std::string & url) { @@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) /* No need to do TLS verification, because we check the hash of the result anyway. */ - DataTransferRequest request(url); + FileTransferRequest request(url); request.verifyTLS = false; request.decompress = false; auto decompressor = makeDecompressionSink( unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); - dataTransfer->download(std::move(request), *decompressor); + fileTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index dbdf07ad4..5cc63a1d8 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -27,9 +27,9 @@ using namespace std::string_literals; namespace nix { -DataTransferSettings dataTransferSettings; +FileTransferSettings fileTransferSettings; -static GlobalConfig::Register r1(&dataTransferSettings); +static GlobalConfig::Register r1(&fileTransferSettings); std::string resolveUri(const std::string & uri) { @@ -39,7 +39,7 @@ std::string resolveUri(const std::string & uri) return uri; } -struct curlDataTransfer : public DataTransfer +struct curlFileTransfer : public FileTransfer { CURLM * curlm = 0; @@ -48,12 +48,12 @@ struct curlDataTransfer : public DataTransfer struct TransferItem : public std::enable_shared_from_this { - curlDataTransfer & dataTransfer; - DataTransferRequest request; - DataTransferResult result; + curlFileTransfer & fileTransfer; + FileTransferRequest request; + FileTransferResult result; Activity act; bool done = false; // whether either the success or failure function has been called - Callback callback; + Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object std::string status; @@ -72,12 +72,12 @@ struct curlDataTransfer : public DataTransfer curl_off_t writtenToSink = 0; - TransferItem(curlDataTransfer & dataTransfer, - const DataTransferRequest & request, - Callback && callback) - : dataTransfer(dataTransfer) + TransferItem(curlFileTransfer & fileTransfer, + const FileTransferRequest & request, + Callback && callback) + : fileTransfer(fileTransfer) , request(request) - , act(*logger, lvlTalkative, actDataTransfer, + , act(*logger, lvlTalkative, actFileTransfer, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) @@ -106,13 +106,13 @@ struct curlDataTransfer : public DataTransfer { if (req) { if (active) - curl_multi_remove_handle(dataTransfer.curlm, req); + curl_multi_remove_handle(fileTransfer.curlm, req); curl_easy_cleanup(req); } if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(DataTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + fail(FileTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri)); } catch (...) { ignoreException(); } @@ -257,12 +257,12 @@ struct curlDataTransfer : public DataTransfer curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (dataTransferSettings.userAgentSuffix != "" ? " " + dataTransferSettings.userAgentSuffix.get() : "")).c_str()); + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif #if LIBCURL_VERSION_NUM >= 0x072f00 - if (dataTransferSettings.enableHttp2) + if (fileTransferSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); @@ -297,10 +297,10 @@ struct curlDataTransfer : public DataTransfer curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, dataTransferSettings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, dataTransferSettings.stalledDownloadTimeout.get()); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ @@ -401,14 +401,14 @@ struct curlDataTransfer : public DataTransfer auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DataTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? DataTransferError(err, + ? FileTransferError(err, fmt("unable to %s '%s': HTTP error %d", request.verb(), request.uri, httpStatus) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) - : DataTransferError(err, + : FileTransferError(err, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -422,13 +422,13 @@ struct curlDataTransfer : public DataTransfer || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(dataTransfer.mt19937)); + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - dataTransfer.enqueueItem(shared_from_this()); + fileTransfer.enqueueItem(shared_from_this()); } else fail(exc); @@ -456,7 +456,7 @@ struct curlDataTransfer : public DataTransfer std::thread workerThread; - curlDataTransfer() + curlFileTransfer() : mt19937(rd()) { static std::once_flag globalInit; @@ -469,7 +469,7 @@ struct curlDataTransfer : public DataTransfer #endif #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - dataTransferSettings.httpConnections.get()); + fileTransferSettings.httpConnections.get()); #endif wakeupPipe.create(); @@ -478,7 +478,7 @@ struct curlDataTransfer : public DataTransfer workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~curlDataTransfer() + ~curlFileTransfer() { stopWorkerThread(); @@ -641,8 +641,8 @@ struct curlDataTransfer : public DataTransfer } #endif - void enqueueDataTransfer(const DataTransferRequest & request, - Callback callback) override + void enqueueFileTransfer(const FileTransferRequest & request, + Callback callback) override { /* Ugly hack to support s3:// URIs. */ if (hasPrefix(request.uri, "s3://")) { @@ -660,9 +660,9 @@ struct curlDataTransfer : public DataTransfer // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); - DataTransferResult res; + FileTransferResult res; if (!s3Res.data) - throw DataTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; callback(std::move(res)); #else @@ -676,22 +676,22 @@ struct curlDataTransfer : public DataTransfer } }; -ref getDataTransfer() +ref getFileTransfer() { - static ref dataTransfer = makeDataTransfer(); - return dataTransfer; + static ref fileTransfer = makeFileTransfer(); + return fileTransfer; } -ref makeDataTransfer() +ref makeFileTransfer() { - return make_ref(); + return make_ref(); } -std::future DataTransfer::enqueueDataTransfer(const DataTransferRequest & request) +std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) { - auto promise = std::make_shared>(); - enqueueDataTransfer(request, - {[promise](std::future fut) { + auto promise = std::make_shared>(); + enqueueFileTransfer(request, + {[promise](std::future fut) { try { promise->set_value(fut.get()); } catch (...) { @@ -701,21 +701,21 @@ std::future DataTransfer::enqueueDataTransfer(const DataTran return promise->get_future(); } -DataTransferResult DataTransfer::download(const DataTransferRequest & request) +FileTransferResult FileTransfer::download(const FileTransferRequest & request) { - return enqueueDataTransfer(request).get(); + return enqueueFileTransfer(request).get(); } -DataTransferResult DataTransfer::upload(const DataTransferRequest & request) +FileTransferResult FileTransfer::upload(const FileTransferRequest & request) { /* Note: this method is the same as download, but helps in readability */ - return enqueueDataTransfer(request).get(); + return enqueueFileTransfer(request).get(); } -void DataTransfer::download(DataTransferRequest && request, Sink & sink) +void FileTransfer::download(FileTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because - that would cause the sink to execute on the dataTransfer + that would cause the sink to execute on the fileTransfer thread. If 'sink' is a coroutine, this will fail. Also, if the sink is expensive (e.g. one that does decompression and writing to the Nix store), it would stall the download thread too much. @@ -761,8 +761,8 @@ void DataTransfer::download(DataTransferRequest && request, Sink & sink) state->avail.notify_one(); }; - enqueueDataTransfer(request, - {[_state](std::future fut) { + enqueueFileTransfer(request, + {[_state](std::future fut) { auto state(_state->lock()); state->quit = true; try { diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index 68d97ceb1..2347f363d 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -9,7 +9,7 @@ namespace nix { -struct DataTransferSettings : Config +struct FileTransferSettings : Config { Setting enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; @@ -31,15 +31,15 @@ struct DataTransferSettings : Config "How often Nix will attempt to download a file before giving up."}; }; -extern DataTransferSettings dataTransferSettings; +extern FileTransferSettings fileTransferSettings; -struct DataTransferRequest +struct FileTransferRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = dataTransferSettings.tries; + size_t tries = fileTransferSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; @@ -47,7 +47,7 @@ struct DataTransferRequest std::string mimeType; std::function dataCallback; - DataTransferRequest(const std::string & uri) + FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() @@ -56,7 +56,7 @@ struct DataTransferRequest } }; -struct DataTransferResult +struct FileTransferResult { bool cached = false; std::string etag; @@ -67,43 +67,43 @@ struct DataTransferResult class Store; -struct DataTransfer +struct FileTransfer { - virtual ~DataTransfer() { } + virtual ~FileTransfer() { } /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a DataTransferError + the download. The future may throw a FileTransferError exception. */ - virtual void enqueueDataTransfer(const DataTransferRequest & request, - Callback callback) = 0; + virtual void enqueueFileTransfer(const FileTransferRequest & request, + Callback callback) = 0; - std::future enqueueDataTransfer(const DataTransferRequest & request); + std::future enqueueFileTransfer(const FileTransferRequest & request); /* Synchronously download a file. */ - DataTransferResult download(const DataTransferRequest & request); + FileTransferResult download(const FileTransferRequest & request); /* Synchronously upload a file. */ - DataTransferResult upload(const DataTransferRequest & request); + FileTransferResult upload(const FileTransferRequest & request); /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ - void download(DataTransferRequest && request, Sink & sink); + void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared DataTransfer object. Using this object is preferred +/* Return a shared FileTransfer object. Using this object is preferred because it enables connection reuse and HTTP/2 multiplexing. */ -ref getDataTransfer(); +ref getFileTransfer(); -/* Return a new DataTransfer object. */ -ref makeDataTransfer(); +/* Return a new FileTransfer object. */ +ref makeFileTransfer(); -class DataTransferError : public Error +class FileTransferError : public Error { public: - DataTransfer::Error error; - DataTransferError(DataTransfer::Error error, const FormatOrString & fs) + FileTransfer::Error error; + FileTransferError(FileTransfer::Error error, const FormatOrString & fs) : Error(fs), error(error) { } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index c75846bc3..ce0c5af3b 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -85,14 +85,14 @@ protected: checkEnabled(); try { - DataTransferRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); request.head = true; - getDataTransfer()->download(request); + getFileTransfer()->download(request); return true; - } catch (DataTransferError & e) { + } catch (FileTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ - if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return false; maybeDisable(); throw; @@ -103,19 +103,19 @@ protected: const std::string & data, const std::string & mimeType) override { - auto req = DataTransferRequest(cacheUri + "/" + path); + auto req = FileTransferRequest(cacheUri + "/" + path); req.data = std::make_shared(data); // FIXME: inefficient req.mimeType = mimeType; try { - getDataTransfer()->upload(req); - } catch (DataTransferError & e) { + getFileTransfer()->upload(req); + } catch (FileTransferError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } } - DataTransferRequest makeRequest(const std::string & path) + FileTransferRequest makeRequest(const std::string & path) { - DataTransferRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); return request; } @@ -124,9 +124,9 @@ protected: checkEnabled(); auto request(makeRequest(path)); try { - getDataTransfer()->download(std::move(request), sink); - } catch (DataTransferError & e) { - if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) + getFileTransfer()->download(std::move(request), sink); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); throw; @@ -142,12 +142,12 @@ protected: auto callbackPtr = std::make_shared(std::move(callback)); - getDataTransfer()->enqueueDataTransfer(request, - {[callbackPtr, this](std::future result) { + getFileTransfer()->enqueueFileTransfer(request, + {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); - } catch (DataTransferError & e) { - if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return (*callbackPtr)(std::shared_ptr()); maybeDisable(); callbackPtr->rethrow(); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 478cb9f84..fccf010a4 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -132,7 +132,7 @@ ref S3Helper::makeConfig(const string & region return res; } -S3Helper::DataTransferResult S3Helper::getObject( +S3Helper::FileTransferResult S3Helper::getObject( const std::string & bucketName, const std::string & key) { debug("fetching 's3://%s/%s'...", bucketName, key); @@ -146,7 +146,7 @@ S3Helper::DataTransferResult S3Helper::getObject( return Aws::New("STRINGSTREAM"); }); - DataTransferResult res; + FileTransferResult res; auto now1 = std::chrono::steady_clock::now(); diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index d7d309243..2042bffcf 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -18,13 +18,13 @@ struct S3Helper ref makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); - struct DataTransferResult + struct FileTransferResult { std::shared_ptr data; unsigned int durationMs; }; - DataTransferResult getObject( + FileTransferResult getObject( const std::string & bucketName, const std::string & key); }; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 001a3fb28..bb437cf1c 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -198,7 +198,7 @@ bool handleJSONLogMessage(const std::string & msg, if (action == "start") { auto type = (ActivityType) json["type"]; - if (trusted || type == actDataTransfer) + if (trusted || type == actFileTransfer) activities.emplace(std::piecewise_construct, std::forward_as_tuple(json["id"]), std::forward_as_tuple(*logger, (Verbosity) json["level"], type, diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 423ed95da..108b5dcb0 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -17,7 +17,7 @@ typedef enum { typedef enum { actUnknown = 0, actCopyPath = 100, - actDataTransfer = 101, + actFileTransfer = 101, actRealise = 102, actCopyPaths = 103, actBuilds = 104, diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 4c81e7d82..e136bb95a 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -180,9 +180,9 @@ static int _main(int argc, char * * argv) FdSink sink(fd.get()); - DataTransferRequest req(actualUri); + FileTransferRequest req(actualUri); req.decompress = false; - getDataTransfer()->download(std::move(req), sink); + getFileTransfer()->download(std::move(req), sink); } /* Optionally unpack the file. */ diff --git a/src/nix/main.cc b/src/nix/main.cc index 6bb04d8b6..140c963e0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -176,10 +176,10 @@ void mainWrapped(int argc, char * * argv) settings.useSubstitutes = false; if (!settings.tarballTtl.overriden) settings.tarballTtl = std::numeric_limits::max(); - if (!dataTransferSettings.tries.overriden) - dataTransferSettings.tries = 0; - if (!dataTransferSettings.connectTimeout.overriden) - dataTransferSettings.connectTimeout = 1; + if (!fileTransferSettings.tries.overriden) + fileTransferSettings.tries = 0; + if (!fileTransferSettings.connectTimeout.overriden) + fileTransferSettings.connectTimeout = 1; } if (args.refresh) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index d108896ad..adc9b9a5d 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -190,8 +190,8 @@ public: i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); } - if ((type == actDataTransfer && hasAncestor(*state, actCopyPath, parent)) - || (type == actDataTransfer && hasAncestor(*state, actQueryPathInfo, parent)) + if ((type == actFileTransfer && hasAncestor(*state, actCopyPath, parent)) + || (type == actFileTransfer && hasAncestor(*state, actQueryPathInfo, parent)) || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; @@ -416,7 +416,7 @@ public: if (!s2.empty()) { res += " ("; res += s2; res += ')'; } } - showActivity(actDataTransfer, "%s MiB DL", "%.1f", MiB); + showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB); { auto s = renderActivity(actOptimiseStore, "%s paths optimised"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a4eda90b4..574f7d8c2 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -138,8 +138,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = DataTransferRequest(storePathsUrl); - auto res = getDataTransfer()->download(req); + auto req = FileTransferRequest(storePathsUrl); + auto res = getFileTransfer()->download(req); auto state = std::make_unique(Strings(), store); auto v = state->allocValue(); From f5095594e723d02c9bf22fe24ad838ebff4f9165 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:57:28 +0200 Subject: [PATCH 039/118] datatransfer.{cc,hh} -> filetransfer.{cc,hh} --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/parser.y | 2 +- src/libstore/build.cc | 2 +- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/{datatransfer.cc => filetransfer.cc} | 2 +- src/libstore/{datatransfer.hh => filetransfer.hh} | 0 src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/main.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 13 files changed, 12 insertions(+), 12 deletions(-) rename src/libstore/{datatransfer.cc => filetransfer.cc} (99%) rename src/libstore/{datatransfer.hh => filetransfer.hh} (100%) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 26460601d..10c9c16bb 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -1,6 +1,6 @@ #include "common-eval-args.hh" #include "shared.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "util.hh" #include "eval.hh" #include "fetchers.hh" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7a20c6fa6..b91a021b4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,7 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "json.hh" #include "function-trace.hh" diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 8fbe59301..235ed5b1d 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -544,7 +544,7 @@ formal #include #include "eval.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "fetchers.hh" #include "store-api.hh" diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9baa2cf1a..0febb8dfb 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -7,7 +7,7 @@ #include "affinity.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "finally.hh" #include "compression.hh" #include "json.hh" diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1432de9d7..486babf14 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,5 +1,5 @@ #include "builtins.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "archive.hh" #include "compression.hh" diff --git a/src/libstore/datatransfer.cc b/src/libstore/filetransfer.cc similarity index 99% rename from src/libstore/datatransfer.cc rename to src/libstore/filetransfer.cc index 5cc63a1d8..e9684b3d4 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,4 +1,4 @@ -#include "datatransfer.hh" +#include "filetransfer.hh" #include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/datatransfer.hh b/src/libstore/filetransfer.hh similarity index 100% rename from src/libstore/datatransfer.hh rename to src/libstore/filetransfer.hh diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ce0c5af3b..451a64785 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,5 +1,5 @@ #include "binary-cache-store.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index fccf010a4..b24e7b7d6 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -6,7 +6,7 @@ #include "nar-info-disk-cache.hh" #include "globals.hh" #include "compression.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "istringstream_nocopy.hh" #include diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 0f8024d7b..7d584f891 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "globals.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "../nix/legacy.hh" #include "fetchers.hh" diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index e136bb95a..748554b9c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -1,6 +1,6 @@ #include "hash.hh" #include "shared.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 140c963e0..2c64c7476 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,7 +8,7 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "finally.hh" #include diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 574f7d8c2..4fcc6a721 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "store-api.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "eval.hh" #include "attr-path.hh" #include "names.hh" From 7867685dcdf1ba3c9290b777e0975ff0b775f667 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Wed, 8 Apr 2020 14:12:22 +0200 Subject: [PATCH 040/118] after flake rebase --- src/libexpr/parser.y | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/github.cc | 2 +- src/libfetchers/tarball.cc | 10 +++++----- src/nix-channel/nix-channel.cc | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 235ed5b1d..b33ab908b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -690,7 +690,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl try { res = { true, store->toRealPath(fetchers::downloadTarball( store, resolveUri(elem.second), "source", false).storePath) }; - } catch (DataTransferError & e) { + } catch (FileTransferError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9586f71ed..43c58485a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -2,7 +2,7 @@ #include "eval-inline.hh" #include "store-api.hh" #include "fetchers.hh" -#include "download.hh" +#include "filetransfer.hh" #include #include diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ef27eaa76..8675a5a66 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -1,4 +1,4 @@ -#include "download.hh" +#include "filetransfer.hh" #include "cache.hh" #include "fetchers.hh" #include "globals.hh" diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 4c4e5828e..695525b31 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,6 +1,6 @@ #include "fetchers.hh" #include "cache.hh" -#include "download.hh" +#include "filetransfer.hh" #include "globals.hh" #include "store-api.hh" #include "archive.hh" @@ -36,13 +36,13 @@ DownloadFileResult downloadFile( if (cached && !cached->expired) return useCached(); - DownloadRequest request(url); + FileTransferRequest request(url); if (cached) request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); - DownloadResult res; + FileTransferResult res; try { - res = getDownloader()->download(request); - } catch (DownloadError & e) { + res = getFileTransfer()->download(request); + } catch (FileTransferError & e) { if (cached) { warn("%s; using cached version", e.msg()); return useCached(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 7d584f891..abd390414 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -113,7 +113,7 @@ static void update(const StringSet & channelNames) // Download the channel tarball. try { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); - } catch (DataTransferError & e) { + } catch (FileTransferError & e) { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } } From a693a9fa4b92df0d3283a9de8b1e968378a026d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 9 Apr 2020 09:45:15 +0200 Subject: [PATCH 041/118] Attach pos to if expression errors --- src/libexpr/eval.cc | 2 +- src/libexpr/nixexpr.hh | 3 ++- src/libexpr/parser.y | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b91a021b4..f963a42ca 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1256,7 +1256,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) void ExprIf::eval(EvalState & state, Env & env, Value & v) { - (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); + (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v); } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 8c96de37c..25798cac6 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -262,8 +262,9 @@ struct ExprWith : Expr struct ExprIf : Expr { + Pos pos; Expr * cond, * then, * else_; - ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; + ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { }; COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index b33ab908b..1993fa6c1 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -335,7 +335,7 @@ expr_function ; expr_if - : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); } + : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } | expr_op ; From a364b1551ae8724657369d68a09626c3cbef25d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 9 Apr 2020 15:03:05 +0200 Subject: [PATCH 042/118] remote .travis.yml as we migrated to github actions --- .travis.yml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ee4ea1ac6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ -matrix: - include: - - language: osx - script: ./tests/install-darwin.sh - - language: nix - script: nix-build release.nix -A build.x86_64-linux -notifications: - email: false From 30d4618cc944a41c2ca202babd0be0da4a1cd9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 9 Apr 2020 15:04:51 +0200 Subject: [PATCH 043/118] README: add github actions badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 61054f8f2..9c0c87887 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) +[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) Nix, the purely functional package manager ------------------------------------------ From 04bedda0b605dcecacb23b65d5a05c8f4aecfbdd Mon Sep 17 00:00:00 2001 From: Philipp Middendorf Date: Thu, 9 Apr 2020 17:05:29 +0200 Subject: [PATCH 044/118] gc.cc: Ignore hidden files in temproots --- src/libstore/gc.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0c3d89611..6bab1e37c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -202,6 +202,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : readDirectory(tempRootsDir)) { + if (i.name[0] == '.') { + // Ignore hidden files. Some package managers (notably portage) create + // those to keep the directory alive. + continue; + } Path path = tempRootsDir + "/" + i.name; pid_t pid = std::stoi(i.name); From 16a4864759a80aa03b9c87b8aeaf7a4c31c03e39 Mon Sep 17 00:00:00 2001 From: Bruce Toll <4109762+tollb@users.noreply.github.com> Date: Sun, 17 Feb 2019 16:26:49 -0500 Subject: [PATCH 045/118] Delete temporary directory on successful build With --check and the --keep-failed (-K) flag, the temporary directory was being retained regardless of whether the build was successful and reproducible. This removes the temporary directory, as expected, on a reproducible check build. Added tests to verify that temporary build directories are not retained unnecessarily, particularly when using --check with --keep-failed. --- src/libstore/build.cc | 1 + tests/check.nix | 33 +++++++++++++++++++++++++++ tests/check.sh | 52 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0febb8dfb..760663ac9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1680,6 +1680,7 @@ void DerivationGoal::buildDone() } if (buildMode == bmCheck) { + deleteTmpDir(true); done(BuildResult::Built); return; } diff --git a/tests/check.nix b/tests/check.nix index 56c82e565..bca04fdaf 100644 --- a/tests/check.nix +++ b/tests/check.nix @@ -1,12 +1,45 @@ +{checkBuildId ? 0}: + with import ./config.nix; { nondeterministic = mkDerivation { + inherit checkBuildId; name = "nondeterministic"; buildCommand = '' mkdir $out date +%s.%N > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + ''; + }; + + deterministic = mkDerivation { + inherit checkBuildId; + name = "deterministic"; + buildCommand = + '' + mkdir $out + echo date > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + ''; + }; + + failed = mkDerivation { + inherit checkBuildId; + name = "failed"; + buildCommand = + '' + mkdir $out + echo date > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + false ''; }; diff --git a/tests/check.sh b/tests/check.sh index bc23a6634..b423dc0b5 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -1,14 +1,62 @@ source common.sh +checkBuildTempDirRemoved () +{ + buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1) + checkBuildIdFile=${buildDir}/checkBuildId + [[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile +} + +# written to build temp directories to verify created by this instance +checkBuildId=$(date +%s%N) + clearStore nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check -nix-build check.nix -A nondeterministic --no-out-link -nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? +# check for dangling temporary build directories +# only retain if build fails and --keep-failed is specified, or... +# ...build is non-deterministic and --check and --keep-failed are both specified +nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log || status=$? +[ "$status" = "100" ] +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link --keep-failed 2> $TEST_ROOT/log || status=$? +[ "$status" = "100" ] +if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi + +nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check --keep-failed 2> $TEST_ROOT/log +if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? + +# The above nix-build fails with status=1 on darwin (not sure why) +# ...but the primary purpose of the test case is to verify the temp directory is retained +if [ "$(uname -s)" != "Darwin" ]; then +grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] +fi +if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi clearStore From 3abf6d03c611417b309fdedf4323c08e6afbcd9c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Apr 2020 17:27:35 +0200 Subject: [PATCH 046/118] Update release script --- maintainers/upload-release.pl | 86 ++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 31 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 77534babb..cb584d427 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -1,5 +1,5 @@ #! /usr/bin/env nix-shell -#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1 +#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1 use strict; use Data::Dumper; @@ -9,12 +9,16 @@ use File::Slurp; use File::Copy; use JSON::PP; use LWP::UserAgent; +use Net::Amazon::S3; my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; -my $releasesDir = "/home/eelco/mnt/releases"; +my $releasesBucketName = "nix-releases"; +my $channelsBucketName = "nix-channels"; my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; +my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; + # FIXME: cut&paste from nixos-channel-scripts. sub fetch { my ($url, $type) = @_; @@ -42,13 +46,31 @@ my $version = $1; print STDERR "Nix revision is $nixRev, version is $version\n"; -File::Path::make_path($releasesDir); -if (system("mountpoint -q $releasesDir") != 0) { - system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die; -} +my $releaseDir = "nix/$releaseName"; -my $releaseDir = "$releasesDir/nix/$releaseName"; -File::Path::make_path($releaseDir); +my $tmpDir = "$TMPDIR/nix-release/$releaseName"; +File::Path::make_path($tmpDir); + +# S3 setup. +my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given."; +my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given."; + +my $s3 = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + host => "s3-eu-west-1.amazonaws.com", + }); + +my $releasesBucket = $s3->bucket($releasesBucketName) or die; + +my $s3_us = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + }); + +my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub downloadFile { my ($jobName, $productNr, $dstName) = @_; @@ -57,40 +79,49 @@ sub downloadFile { my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n"; $dstName //= basename($srcFile); - my $dstFile = "$releaseDir/" . $dstName; + my $tmpFile = "$tmpDir/$dstName"; - if (! -e $dstFile) { - print STDERR "downloading $srcFile to $dstFile...\n"; - system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 + if (!-e $tmpFile) { + print STDERR "downloading $srcFile to $tmpFile...\n"; + system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0 or die "unable to fetch $srcFile\n"; - rename("$dstFile.tmp", $dstFile) or die; } my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; - my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`; + my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { - print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; + print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; exit 1; } - write_file("$dstFile.sha256", $sha256_expected); + write_file("$tmpFile.sha256", $sha256_expected); - if (! -e "$dstFile.asc") { - system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n"; + if (! -e "$tmpFile.asc") { + system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n"; } - return ($dstFile, $sha256_expected); + return $sha256_expected; } downloadFile("tarball", "2"); # .tar.bz2 -my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz +my $tarballHash = downloadFile("tarball", "3"); # .tar.xz downloadFile("binaryTarball.i686-linux", "1"); downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("installerScript", "1"); +for my $fn (glob "$tmpDir/*") { + my $name = basename($fn); + my $dstKey = "$releaseDir/" . $name; + unless (defined $releasesBucket->head_key($dstKey)) { + print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n"; + $releasesBucket->add_key_filename($dstKey, $fn) + or die $releasesBucket->err . ": " . $releasesBucket->errstr; + } +} + exit if $version =~ /pre/; # Update Nixpkgs in a very hacky way. @@ -125,18 +156,11 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; -# Extract the HTML manual. -File::Path::make_path("$releaseDir/manual"); - -system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die; - -if (! -e "$releaseDir/manual/index.html") { - symlink("manual.html", "$releaseDir/manual/index.html") or die; -} - # Update the "latest" symlink. -symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die; -rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die; +$channelsBucket->add_key( + "nix-latest/install", "", + { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" }) + or die $channelsBucket->err . ": " . $channelsBucket->errstr; # Tag the release in Git. chdir("/home/eelco/Dev/nix-pristine") or die; From 8132d0a12e1b1d30973ae2c543622a46c24ec075 Mon Sep 17 00:00:00 2001 From: Bruce Toll <4109762+tollb@users.noreply.github.com> Date: Sun, 17 Feb 2019 14:34:31 -0500 Subject: [PATCH 047/118] Fix nix-build --check -K in sandbox w/o root Temporarily add user-write permission to build directory so that it can be moved out of the sandbox to the store with a .check suffix. This is necessary because the build directory has already had its permissions set read-only, but write permission is required to update the directory's parent link to move it out of the sandbox. Updated the related --check "derivation may not be deterministic" messages to consistently use the real store paths. Added test for non-root sandbox nix-build --check -K to demonstrate issue and help prevent regressions. --- src/libstore/build.cc | 30 ++++++++++++++++++++++++++---- tests/linux-sandbox.sh | 7 +++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 760663ac9..b4207e1b8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3537,6 +3537,29 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv } +static void moveCheckToStore(const Path & src, const Path & dst) +{ + /* For the rename of directory to succeed, we must be running as root or + the directory must be made temporarily writable (to update the + directory's parent link ".."). */ + struct stat st; + if (lstat(src.c_str(), &st) == -1) { + throw SysError(format("getting attributes of path '%1%'") % src); + } + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError(format("renaming '%1%' to '%2%'") % src % dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output @@ -3715,8 +3738,7 @@ void DerivationGoal::registerOutputs() if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); - if (rename(actualPath.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + moveCheckToStore(actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), @@ -3724,10 +3746,10 @@ void DerivationGoal::registerOutputs() path, dst, worker.store.printStorePath(drvPath), tmpDir); throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), path, dst); + worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst); } else throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), path); + worker.store.printStorePath(drvPath), worker.store.toRealPath(path)); } /* Since we verified the build, it's now ultimately trusted. */ diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 52967d07d..16abd974c 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -28,3 +28,10 @@ nix cat-store $outPath/foobar | grep FOOBAR # Test --check without hash rewriting. nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store + +# Test that sandboxed builds with --check and -K can move .check directory to store +nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link + +(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log) +if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi +grep -q 'may not be deterministic' $TEST_ROOT/log From e8bd1bc732ff19f33e6915d14e15df1d7de612c0 Mon Sep 17 00:00:00 2001 From: Bruce Toll <4109762+tollb@users.noreply.github.com> Date: Fri, 10 Apr 2020 18:20:12 -0400 Subject: [PATCH 048/118] Add test case for temporary directories on darwin A test case for correct handling of temporary directory deletion that was added to check.sh as part of PR #2689 was initially disabled for Darwin because of a directory permission issue in PR #2688. Now that the issue in PR #2688 is fixed, this commit enables the test case for Darwin. --- tests/check.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/check.sh b/tests/check.sh index b423dc0b5..5f25d04cb 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -49,13 +49,8 @@ checkBuildTempDirRemoved $TEST_ROOT/log nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? - -# The above nix-build fails with status=1 on darwin (not sure why) -# ...but the primary purpose of the test case is to verify the temp directory is retained -if [ "$(uname -s)" != "Darwin" ]; then grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] -fi if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi clearStore From fc14213d2da7302b237dfe5ca51f6f4d9af34e5c Mon Sep 17 00:00:00 2001 From: DavHau <42246742+DavHau@users.noreply.github.com> Date: Wed, 25 Mar 2020 07:19:00 +0000 Subject: [PATCH 049/118] improve toFile error message when containing potential drv path --- src/libexpr/primops.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..6d7622900 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1021,7 +1021,9 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu for (auto path : context) { if (path.at(0) != '/') - throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); + throw EvalError(format( + "in 'toFile': the file named '%1%' must not contain a reference " + "to a derivation but contains (%2%), at %3%") % name % path % pos); refs.insert(state.store->parseStorePath(path)); } From 4d9db420ffc9bd48da107a61c093b0d65d9d8db1 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sun, 12 Apr 2020 09:57:22 +0200 Subject: [PATCH 050/118] never use /var/folders for TMPDIR on darwin This doesn't just cause problems for nix-store --serve but also results in certain build failures. Builds that use unix domain sockets in their tests often fail because the /var/folders prefix already consumes more than half of the maximum length of socket paths. struct sockaddr_un { sa_family_t sun_family; /* AF_UNIX */ char sun_path[108]; /* Pathname */ }; --- src/libmain/shared.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d41e772e9..72ba717e5 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -155,7 +155,7 @@ void initNix() sshd). This breaks build users because they don't have access to the TMPDIR, in particular in ‘nix-store --serve’. */ #if __APPLE__ - if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) unsetenv("TMPDIR"); #endif } From d2c371927e45c1d4e4f7e4ad331a3f68d67d5469 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Mon, 13 Apr 2020 21:09:47 +0200 Subject: [PATCH 051/118] SourceExprCommand: allocate the vSourceExpr via uncollectable memory Previously the memory would occasionally be collected during eval since the GC doesn't consider the member variable as alive / doesn't scan the region of memory where the pointer lives. By using the traceable_allocator allocator provided by Boehm GC we can ensure the memory isn't collected. It should be properly freed when SourceExprCommand goes out of scope. --- src/nix/command.hh | 2 +- src/nix/installables.cc | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/nix/command.hh b/src/nix/command.hh index 23f5c9898..2c2303208 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -41,7 +41,7 @@ private: std::shared_ptr evalState; - Value * vSourceExpr = 0; + std::shared_ptr vSourceExpr; }; enum RealiseMode { Build, NoBuild, DryRun }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index f464d0aa1..902383bff 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -8,10 +8,13 @@ #include "store-api.hh" #include "shared.hh" +#include + #include namespace nix { + SourceExprCommand::SourceExprCommand() { mkFlag() @@ -24,11 +27,14 @@ SourceExprCommand::SourceExprCommand() Value * SourceExprCommand::getSourceExpr(EvalState & state) { - if (vSourceExpr) return vSourceExpr; + if (vSourceExpr) return vSourceExpr.get(); auto sToplevel = state.symbols.create("_toplevel"); - vSourceExpr = state.allocValue(); + // Allocate the vSourceExpr Value as uncollectable. Boehm GC doesn't + // consider the member variable "alive" during execution causing it to be + // GC'ed in the middle of evaluation. + vSourceExpr = std::allocate_shared(traceable_allocator()); if (file != "") state.evalFile(lookupFileArg(state, file), *vSourceExpr); @@ -69,7 +75,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) vSourceExpr->attrs->sort(); } - return vSourceExpr; + return vSourceExpr.get(); } ref SourceExprCommand::getEvalState() From 895516cadf0739dd8359a38d503e0aaf5068dc4b Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 30 Mar 2020 15:31:14 +0200 Subject: [PATCH 052/118] add NIX_USER_CONF_FILES Motivation: maintain project-level configuration files. Document the whole situation a bit better so that it corresponds to the implementation, and add NIX_USER_CONF_FILES that allows overriding which user files Nix will load during startup. --- doc/manual/command-ref/conf-file.xml | 36 ++++++++++++++----------- doc/manual/command-ref/env-common.xml | 13 ++++++--- src/libmain/shared.cc | 5 +++- src/libstore/globals.cc | 25 ++++++++++++++--- src/libstore/globals.hh | 8 +++++- src/libutil/util.hh | 1 + tests/common.sh.in | 3 +++ tests/config.sh | 18 +++++++++++++ tests/config/nix-with-substituters.conf | 2 ++ tests/local.mk | 1 + 10 files changed, 87 insertions(+), 25 deletions(-) create mode 100644 tests/config.sh create mode 100644 tests/config/nix-with-substituters.conf diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 48dce7c95..1820598e5 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -19,26 +19,30 @@ Description -Nix reads settings from two configuration files: +By default Nix reads settings from the following places: - +The system-wide configuration file +sysconfdir/nix/nix.conf +(i.e. /etc/nix/nix.conf on most systems), or +$NIX_CONF_DIR/nix.conf if +NIX_CONF_DIR is set. Values loaded in this file are not forwarded to the Nix daemon. The +client assumes that the daemon has already loaded them. + - - The system-wide configuration file - sysconfdir/nix/nix.conf - (i.e. /etc/nix/nix.conf on most systems), or - $NIX_CONF_DIR/nix.conf if - NIX_CONF_DIR is set. - +User-specific configuration files: - - The user configuration file - $XDG_CONFIG_HOME/nix/nix.conf, or - ~/.config/nix/nix.conf if - XDG_CONFIG_HOME is not set. - + + If NIX_USER_CONF_FILES is set, then each path separated by + : will be loaded in reverse order. + - + + Otherwise it will look for nix/nix.conf files in + XDG_CONFIG_DIRS and XDG_CONFIG_HOME. + + The default location is $HOME/.config/nix.conf if + those environment variables are unset. + The configuration files consist of name = diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index 696d68c34..0217de7b2 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -33,7 +33,7 @@ will cause Nix to look for paths relative to /home/eelco/Dev and - /etc/nixos, in that order. It is also + /etc/nixos, in this order. It is also possible to match paths against a prefix. For example, the value @@ -59,7 +59,7 @@ nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz A following shorthand can be used to refer to the official channels: - + nixpkgs=channel:nixos-15.09 @@ -137,12 +137,19 @@ $ mount -o bind /mnt/otherdisk/nix /nix NIX_CONF_DIR - Overrides the location of the Nix configuration + Overrides the location of the system Nix configuration directory (default prefix/etc/nix). +NIX_USER_CONF_FILES + + Overrides the location of the user Nix configuration files + to load from (defaults to the XDG spec locations). The variable is treated + as a list separated by the : token. + + TMPDIR diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d41e772e9..3c141a5fa 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -260,7 +260,10 @@ void printVersion(const string & programName) cfg.push_back("signed-caches"); #endif std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "User configuration files: " << + concatStringsSep(":", settings.nixUserConfFiles) + << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7e97f3c22..d6ea0318e 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -31,6 +31,7 @@ Settings::Settings() , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixUserConfFiles(getUserConfigFiles()) , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) @@ -77,13 +78,29 @@ void loadConfFile() ~/.nix/nix.conf or the command line. */ globalConfig.resetOverriden(); - auto dirs = getConfigDirs(); - // Iterate over them in reverse so that the ones appearing first in the path take priority - for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { - globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + auto files = settings.nixUserConfFiles; + for (auto file = files.rbegin(); file != files.rend(); file++) { + globalConfig.applyConfigFile(*file); } } +std::vector getUserConfigFiles() +{ + // Use the paths specified in NIX_USER_CONF_FILES if it has been defined + auto nixConfFiles = getEnv("NIX_USER_CONF_FILES"); + if (nixConfFiles.has_value()) { + return tokenizeString>(nixConfFiles.value(), ":"); + } + + // Use the paths specified by the XDG spec + std::vector files; + auto dirs = getConfigDirs(); + for (auto & dir : dirs) { + files.insert(files.end(), dir + "/nix/nix.conf"); + } + return files; +} + unsigned int Settings::getDefaultCores() { return std::max(1U, std::thread::hardware_concurrency()); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 40f350f0b..da95fd3ae 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -53,9 +53,12 @@ public: /* The directory where state is stored. */ Path nixStateDir; - /* The directory where configuration files are stored. */ + /* The directory where system configuration files are stored. */ Path nixConfDir; + /* A list of user configuration files to load. */ + std::vector nixUserConfFiles; + /* The directory where internal helper programs are stored. */ Path nixLibexecDir; @@ -378,6 +381,9 @@ void initPlugins(); void loadConfFile(); +// Used by the Settings constructor +std::vector getUserConfigFiles(); + extern const string nixVersion; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 1f85c7c46..815b1f288 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -16,6 +16,7 @@ #include #include #include +#include #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 diff --git a/tests/common.sh.in b/tests/common.sh.in index 15d7b1ef9..dd7e61822 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var export NIX_LOG_DIR=$TEST_ROOT/var/log/nix export NIX_STATE_DIR=$TEST_ROOT/var/nix export NIX_CONF_DIR=$TEST_ROOT/etc +unset NIX_USER_CONF_FILES export _NIX_TEST_SHARED=$TEST_ROOT/shared if [[ -n $NIX_STORE ]]; then export _NIX_TEST_NO_SANDBOX=1 @@ -21,6 +22,8 @@ export NIX_REMOTE=$NIX_REMOTE_ unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home export HOME=$TEST_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS unset XDG_CACHE_HOME mkdir -p $TEST_HOME diff --git a/tests/config.sh b/tests/config.sh new file mode 100644 index 000000000..8fa349f11 --- /dev/null +++ b/tests/config.sh @@ -0,0 +1,18 @@ +source common.sh + +# Test that files are loaded from XDG by default +export XDG_CONFIG_HOME=/tmp/home +export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2 +files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) +[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]] + +# Test that setting NIX_USER_CONF_FILES overrides all the default user config files +export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf +files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) +[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]] + +# Test that it's possible to load the config from a custom location +here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") +export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf +var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +[[ $var == https://example.com ]] diff --git a/tests/config/nix-with-substituters.conf b/tests/config/nix-with-substituters.conf new file mode 100644 index 000000000..90f359a6f --- /dev/null +++ b/tests/config/nix-with-substituters.conf @@ -0,0 +1,2 @@ +experimental-features = nix-command +substituters = https://example.com diff --git a/tests/local.mk b/tests/local.mk index 01fac4fcd..c5602773d 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,5 +1,6 @@ nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ + config.sh \ gc.sh \ gc-concurrent.sh \ gc-auto.sh \ From b865b5b40c13cfff32e1a0ab685baff1bef5ae5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 16 Apr 2020 12:32:07 +0200 Subject: [PATCH 053/118] pass Pos to forceValue to improve infinite recursion error --- src/libexpr/eval-inline.hh | 4 +-- src/libexpr/eval.cc | 6 ++-- src/libexpr/primops.cc | 64 +++++++++++++++++++------------------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index c27116e3b..942cda1ea 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -57,7 +57,7 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tAttrs) throwTypeError("value is %1% while a set was expected, at %2%", v, pos); } @@ -73,7 +73,7 @@ inline void EvalState::forceList(Value & v) inline void EvalState::forceList(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (!v.isList()) throwTypeError("value is %1% while a list was expected, at %2%", v, pos); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f963a42ca..3b4d9bfdc 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1502,7 +1502,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) bool EvalState::forceBool(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tBool) throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); return v.boolean; @@ -1517,7 +1517,7 @@ bool EvalState::isFunctor(Value & fun) void EvalState::forceFunction(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) throwTypeError("value is %1% while a function was expected, at %2%", v, pos); } @@ -1594,7 +1594,7 @@ std::optional EvalState::tryAttrsToString(const Pos & pos, Value & v, string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore, bool copyToStore) { - forceValue(v); + forceValue(v, pos); string s; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fc6c8296b..a00b93676 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -241,7 +241,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); string t; switch (args[0]->type) { case tInt: t = "int"; break; @@ -269,7 +269,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu /* Determine whether the argument is the null value. */ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tNull); } @@ -277,7 +277,7 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu /* Determine whether the argument is a function. */ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); bool res; switch (args[0]->type) { case tLambda: @@ -296,21 +296,21 @@ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, /* Determine whether the argument is an integer. */ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tInt); } /* Determine whether the argument is a float. */ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tFloat); } /* Determine whether the argument is a string. */ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tString); } @@ -318,14 +318,14 @@ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Va /* Determine whether the argument is a Boolean. */ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tBool); } /* Determine whether the argument is a path. */ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tPath); } @@ -382,7 +382,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar args[0]->attrs->find(state.symbols.create("operator")); if (op == args[0]->attrs->end()) throw EvalError(format("attribute 'operator' required, at %1%") % pos); - state.forceValue(*op->value); + state.forceValue(*op->value, pos); /* Construct the closure by applying the operator to element of `workSet', adding the result to `workSet', continuing until @@ -401,7 +401,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) throw EvalError(format("attribute 'key' required, at %1%") % pos); - state.forceValue(*key->value); + state.forceValue(*key->value, pos); if (!doneKeys.insert(key->value).second) continue; res.push_back(e); @@ -413,7 +413,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar /* Add the values returned by the operator to the work set. */ for (unsigned int n = 0; n < call.listSize(); ++n) { - state.forceValue(*call.listElems()[n]); + state.forceValue(*call.listElems()[n], pos); workSet.push_back(call.listElems()[n]); } } @@ -445,7 +445,7 @@ static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { try { - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { PathSet context; @@ -461,7 +461,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val { state.mkAttrs(v, 2); try { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); v.attrs->push_back(Attr(state.sValue, args[0])); mkBool(*state.allocAttr(v, state.symbols.create("success")), true); } catch (AssertionError & e) { @@ -483,8 +483,8 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu /* Evaluate the first argument, then return the second argument. */ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -494,7 +494,7 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValueDeep(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -503,12 +503,12 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val return the second expression. Useful for debugging. */ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type == tString) printError(format("trace: %1%") % args[0]->string.s); else printError(format("trace: %1%") % *args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -599,7 +599,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * try { if (ignoreNulls) { - state.forceValue(*i->value); + state.forceValue(*i->value, pos); if (i->value->type == tNull) continue; } @@ -1092,7 +1092,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args if (!context.empty()) throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type != tLambda) throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); @@ -1118,7 +1118,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value } else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { - state.forceValue(*attr.value); + state.forceValue(*attr.value, pos); filterFun = attr.value; } else if (n == "recursive") recursive = state.forceBool(*attr.value, *attr.pos); @@ -1189,7 +1189,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos); // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; - state.forceValue(*i->value); + state.forceValue(*i->value, pos); v = *i->value; } @@ -1219,7 +1219,7 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val /* Determine whether the argument is a set. */ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tAttrs); } @@ -1345,7 +1345,7 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va */ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type != tLambda) throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); @@ -1391,7 +1391,7 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va /* Determine whether the argument is a list. */ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->isList()); } @@ -1401,7 +1401,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); - state.forceValue(*list.listElems()[n]); + state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } @@ -1524,9 +1524,9 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); } - state.forceValue(v); + state.forceValue(v, pos); } else { - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } } @@ -1591,7 +1591,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value auto len = args[1]->listSize(); state.mkList(v, len); for (unsigned int n = 0; n < len; ++n) { - state.forceValue(*args[1]->listElems()[n]); + state.forceValue(*args[1]->listElems()[n], pos); v.listElems()[n] = args[1]->listElems()[n]; } @@ -1626,7 +1626,7 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V for (unsigned int n = 0; n < len; ++n) { auto vElem = args[1]->listElems()[n]; - state.forceValue(*vElem); + state.forceValue(*vElem, pos); Value res; state.callFunction(*args[0], *vElem, res, pos); if (state.forceBool(res, pos)) @@ -1757,8 +1757,8 @@ static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Valu static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); CompareValues comp; mkBool(v, comp(args[0], args[1])); } From 1290411c2d0c62dd1761485f55292dc944eae55d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Apr 2020 14:00:12 +0200 Subject: [PATCH 054/118] fetchMercurial: Use inputFromAttrs() --- src/libexpr/primops/fetchMercurial.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index f18351646..0a1ba49d5 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -54,15 +54,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar if (evalSettings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - auto parsedUrl = parseURL( - url.find("://") != std::string::npos - ? "hg+" + url - : "hg+file://" + url); - if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); - if (ref) parsedUrl.query.insert_or_assign("ref", *ref); - // FIXME: use name - auto input = fetchers::inputFromURL(parsedUrl); + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "hg"); + attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + if (ref) attrs.insert_or_assign("ref", *ref); + if (rev) attrs.insert_or_assign("rev", rev->gitRev()); + auto input = fetchers::inputFromAttrs(attrs); + // FIXME: use name auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); From b3e5eea4a91400fb2a12aba4b07a94d03ba54605 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 16:28:07 +0200 Subject: [PATCH 055/118] Add function to allocate a Value in traceable memory --- src/libexpr/eval.cc | 8 ++++++++ src/libexpr/value.hh | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f963a42ca..2f5b1db47 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -22,6 +22,8 @@ #if HAVE_BOEHMGC +#define GC_INCLUDE_NEW + #include #include @@ -56,6 +58,12 @@ static char * dupStringWithLen(const char * s, size_t size) } +RootValue allocRootValue(Value * v) +{ + return std::allocate_shared(traceable_allocator(), v); +} + + static void printValue(std::ostream & str, std::set & active, const Value & v) { checkInterrupt(); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 689373873..9cf2bf06a 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -261,4 +261,9 @@ typedef std::map ValueMap; #endif +/* A value allocated in traceable memory. */ +typedef std::shared_ptr RootValue; + +RootValue allocRootValue(Value * v); + } From 10e17eaa5802a3c368eee8408821828072e76ba7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 17:24:28 +0200 Subject: [PATCH 056/118] ValueMap, VectorVector: Use traceable_allocator We want to *trace* the 'Value *' arrays, not garbage-collect them! Otherwise the vectors/maps can end up pointing to nowhere. Fixes #3377. Closes #3384. --- src/libexpr/value.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 9cf2bf06a..71025824e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -253,8 +253,8 @@ void mkPath(Value & v, const char * s); #if HAVE_BOEHMGC -typedef std::vector > ValueVector; -typedef std::map, gc_allocator > > ValueMap; +typedef std::vector > ValueVector; +typedef std::map, traceable_allocator > > ValueMap; #else typedef std::vector ValueVector; typedef std::map ValueMap; From 9f46f54de4e55267df492456fc0393f74616366b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 17:28:32 +0200 Subject: [PATCH 057/118] JSONSax: Use a RootValue More #3377. --- src/libexpr/json-to-value.cc | 63 ++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 1fdce1983..76e1a26bf 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -4,7 +4,6 @@ #include using json = nlohmann::json; -using std::unique_ptr; namespace nix { @@ -13,69 +12,69 @@ namespace nix { class JSONSax : nlohmann::json_sax { class JSONState { protected: - unique_ptr parent; - Value * v; + std::unique_ptr parent; + RootValue v; public: - virtual unique_ptr resolve(EvalState &) + virtual std::unique_ptr resolve(EvalState &) { throw std::logic_error("tried to close toplevel json parser state"); - }; - explicit JSONState(unique_ptr&& p) : parent(std::move(p)), v(nullptr) {}; - explicit JSONState(Value* v) : v(v) {}; - JSONState(JSONState& p) = delete; - Value& value(EvalState & state) + } + explicit JSONState(std::unique_ptr && p) : parent(std::move(p)) {} + explicit JSONState(Value * v) : v(allocRootValue(v)) {} + JSONState(JSONState & p) = delete; + Value & value(EvalState & state) { - if (v == nullptr) - v = state.allocValue(); - return *v; - }; - virtual ~JSONState() {}; - virtual void add() {}; + if (!v) + v = allocRootValue(state.allocValue()); + return **v; + } + virtual ~JSONState() {} + virtual void add() {} }; class JSONObjectState : public JSONState { using JSONState::JSONState; - ValueMap attrs = ValueMap(); - virtual unique_ptr resolve(EvalState & state) override + ValueMap attrs; + std::unique_ptr resolve(EvalState & state) override { - Value& v = parent->value(state); + Value & v = parent->value(state); state.mkAttrs(v, attrs.size()); for (auto & i : attrs) v.attrs->push_back(Attr(i.first, i.second)); return std::move(parent); } - virtual void add() override { v = nullptr; }; + void add() override { v = nullptr; } public: - void key(string_t& name, EvalState & state) + void key(string_t & name, EvalState & state) { - attrs[state.symbols.create(name)] = &value(state); + attrs.insert_or_assign(state.symbols.create(name), &value(state)); } }; class JSONListState : public JSONState { - ValueVector values = ValueVector(); - virtual unique_ptr resolve(EvalState & state) override + ValueVector values; + std::unique_ptr resolve(EvalState & state) override { - Value& v = parent->value(state); + Value & v = parent->value(state); state.mkList(v, values.size()); for (size_t n = 0; n < values.size(); ++n) { v.listElems()[n] = values[n]; } return std::move(parent); } - virtual void add() override { - values.push_back(v); + void add() override { + values.push_back(*v); v = nullptr; - }; + } public: - JSONListState(unique_ptr&& p, std::size_t reserve) : JSONState(std::move(p)) + JSONListState(std::unique_ptr && p, std::size_t reserve) : JSONState(std::move(p)) { values.reserve(reserve); } }; EvalState & state; - unique_ptr rs; + std::unique_ptr rs; template inline bool handle_value(T f, Args... args) { @@ -107,12 +106,12 @@ public: return handle_value(mkInt, val); } - bool number_float(number_float_t val, const string_t& s) + bool number_float(number_float_t val, const string_t & s) { return handle_value(mkFloat, val); } - bool string(string_t& val) + bool string(string_t & val) { return handle_value(mkString, val.c_str()); } @@ -123,7 +122,7 @@ public: return true; } - bool key(string_t& name) + bool key(string_t & name) { dynamic_cast(rs.get())->key(name, state); return true; From fcd048a526bd239fa615457e77d61d69d679bf03 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 16:54:34 +0200 Subject: [PATCH 058/118] Use RootValue --- src/libexpr/primops.cc | 10 +++++----- src/nix/command.hh | 2 +- src/nix/installables.cc | 21 ++++++++------------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fc6c8296b..a3f2b92ce 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -121,16 +121,16 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args } w.attrs->sort(); - static Value * fun = nullptr; + static RootValue fun; if (!fun) { - fun = state.allocValue(); + fun = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , "/"), *fun); + , "/"), **fun); } - state.forceFunction(*fun, pos); - mkApp(v, *fun, w); + state.forceFunction(**fun, pos); + mkApp(v, **fun, w); state.forceAttrs(v, pos); } else { state.forceAttrs(*args[0]); diff --git a/src/nix/command.hh b/src/nix/command.hh index 2c2303208..bf43d950f 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -41,7 +41,7 @@ private: std::shared_ptr evalState; - std::shared_ptr vSourceExpr; + RootValue vSourceExpr; }; enum RealiseMode { Build, NoBuild, DryRun }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 902383bff..1d70ad3d5 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -8,8 +8,6 @@ #include "store-api.hh" #include "shared.hh" -#include - #include namespace nix { @@ -27,17 +25,14 @@ SourceExprCommand::SourceExprCommand() Value * SourceExprCommand::getSourceExpr(EvalState & state) { - if (vSourceExpr) return vSourceExpr.get(); + if (vSourceExpr) return *vSourceExpr; auto sToplevel = state.symbols.create("_toplevel"); - // Allocate the vSourceExpr Value as uncollectable. Boehm GC doesn't - // consider the member variable "alive" during execution causing it to be - // GC'ed in the middle of evaluation. - vSourceExpr = std::allocate_shared(traceable_allocator()); + vSourceExpr = allocRootValue(state.allocValue()); if (file != "") - state.evalFile(lookupFileArg(state, file), *vSourceExpr); + state.evalFile(lookupFileArg(state, file), **vSourceExpr); else { @@ -45,9 +40,9 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) auto searchPath = state.getSearchPath(); - state.mkAttrs(*vSourceExpr, 1024); + state.mkAttrs(**vSourceExpr, 1024); - mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); + mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true); std::unordered_set seen; @@ -58,7 +53,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); Value * v2 = state.allocValue(); mkApp(*v2, *v1, mkString(*state.allocValue(), name)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), + mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)), state.getBuiltin("import"), *v2); }; @@ -72,10 +67,10 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) } else addEntry(i.first); - vSourceExpr->attrs->sort(); + (*vSourceExpr)->attrs->sort(); } - return vSourceExpr.get(); + return *vSourceExpr; } ref SourceExprCommand::getEvalState() From 67a5941472ab56a798556ee2d5afc62fc38a799a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 13:12:58 +0200 Subject: [PATCH 059/118] Logger: Add method for writing to stdout Usually this just writes to stdout, but for ProgressBar, we need to clear the current line, write the line to stdout, and then redraw the progress bar. (cherry picked from commit 696c026006a6ac46adc990ed5cb0f31535bac076) --- src/libutil/logging.cc | 6 ++++++ src/libutil/logging.hh | 10 ++++++++++ src/nix/progress-bar.cc | 13 +++++++++++++ 3 files changed, 29 insertions(+) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index bb437cf1c..3cc4ef8f1 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,6 +3,7 @@ #include #include +#include namespace nix { @@ -24,6 +25,11 @@ void Logger::warn(const std::string & msg) log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg); } +void Logger::writeToStdout(std::string_view s) +{ + std::cout << s << "\n"; +} + class SimpleLogger : public Logger { public: diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 108b5dcb0..18c24d508 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -78,6 +78,16 @@ public: virtual void stopActivity(ActivityId act) { }; virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; + + virtual void writeToStdout(std::string_view s); + + template + inline void stdout(const std::string & fs, const Args & ... args) + { + boost::format f(fs); + formatHelper(f, args...); + writeToStdout(f.str()); + } }; ActivityId getCurActivity(); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index adc9b9a5d..8e7ba95a3 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -7,6 +7,7 @@ #include #include #include +#include namespace nix { @@ -442,6 +443,18 @@ public: return res; } + + void writeToStdout(std::string_view s) override + { + auto state(state_.lock()); + if (state->active) { + std::cerr << "\r\e[K"; + Logger::writeToStdout(s); + draw(*state); + } else { + Logger::writeToStdout(s); + } + } }; void startProgressBar(bool printBuildLogs) From efaffaa9d1de38efecb718aa7a99ba1f2e342ade Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 13:46:37 +0200 Subject: [PATCH 060/118] Use Logger::stdout() (cherry picked from commit 8f41847394524fcac40d3b5620139ca7e94a18e3) --- src/nix/add-to-store.cc | 2 +- src/nix/eval.cc | 5 ++--- src/nix/hash.cc | 5 ++--- src/nix/ls.cc | 10 ++++------ src/nix/show-config.cc | 2 +- src/nix/why-depends.cc | 2 +- 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 139db3657..ed02227db 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -50,7 +50,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand if (!dryRun) store->addToStore(info, sink.s); - std::cout << fmt("%s\n", store->printStorePath(info.path)); + logger->stdout("%s", store->printStorePath(info.path)); } }; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 6398fc58e..86a1e8b68 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -55,16 +55,15 @@ struct CmdEval : MixJSON, InstallableCommand auto v = installable->toValue(*state).first; PathSet context; - stopProgressBar(); - if (raw) { + stopProgressBar(); std::cout << state->coerceToString(noPos, *v, context); } else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, jsonOut, context); } else { state->forceValueDeep(*v); - std::cout << *v << "\n"; + logger->stdout("%s", *v); } } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0cc523f50..01628cf6c 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -60,8 +60,7 @@ struct CmdHash : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - h.to_string(base, base == SRI); + logger->stdout(h.to_string(base, base == SRI)); } } }; @@ -95,7 +94,7 @@ struct CmdToBase : Command void run() override { for (auto s : args) - std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI)); + logger->stdout(Hash(s, ht).to_string(base, base == SRI)); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 3ef1f2750..8590199d7 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -34,16 +34,14 @@ struct MixLs : virtual Args, MixJSON (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - std::cout << - (format("%s %20d %s") % tp % st.fileSize % relPath); + auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) - std::cout << " -> " << accessor->readLink(curPath) - ; - std::cout << "\n"; + line += " -> " + accessor->readLink(curPath); + logger->stdout(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - std::cout << relPath << "\n"; + logger->stdout(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 87544f937..6104b10bc 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -23,7 +23,7 @@ struct CmdShowConfig : Command, MixJSON std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) - std::cout << s.first + " = " + s.second.value + "\n"; + logger->stdout("%s = %s", s.first, s.second.value); } } }; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index d3b7a674a..f9acc7f13 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -149,7 +149,7 @@ struct CmdWhyDepends : SourceExprCommand auto pathS = store->printStorePath(node.path); assert(node.dist != inf); - std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", + logger->stdout("%s%s%s%s" ANSI_NORMAL, firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", From c0d940978a66841df638038e3ca85501bcee5734 Mon Sep 17 00:00:00 2001 From: Dustin DeWeese Date: Mon, 20 Apr 2020 17:32:50 -0700 Subject: [PATCH 061/118] Replace select() with poll() to allow waiting on more than FD_SETSIZE fds --- src/libstore/build.cc | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index b4207e1b8..572634765 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -43,6 +42,7 @@ #include #include #include +#include #include #include @@ -4789,8 +4789,7 @@ void Worker::waitForInput() terminated. */ bool useTimeout = false; - struct timeval timeout; - timeout.tv_usec = 0; + long timeout = 0; auto before = steady_time_point::clock::now(); /* If we're monitoring for silence on stdout/stderr, or if there @@ -4808,7 +4807,7 @@ void Worker::waitForInput() nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } if (nearest != steady_time_point::max()) { - timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); + timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); useTimeout = true; } @@ -4819,30 +4818,28 @@ void Worker::waitForInput() if (lastWokenUp == steady_time_point::min()) printError("waiting for locks or build slots..."); if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max(1L, + timeout = std::max(1L, (long) std::chrono::duration_cast( lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); } else lastWokenUp = steady_time_point::min(); if (useTimeout) - vomit("sleeping %d seconds", timeout.tv_sec); + vomit("sleeping %d seconds", timeout); /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ - fd_set fds; - FD_ZERO(&fds); - int fdMax = 0; + std::vector pollStatus; + std::map fdToPollStatus; for (auto & i : children) { for (auto & j : i.fds) { - if (j >= FD_SETSIZE) - throw Error("reached FD_SETSIZE limit"); - FD_SET(j, &fds); - if (j >= fdMax) fdMax = j + 1; + pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); + fdToPollStatus[j] = pollStatus.size() - 1; } } - if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { + if (poll(pollStatus.data(), pollStatus.size(), + useTimeout ? timeout * 1000 : -1) == -1) { if (errno == EINTR) return; throw SysError("waiting for input"); } @@ -4863,7 +4860,7 @@ void Worker::waitForInput() set fds2(j->fds); std::vector buffer(4096); for (auto & k : fds2) { - if (FD_ISSET(k, &fds)) { + if (pollStatus.at(fdToPollStatus.at(k)).revents) { ssize_t rd = read(k, buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? From 2ea4d45449ea676e27cc678145ef39af7ac05ca8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Apr 2020 10:15:32 +0200 Subject: [PATCH 062/118] Path fetcher: Fix store path name (cherry picked from commit c7af247beacd418e6f2c4d33dffc35299101cd12) --- src/libfetchers/path.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 7c7e20f4e..ba2cc192e 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -72,7 +72,7 @@ struct PathInput : Input if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) // FIXME: try to substitute storePath. - storePath = store->addToStore("name", path); + storePath = store->addToStore("source", path); return { From 7114f088fceb366e9ff0fb60e2970a11861d11fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Apr 2020 14:54:16 +0200 Subject: [PATCH 063/118] Don't install error-demo --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f6b4a3101..e3057c36c 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,7 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ tests/local.mk \ - tests/plugins/local.mk \ - src/error-demo/local.mk + tests/plugins/local.mk -include Makefile.config From c9d0cf7e02d4b3af3c027d7d74d5caa3c8963d26 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Apr 2020 15:03:41 +0200 Subject: [PATCH 064/118] Don't include error.hh in util.hh to prevent header bloat --- src/libutil/util.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8e24ef968..d7ee62bcc 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "logging.hh" #include "ansicolor.hh" -#include "error.hh" #include #include From f59404e1a619934b9430b3a5bf352dbfffef2c31 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sat, 25 Apr 2020 13:18:39 +0200 Subject: [PATCH 065/118] nix-env: refactor uninstallDerivations Reduces the number of store queries it performs. Also prints a warning if any of the selectors did not match any installed derivations. UX Caveats: - Will print a warning that nothing matched if a previous selector already removed the path - Will not do anything if no selectors were provided (no change from before). Fixes #3531 --- src/nix-env/nix-env.cc | 45 ++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 1a2bb42a3..6c9cd9548 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -718,28 +718,39 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, while (true) { string lockToken = optimisticLockProfile(profile); - DrvInfos installedElems = queryInstalled(*globals.state, profile); - DrvInfos newElems; + DrvInfos workingElems = queryInstalled(*globals.state, profile); - for (auto & i : installedElems) { - DrvName drvName(i.queryName()); - bool found = false; - for (auto & j : selectors) - /* !!! the repeated calls to followLinksToStorePath() - are expensive, should pre-compute them. */ - if ((isPath(j) && globals.state->store->parseStorePath(i.queryOutPath()) == globals.state->store->followLinksToStorePath(j)) - || DrvName(j).matches(drvName)) - { - printInfo("uninstalling '%s'", i.queryName()); - found = true; - break; - } - if (!found) newElems.push_back(i); + for (auto & selector : selectors) { + DrvInfos::iterator split = workingElems.begin(); + if (isPath(selector)) { + StorePath selectorStorePath = globals.state->store->followLinksToStorePath(selector); + split = std::partition( + workingElems.begin(), workingElems.end(), + [&selectorStorePath, globals](auto &elem) { + return selectorStorePath != globals.state->store->parseStorePath(elem.queryOutPath()); + } + ); + } else { + DrvName selectorName(selector); + split = std::partition( + workingElems.begin(), workingElems.end(), + [&selectorName](auto &elem){ + DrvName elemName(elem.queryName()); + return !selectorName.matches(elemName); + } + ); + } + if (split == workingElems.end()) + warn("Selector '%s' matched no installed paths", selector); + for (auto removedElem = split; removedElem != workingElems.end(); removedElem++) { + printInfo("uninstalling '%s'", removedElem->queryName()); + } + workingElems.erase(split, workingElems.end()); } if (globals.dryRun) return; - if (createUserEnv(*globals.state, newElems, + if (createUserEnv(*globals.state, workingElems, profile, settings.envKeepDerivations, lockToken)) break; } } From a3bc695e7dd60a0f1860942ba0343572b85e05e4 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Mon, 27 Apr 2020 11:12:54 -0600 Subject: [PATCH 066/118] Set GCROOT to store path to prevent garbage collection --- src/nix/shell.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 71e640667..bd07104cf 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -200,13 +200,15 @@ struct Common : InstallableCommand, MixProfile } } - BuildEnvironment getBuildEnvironment(ref store) + std::pair getBuildEnvironment(ref store) { auto shellOutPath = getShellOutPath(store); + auto strPath = store->printStorePath(shellOutPath); + updateProfile(shellOutPath); - return readEnvironment(store->printStorePath(shellOutPath)); + return {readEnvironment(strPath), strPath}; } }; @@ -253,7 +255,7 @@ struct CmdDevShell : Common, MixEnvironment void run(ref store) override { - auto buildEnvironment = getBuildEnvironment(store); + auto [buildEnvironment, gcroot] = getBuildEnvironment(store); auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); @@ -276,6 +278,7 @@ struct CmdDevShell : Common, MixEnvironment auto shell = getEnv("SHELL").value_or("bash"); setEnviron(); + setenv("GCROOT", gcroot.data(), 1); auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; @@ -307,7 +310,7 @@ struct CmdPrintDevEnv : Common void run(ref store) override { - auto buildEnvironment = getBuildEnvironment(store); + auto buildEnvironment = getBuildEnvironment(store).first; stopProgressBar(); From 9e95b95a5da59252d140fb61cb1177645935dcd6 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Mon, 27 Apr 2020 13:18:26 -0600 Subject: [PATCH 067/118] comment --- src/nix/shell.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index bd07104cf..fa8fe2e9b 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -278,6 +278,7 @@ struct CmdDevShell : Common, MixEnvironment auto shell = getEnv("SHELL").value_or("bash"); setEnviron(); + // prevent garbage collection until shell exits setenv("GCROOT", gcroot.data(), 1); auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; From c05e20daa1abb3446e378331697938b78af2b3d7 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Mon, 27 Apr 2020 14:15:15 +0000 Subject: [PATCH 068/118] Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross Thanks-to: Puck Meerburg Tested-by: Puck Meerburg Reviewed-by: Puck Meerburg --- src/libutil/util.cc | 54 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 332c1c43a..615a7656c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -268,16 +268,13 @@ bool isLink(const Path & path) } -DirEntries readDirectory(const Path & path) +DirEntries readDirectory(DIR *dir, const Path & path) { DirEntries entries; entries.reserve(64); - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + while (errno = 0, dirent = readdir(dir)) { /* sic */ checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; @@ -294,6 +291,14 @@ DirEntries readDirectory(const Path & path) return entries; } +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % path); + + return readDirectory(dir.get(), path); +} + unsigned char getFileType(const Path & path) { @@ -389,12 +394,14 @@ void writeLine(int fd, string s) } -static void _deletePath(const Path & path, unsigned long long & bytesFreed) +static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed) { checkInterrupt(); + string name(baseNameOf(path)); + struct stat st; - if (lstat(path.c_str(), &st) == -1) { + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { if (errno == ENOENT) return; throw SysError(format("getting status of '%1%'") % path); } @@ -406,20 +413,45 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) /* Make the directory accessible. */ const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1) + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) throw SysError(format("chmod '%1%'") % path); } - for (auto & i : readDirectory(path)) - _deletePath(path + "/" + i.name, bytesFreed); + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (!fd) + throw SysError(format("opening directory '%1%'") % path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError(format("opening directory '%1%'") % path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); } - if (remove(path.c_str()) == -1) { + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { if (errno == ENOENT) return; throw SysError(format("cannot unlink '%1%'") % path); } } +static void _deletePath(const Path & path, unsigned long long & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY)); + if (!dirfd) { + // This really shouldn't fail silently, but it's left this way + // for backwards compatibility. + if (errno == ENOENT) return; + + throw SysError(format("opening directory '%1%'") % path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + void deletePath(const Path & path) { From 52a3ca823dd8aca8da32a198be2b1238cb321e6c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Apr 2020 17:56:01 +0200 Subject: [PATCH 069/118] Tweak warning message --- src/nix-env/nix-env.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 6c9cd9548..d62febaff 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -741,7 +741,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, ); } if (split == workingElems.end()) - warn("Selector '%s' matched no installed paths", selector); + warn("selector '%s' matched no installed derivations", selector); for (auto removedElem = split; removedElem != workingElems.end(); removedElem++) { printInfo("uninstalling '%s'", removedElem->queryName()); } From 6d40fe573c859725cac6eb2c884ecf84965c8853 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Tue, 28 Apr 2020 11:18:54 -0600 Subject: [PATCH 070/118] rename to NIX_GCROOT --- src/nix/shell.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index fa8fe2e9b..fbfe24bbe 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -279,7 +279,7 @@ struct CmdDevShell : Common, MixEnvironment setEnviron(); // prevent garbage collection until shell exits - setenv("GCROOT", gcroot.data(), 1); + setenv("NIX_GCROOT", gcroot.data(), 1); auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; From 5a34a473dd14200ffeff496f0e8e4518f7992765 Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Wed, 29 Apr 2020 13:10:03 +0200 Subject: [PATCH 071/118] builtins.readFile: do not truncate content This closes #3026 by allowing `builtins.readFile` to read a file with a wrongly reported file size, for example, files in `/proc` may report a file size of 0. Reading file in `/proc` is not a good enough motivation, however I do think it just makes nix more robust by allowing more file to be read. Especially, I do considerer the previous behavior to be dangerous because nix was previously reading truncated files. Examples of file system which incorrectly report file size may be network file system or dynamic file system (for performance reason, a dynamic file system such as FUSE may generate the content of the file on demand). ``` nix-repl> builtins.readFile "/proc/version" "" ``` With this commit: ``` nix-repl> builtins.readFile "/proc/version" "Linux version 5.6.7 (nixbld@localhost) (gcc version 9.3.0 (GCC)) #1-NixOS SMP Thu Apr 23 08:38:27 UTC 2020\n" ``` Here is a summary of the behavior changes: - If the reported size is smaller, previous implementation was silently returning a truncated file content. The new implementation is returning the correct file content. - If a file had a bigger reported file size, previous implementation was failing with an exception, but the new implementation is returning the correct file content. This change of behavior is coherent with this pull request. Open questions - The behavior is unchanged for correctly reported file size, however performances may vary because it uses the more complex sink interface. Considering that sink is used a lot, I don't think this impacts the performance a lot. - `builtins.readFile` on an infinite file, such as `/dev/random` may fill the memory. - it does not support adding file to store, such as `${/proc/version}`. --- src/libutil/util.cc | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 615a7656c..d001bc3b7 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -312,14 +312,7 @@ unsigned char getFileType(const Path & path) string readFile(int fd) { - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - std::vector buf(st.st_size); - readFull(fd, buf.data(), st.st_size); - - return string((char *) buf.data(), st.st_size); + return drainFD(fd, true); } From ca93b26db6d2d73e702ea8ecdd0a98f17ace2c7b Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Wed, 29 Apr 2020 14:39:37 +0200 Subject: [PATCH 072/118] Only call grantpt on MacOS systems The commit 3cc1125595d97b4ab7369e37e4ad22f4cfecb8b2 adds a `grantpt` call on the builder pseudo terminal fd. This call is actually only required for MacOS, but it however requires a RW access to /dev/pts which is only RO bindmounted in the Bazel Linux sandbox. So, Nix can not be actually run in the Bazel Linux sandbox for unneeded reasons. --- src/libstore/build.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 572634765..147093fae 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2250,10 +2250,13 @@ void DerivationGoal::startBuilder() if (chown(slaveName.c_str(), buildUser->getUID(), 0)) throw SysError("changing owner of pseudoterminal slave"); - } else { + } +#if __APPLE__ + else { if (grantpt(builderOut.readSide.get())) throw SysError("granting access to pseudoterminal slave"); } +#endif #if 0 // Mount the pt in the sandbox so that the "tty" command works. From 7afcb5af988eb0ce73c9916b809f8528dfb14c0f Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Wed, 29 Apr 2020 18:42:19 +0200 Subject: [PATCH 073/118] Remove the `drain` argument from `readFile` Now it is always `drain` (see previous commit). --- src/libstore/gc.cc | 4 ++-- src/libutil/serialise.hh | 3 +++ src/libutil/util.cc | 4 ++-- src/libutil/util.hh | 2 +- src/nix/ls.cc | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 6bab1e37c..3cd35c945 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -419,7 +419,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) try { auto mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); + auto mapLines = tokenizeString>(readFile(mapFile), "\n"); for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) @@ -427,7 +427,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } auto envFile = fmt("/proc/%s/environ", ent->d_name); - auto envString = readFile(envFile, true); + auto envString = readFile(envFile); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) unchecked[i->str()].emplace(envFile); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5780c93a6..a04118512 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -148,6 +148,9 @@ struct StringSink : Sink { ref s; StringSink() : s(make_ref()) { }; + explicit StringSink(const size_t reservedSize) : s(make_ref()) { + s->reserve(reservedSize); + }; StringSink(ref s) : s(s) { }; void operator () (const unsigned char * data, size_t len) override; }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index d001bc3b7..bb4af747b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -316,12 +316,12 @@ string readFile(int fd) } -string readFile(const Path & path, bool drain) +string readFile(const Path & path) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) throw SysError(format("opening file '%1%'") % path); - return drain ? drainFD(fd.get()) : readFile(fd.get()); + return readFile(fd.get()); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d7ee62bcc..32ef9a79a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -103,7 +103,7 @@ unsigned char getFileType(const Path & path); /* Read the contents of a file into a string. */ string readFile(int fd); -string readFile(const Path & path, bool drain = false); +string readFile(const Path & path); void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 8590199d7..ee308a6ec 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -136,7 +136,7 @@ struct CmdLsNar : Command, MixLs void run() override { - list(makeNarAccessor(make_ref(readFile(narPath, true)))); + list(makeNarAccessor(make_ref(readFile(narPath)))); } }; From 2e5be2a7495ac0b204454c74664e590a38d039d3 Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Wed, 29 Apr 2020 18:44:01 +0200 Subject: [PATCH 074/118] StringSink pre allocate When used with `readFile`, we have a pretty good heuristic of the file size, so `reserve` this in the `string`. This will save some allocation / copy when the string is growing. --- src/libutil/util.cc | 10 +++++++--- src/libutil/util.hh | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index bb4af747b..71db92d77 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -312,7 +312,11 @@ unsigned char getFileType(const Path & path) string readFile(int fd) { - return drainFD(fd, true); + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); } @@ -658,9 +662,9 @@ void writeFull(int fd, const string & s, bool allowInterrupts) } -string drainFD(int fd, bool block) +string drainFD(int fd, bool block, const size_t reserveSize) { - StringSink sink; + StringSink sink(reserveSize); drainFD(fd, sink, block); return std::move(*sink.s); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 32ef9a79a..1b263abcc 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -162,7 +162,7 @@ MakeError(EndOfFile, Error); /* Read a file descriptor until EOF occurs. */ -string drainFD(int fd, bool block = true); +string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); From d1229859c2612587bfd04111bae6584c0c9df051 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 29 Apr 2020 22:48:29 +0200 Subject: [PATCH 075/118] Fix displaying error-position in `builtins.fetch{Tree,Tarball}` Without dereferencing this pointer, you'd get an error like this: ``` error: unsupported argument 'abc' to 'fetchTarball', at 0x13627e8 ``` --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 43c58485a..c5a0d9886 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -108,7 +108,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to '%s', at %s", - attr.name, who, attr.pos); + attr.name, who, *attr.pos); } if (!url) From 2fcfc6c2c6ff144131d669ebfb2caa63c8b41417 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2020 13:05:29 +0200 Subject: [PATCH 076/118] nix dev-shell: Refactor script for getting the environment --- src/nix/get-env.sh | 8 ++++++++ src/nix/local.mk | 2 ++ src/nix/shell.cc | 18 ++++++++---------- 3 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 src/nix/get-env.sh diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh new file mode 100644 index 000000000..782f97b75 --- /dev/null +++ b/src/nix/get-env.sh @@ -0,0 +1,8 @@ +set -e +export IN_NIX_SHELL=impure +export dontAddDisableDepTrack=1 +if [[ -n $stdenv ]]; then + source $stdenv/setup +fi +export > $out +set >> $out diff --git a/src/nix/local.mk b/src/nix/local.mk index 033675e89..15cef55bb 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -27,3 +27,5 @@ $(foreach name, \ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh + +src/nix/shell.cc: src/nix/get-env.sh.gen.hh diff --git a/src/nix/shell.cc b/src/nix/shell.cc index fbfe24bbe..c8bf35a13 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -83,6 +83,10 @@ BuildEnvironment readEnvironment(const Path & path) return res; } +const static std::string getEnvSh = + #include "get-env.sh.gen.hh" + ; + /* Given an existing derivation, return the shell environment as initialised by stdenv's setup script. We do this by building a modified derivation with the same dependencies and nearly the same @@ -94,16 +98,9 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) if (builder != "bash") throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); - drv.args = { - "-c", - "set -e; " - "export IN_NIX_SHELL=impure; " - "export dontAddDisableDepTrack=1; " - "if [[ -n $stdenv ]]; then " - " source $stdenv/setup; " - "fi; " - "export > $out; " - "set >> $out "}; + auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); + + drv.args = {store->printStorePath(getEnvShPath)}; /* Remove derivation checks. */ drv.env.erase("allowedReferences"); @@ -120,6 +117,7 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) drv.env.erase(output.first); drv.env["out"] = ""; drv.env["outputs"] = "out"; + drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", "")); From efe6c186eabeb85830be7bef9aea4dd7eb2357e7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2020 14:39:26 +0200 Subject: [PATCH 077/118] nix dev-shell: Support structured attrs Tested against https://github.com/NixOS/nixpkgs/pull/72074. Fixes #3540. --- src/nix/get-env.sh | 1 + src/nix/shell.cc | 45 ++++++++++++++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 782f97b75..a25ec43a9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,4 +1,5 @@ set -e +if [ -e .attrs.sh ]; then source .attrs.sh; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 if [[ -n $stdenv ]]; then diff --git a/src/nix/shell.cc b/src/nix/shell.cc index c8bf35a13..f0ffceaa7 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -13,7 +13,8 @@ using namespace nix; struct Var { - bool exported; + bool exported = true; + bool associative = false; std::string value; // quoted string or array }; @@ -48,11 +49,17 @@ BuildEnvironment readEnvironment(const Path & path) static std::string quotedStringRegex = R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; - static std::string arrayRegex = - R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re"; + static std::string indexedArrayRegex = + R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re"; static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n"); + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); + + /* Note: we distinguish between an indexed and associative array + using the space before the closing parenthesis. Will + undoubtedly regret this some day. */ + static std::regex assocArrayRegex( + "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); static std::regex functionRegex( "^" + varNameRegex + " \\(\\) *\n"); @@ -68,7 +75,12 @@ BuildEnvironment readEnvironment(const Path & path) else if (std::regex_search(pos, file.cend(), match, varRegex)) { pos = match[0].second; - res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }}); + res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); + } + + else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); } else if (std::regex_search(pos, file.cend(), match, functionRegex)) { @@ -92,8 +104,10 @@ const static std::string getEnvSh = modified derivation with the same dependencies and nearly the same initial environment variables, that just writes the resulting environment to a file and exits. */ -StorePath getDerivationEnvironment(ref store, Derivation drv) +StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) { + auto drv = store->derivationFromPath(drvPath); + auto builder = baseNameOf(drv.builder); if (builder != "bash") throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); @@ -108,11 +122,12 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) drv.env.erase("disallowedReferences"); drv.env.erase("disallowedRequisites"); - // FIXME: handle structured attrs - /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ - auto drvName = drv.env["name"] + "-env"; + auto drvName = std::string(drvPath.name()); + assert(hasSuffix(drvName, ".drv")); + drvName.resize(drvName.size() - 4); + drvName += "-env"; for (auto & output : drv.outputs) drv.env.erase(output.first); drv.env["out"] = ""; @@ -161,9 +176,13 @@ struct Common : InstallableCommand, MixProfile for (auto & i : buildEnvironment.env) { if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { - out << fmt("%s=%s\n", i.first, i.second.value); - if (i.second.exported) - out << fmt("export %s\n", i.first); + if (i.second.associative) + out << fmt("declare -A %s=(%s)\n", i.first, i.second.value); + else { + out << fmt("%s=%s\n", i.first, i.second.value); + if (i.second.exported) + out << fmt("export %s\n", i.first); + } } } @@ -194,7 +213,7 @@ struct Common : InstallableCommand, MixProfile auto & drvPath = *drvs.begin(); - return getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + return getDerivationEnvironment(store, drvPath); } } From 0135fd6ec4dcd15771008b867f28047e5c2bb605 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2020 14:46:51 +0200 Subject: [PATCH 078/118] nix dev-shell: Unset shellHook This avoids inheriting the caller's shellHook, which can happen when running a dev-shell inside a dev-shell. --- src/nix/shell.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index f0ffceaa7..9c45a935d 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -172,6 +172,8 @@ struct Common : InstallableCommand, MixProfile void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) { + out << "unset shellHook\n"; + out << "nix_saved_PATH=\"$PATH\"\n"; for (auto & i : buildEnvironment.env) { From a15f918cbac51dee2e82b4cb0535e7ae5f4ff587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 30 Apr 2020 14:57:40 +0200 Subject: [PATCH 079/118] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 26 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..5c9e1d9cc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,26 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**`nix-env --version` output: ** + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..392ed30c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: improvement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 625868b33dab37c0750081843be8091e22808376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 30 Apr 2020 15:01:23 +0200 Subject: [PATCH 080/118] Update issue templates --- .github/ISSUE_TEMPLATE.md | 27 --------------------------- .github/ISSUE_TEMPLATE/bug_report.md | 10 ++++++++-- 2 files changed, 8 insertions(+), 29 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 3372b1f03..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,27 +0,0 @@ - diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 5c9e1d9cc..e6d346bc1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -8,19 +8,25 @@ assignees: '' --- **Describe the bug** + A clear and concise description of what the bug is. +If you have a problem with a specific package or NixOS, +you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. + **Steps To Reproduce** -Steps to reproduce the behavior: + 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Expected behavior** + A clear and concise description of what you expected to happen. -**`nix-env --version` output: ** +**`nix-env --version` output** **Additional context** + Add any other context about the problem here. From 533343628dcd4cacc8ff74d429372bfbd860a683 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Thu, 30 Apr 2020 22:01:28 +0200 Subject: [PATCH 081/118] Revamp README.md --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9c0c87887..9aaca6ed3 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,55 @@ +# Nix + [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) [![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) -Nix, the purely functional package manager ------------------------------------------- +Nix is a powerful package manager for Linux and other Unix systems that makes package +management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +for more details. -Nix is a new take on package management that is fairly unique. Because of its -purity aspects, a lot of issues found in traditional package managers don't -appear with Nix. +## Installation -To find out more about the tool, usage and installation instructions, please -read the manual, which is available on the Nix website at -. +On Linux and macOS the easiest way to Install Nix is to run the following shell command +(as a user other than root): -## Contributing +``` +$ curl -L https://nixos.org/nix/install | sh +``` -Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking) -of the manual. It helps you to get started with building Nix from source. +Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). + +## Building And Developing + +### Building Nix + +You can build Nix via Nix via one of the targets provided by [release.nix](./release.nix): + +``` +$ nix-build ./release.nix -A build.aarch64-linux +$ nix-build ./release.nix -A build.x86_64-darwin +$ nix-build ./release.nix -A build.i686-linux +$ nix-build ./release.nix -A build.x86_64-linux +``` + +### Development Environment + +You can use the provided `shell.nix` to easily bootstrap working development environment: + +``` +$ nix-shell +$ ./bootstrap.sh +$ ./configure +$ make +``` + +## Additional Resources + +- [Nix manual](https://nixos.org/nix/manual) +- [Nix jobs on nixos.hydra.org](https://hydra.nixos.org/project/nix) +- [Nix - A One Pager](https://github.com/tazjin/nix-1p) +- [NixOS Discourse](https://discourse.nixos.org/) +- #nix / #nixos on irc.freenode.net ## License -Nix is released under the LGPL v2.1 +Nix is released under the [LGPL v2.1](./COPYING) From 536512d273f1d5821f65e9df77fe8f47b823ce6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 1 May 2020 11:17:38 +0200 Subject: [PATCH 082/118] Update README.md Co-authored-by: worldofpeace --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9aaca6ed3..454b0ffa0 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ $ nix-build ./release.nix -A build.x86_64-linux ### Development Environment -You can use the provided `shell.nix` to easily bootstrap working development environment: +You can use the provided `shell.nix` to easily bootstrap a working development environment: ``` $ nix-shell From 30616d8e86f7ef635de561905eab0022000547b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 1 May 2020 11:19:18 +0200 Subject: [PATCH 083/118] Update README.md Co-authored-by: worldofpeace --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 454b0ffa0..0f2f9bd1a 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ $ make ## Additional Resources - [Nix manual](https://nixos.org/nix/manual) -- [Nix jobs on nixos.hydra.org](https://hydra.nixos.org/project/nix) +- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [Nix - A One Pager](https://github.com/tazjin/nix-1p) - [NixOS Discourse](https://discourse.nixos.org/) - #nix / #nixos on irc.freenode.net From 2a434fc62bdb4391620d8030aec6b8d4a9f8ba95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 1 May 2020 11:20:39 +0200 Subject: [PATCH 084/118] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f2f9bd1a..44687a150 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ $ make - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [Nix - A One Pager](https://github.com/tazjin/nix-1p) - [NixOS Discourse](https://discourse.nixos.org/) -- #nix / #nixos on irc.freenode.net +- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) ## License From 404a94ab6914405948640bedb92d1f9ddd502a8a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 May 2020 11:47:26 +0200 Subject: [PATCH 085/118] Tweak README.md --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 44687a150..a1588284d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Information on additional installation methods is available on the [Nix download ### Building Nix -You can build Nix via Nix via one of the targets provided by [release.nix](./release.nix): +You can build Nix using one of the targets provided by [release.nix](./release.nix): ``` $ nix-build ./release.nix -A build.aarch64-linux @@ -33,7 +33,7 @@ $ nix-build ./release.nix -A build.x86_64-linux ### Development Environment -You can use the provided `shell.nix` to easily bootstrap a working development environment: +You can use the provided `shell.nix` to get a working development environment: ``` $ nix-shell @@ -46,10 +46,9 @@ $ make - [Nix manual](https://nixos.org/nix/manual) - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) -- [Nix - A One Pager](https://github.com/tazjin/nix-1p) - [NixOS Discourse](https://discourse.nixos.org/) - [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) ## License -Nix is released under the [LGPL v2.1](./COPYING) +Nix is released under the [LGPL v2.1](./COPYING). From 5eebc4ad1d3965096b57c574d361d9274a7daafb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Mar 2020 13:39:39 +0100 Subject: [PATCH 086/118] Fix coverage job (cherry picked from commit bc5d4843a916dc4f99516e187a63108e8006f81f) --- release.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/release.nix b/release.nix index 65b035957..f5729cee3 100644 --- a/release.nix +++ b/release.nix @@ -241,6 +241,11 @@ let src = nix; + preConfigure = + '' + ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor + ''; + enableParallelBuilding = true; buildInputs = buildDeps ++ propagatedDeps; From d6b4047c2fd8c197bf7a9f0f9a2f1cfb214e2e89 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Wed, 29 Apr 2020 14:02:37 -0600 Subject: [PATCH 087/118] rename nix run to nix shell and nix app to nix run (cherry picked from commit 5d8504b9789ffebabe8226227c4061dd48354177) --- src/nix/{shell.cc => dev-shell.cc} | 0 src/nix/run.cc | 14 +++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) rename src/nix/{shell.cc => dev-shell.cc} (100%) diff --git a/src/nix/shell.cc b/src/nix/dev-shell.cc similarity index 100% rename from src/nix/shell.cc rename to src/nix/dev-shell.cc diff --git a/src/nix/run.cc b/src/nix/run.cc index 5334531fd..ebfec36d9 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -57,11 +57,11 @@ struct RunCommon : virtual Command } }; -struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment +struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment { std::vector command = { getEnv("SHELL").value_or("bash") }; - CmdRun() + CmdShell() { mkFlag() .longName("command") @@ -85,19 +85,19 @@ struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment return { Example{ "To start a shell providing GNU Hello from NixOS 17.03:", - "nix run -f channel:nixos-17.03 hello" + "nix shell -f channel:nixos-17.03 hello" }, Example{ "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix run nixpkgs.youtube-dl" + "nix shell nixpkgs.youtube-dl" }, Example{ "To run GNU Hello:", - "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'" + "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'" }, Example{ "To run GNU Hello in a chroot store:", - "nix run --store ~/my-nix nixpkgs.hello -c hello" + "nix shell --store ~/my-nix nixpkgs.hello -c hello" }, }; } @@ -141,7 +141,7 @@ struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment } }; -static auto r1 = registerCommand("run"); +static auto r1 = registerCommand("shell"); void chrootHelper(int argc, char * * argv) { From 2852a486f864c385696fb7ee28836b183976ca63 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Wed, 29 Apr 2020 15:37:21 -0600 Subject: [PATCH 088/118] rename run to shell in tests (cherry picked from commit f459ca547fda4b0ca477d895413e86b69a580ab0) --- tests/local.mk | 2 +- tests/run.sh | 28 ---------------------------- tests/{run.nix => shell-hello.nix} | 0 tests/shell.sh | 28 ++++++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 29 deletions(-) delete mode 100644 tests/run.sh rename tests/{run.nix => shell-hello.nix} (100%) create mode 100644 tests/shell.sh diff --git a/tests/local.mk b/tests/local.mk index c5602773d..56e5640ca 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -21,7 +21,7 @@ nix_tests = \ fetchGitSubmodules.sh \ fetchMercurial.sh \ signing.sh \ - run.sh \ + shell.sh \ brotli.sh \ pure-eval.sh \ check.sh \ diff --git a/tests/run.sh b/tests/run.sh deleted file mode 100644 index d1dbfd6bd..000000000 --- a/tests/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -source common.sh - -clearStore -clearCache - -nix run -f run.nix hello -c hello | grep 'Hello World' -nix run -f run.nix hello -c hello NixOS | grep 'Hello NixOS' - -if ! canUseSandbox; then exit; fi - -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 - -clearStore - -path=$(nix eval --raw -f run.nix hello) - -# Note: we need the sandbox paths to ensure that the shell is -# visible in the sandbox. -nix run --sandbox-build-dir /build-tmp \ - --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ - --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World' - -path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello') - -[[ $path/bin/hello = $path2 ]] - -[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] diff --git a/tests/run.nix b/tests/shell-hello.nix similarity index 100% rename from tests/run.nix rename to tests/shell-hello.nix diff --git a/tests/shell.sh b/tests/shell.sh new file mode 100644 index 000000000..7a9ee8ab0 --- /dev/null +++ b/tests/shell.sh @@ -0,0 +1,28 @@ +source common.sh + +clearStore +clearCache + +nix shell -f shell-hello.nix hello -c hello | grep 'Hello World' +nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' + +if ! canUseSandbox; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +clearStore + +path=$(nix eval --raw -f shell-hello.nix hello) + +# Note: we need the sandbox paths to ensure that the shell is +# visible in the sandbox. +nix shell --sandbox-build-dir /build-tmp \ + --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ + --store $TEST_ROOT/store0 -f shell-hello.nix hello -c hello | grep 'Hello World' + +path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f shell-hello.nix hello -c $SHELL -c 'type -p hello') + +[[ $path/bin/hello = $path2 ]] + +[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] From c089c52d5f1cff888552f485775b74226dcbe618 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 May 2020 12:42:39 +0200 Subject: [PATCH 089/118] Fix build --- src/nix/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/local.mk b/src/nix/local.mk index 15cef55bb..8c0eed19e 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -28,4 +28,4 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh -src/nix/shell.cc: src/nix/get-env.sh.gen.hh +src/nix/dev-shell.cc: src/nix/get-env.sh.gen.hh From 2aeb874e832ae4c326f1115e55fdcaf7c2f4e916 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 1 May 2020 23:32:01 +0200 Subject: [PATCH 090/118] Improve help-message for nix-repl * Remove obsolete `printHelp` function * Add an example to demonstrate how to list all available commands within the REPL --- src/nix/repl.cc | 44 ++++++++++---------------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 27727bd25..ea8ff1553 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -82,40 +82,6 @@ struct NixRepl : gc }; -void printHelp() -{ - std::cout - << "Usage: nix-repl [--help] [--version] [-I path] paths...\n" - << "\n" - << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n" - << "\n" - << "Options:\n" - << " --help\n" - << " Prints out a summary of the command syntax and exits.\n" - << "\n" - << " --version\n" - << " Prints out the Nix version number on standard output and exits.\n" - << "\n" - << " -I path\n" - << " Add a path to the Nix expression search path. This option may be given\n" - << " multiple times. See the NIX_PATH environment variable for information on\n" - << " the semantics of the Nix search path. Paths added through -I take\n" - << " precedence over NIX_PATH.\n" - << "\n" - << " paths...\n" - << " A list of paths to files containing Nix expressions which nix-repl will\n" - << " load and add to its scope.\n" - << "\n" - << " A path surrounded in < and > will be looked up in the Nix expression search\n" - << " path, as in the Nix language itself.\n" - << "\n" - << " If an element of paths starts with http:// or https://, it is interpreted\n" - << " as the URL of a tarball that will be downloaded and unpacked to a temporary\n" - << " location. The tarball must include a single top-level directory containing\n" - << " at least a file named default.nix.\n"; -} - - string removeWhitespace(string s) { s = chomp(s); @@ -809,6 +775,16 @@ struct CmdRepl : StoreCommand, MixEvalArgs return "start an interactive environment for evaluating Nix expressions"; } + Examples examples() override + { + return { + Example{ + "Display all special commands within the REPL:", + "nix repl\n nix-repl> :?" + } + }; + } + void run(ref store) override { auto repl = std::make_unique(searchPath, openStore()); From f16e24f95e7d7fa3a5c3b2c8d43d171fa5b9cf85 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 2 May 2020 10:58:33 +0200 Subject: [PATCH 091/118] remote-store: don't log raw stderr by default For remote stores the log messages are already forwarded as structured STDERR_RESULT messages so the old format is duplicate information. But still included with -vvv since it could be useful for debugging problems. $ nix build -L /nix/store/nl71b2niws857ffiaggyrkjwgx9jjzc0-foo.drv --store ssh-ng://localhost Hello World! foo> Hello World! [1/0/1 built] building foo Fixes #3556 --- src/libstore/remote-store.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8c55da268..07ef79382 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -779,8 +779,10 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return std::make_exception_ptr(Error(status, error)); } - else if (msg == STDERR_NEXT) - printError(chomp(readString(from))); + else if (msg == STDERR_NEXT) { + string s = chomp(readString(from)); + printMsg(lvlVomit, "stderr %s", s); + } else if (msg == STDERR_START_ACTIVITY) { auto act = readNum(from); From 4769eea5e2229c01c3b5d050891b071a45a024d5 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 2 May 2020 23:26:12 +0200 Subject: [PATCH 092/118] logging: handle build log lines in simple logger The raw stderr output isn't logged anymore so the build logs need to be printed by the default logger in order for the old commands like nix-build to still show build output. --- src/libutil/logging.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 3cc4ef8f1..777650de5 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -63,6 +63,16 @@ public: writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); } + void result(ActivityId act, ResultType type, const std::vector & fields) override + { + if (type == resBuildLogLine || type == resPostBuildLogLine) { + assert(0 < fields.size()); + assert(fields[0].type == Logger::Field::tString); + auto lastLine = fields[0].s; + log(lvlInfo, lastLine); + } + } + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) override From e9f10beed1a30fe80b315a9de4c4c2ab11527db8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 May 2020 22:40:02 +0200 Subject: [PATCH 093/118] precompiled-headers.h: Don't include our own headers --- precompiled-headers.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/precompiled-headers.h b/precompiled-headers.h index e0d885b23..079aa496e 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -56,6 +56,3 @@ #include #include #include - -#include "util.hh" -#include "args.hh" From a721a0b1140bf489d645f5d85737acafc1c57c65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 May 2020 22:40:19 +0200 Subject: [PATCH 094/118] Flag: Use designated initializers --- src/libexpr/common-eval-args.cc | 35 ++++---- src/libmain/common-args.cc | 60 +++++++------- src/libmain/shared.cc | 53 ++++++------ src/libstore/globals.cc | 33 ++++---- src/libutil/args.cc | 44 +++++----- src/libutil/args.hh | 139 +++++++++++++++++--------------- src/libutil/config.cc | 35 ++++---- src/nix/add-to-store.cc | 13 +-- src/nix/build.cc | 22 ++--- src/nix/command.cc | 73 +++++++++-------- src/nix/copy.cc | 43 +++++----- src/nix/dev-shell.cc | 16 ++-- src/nix/hash.cc | 8 +- src/nix/installables.cc | 13 +-- src/nix/main.cc | 58 +++++++------ src/nix/run.cc | 16 ++-- src/nix/search.cc | 20 ++--- src/nix/show-derivation.cc | 11 +-- src/nix/sigs.cc | 27 ++++--- src/nix/upgrade-nix.cc | 24 +++--- src/nix/verify.cc | 14 ++-- src/nix/why-depends.cc | 11 +-- 22 files changed, 406 insertions(+), 362 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 10c9c16bb..44baadd53 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -10,24 +10,27 @@ namespace nix { MixEvalArgs::MixEvalArgs() { - mkFlag() - .longName("arg") - .description("argument to be passed to Nix functions") - .labels({"name", "expr"}) - .handler([&](std::vector ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); + addFlag({ + .longName = "arg", + .description = "argument to be passed to Nix functions", + .labels = {"name", "expr"}, + .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + }); - mkFlag() - .longName("argstr") - .description("string-valued argument to be passed to Nix functions") - .labels({"name", "string"}) - .handler([&](std::vector ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); + addFlag({ + .longName = "argstr", + .description = "string-valued argument to be passed to Nix functions", + .labels = {"name", "string"}, + .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + }); - mkFlag() - .shortName('I') - .longName("include") - .description("add a path to the list of locations used to look up <...> file names") - .label("path") - .handler([&](std::string s) { searchPath.push_back(s); }); + addFlag({ + .longName = "include", + .shortName = 'I', + .description = "add a path to the list of locations used to look up <...> file names", + .labels = {"path"}, + .handler = {[&](std::string s) { searchPath.push_back(s); }} + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 9e1d7cee6..51e199ea5 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -6,43 +6,47 @@ namespace nix { MixCommonArgs::MixCommonArgs(const string & programName) : programName(programName) { - mkFlag() - .longName("verbose") - .shortName('v') - .description("increase verbosity level") - .handler([]() { verbosity = (Verbosity) (verbosity + 1); }); + addFlag({ + .longName = "verbose", + .shortName = 'v', + .description = "increase verbosity level", + .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, + }); - mkFlag() - .longName("quiet") - .description("decrease verbosity level") - .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }); + addFlag({ + .longName = "quiet", + .description = "decrease verbosity level", + .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, + }); - mkFlag() - .longName("debug") - .description("enable debug output") - .handler([]() { verbosity = lvlDebug; }); + addFlag({ + .longName = "debug", + .description = "enable debug output", + .handler = {[]() { verbosity = lvlDebug; }}, + }); - mkFlag() - .longName("option") - .labels({"name", "value"}) - .description("set a Nix configuration option (overriding nix.conf)") - .arity(2) - .handler([](std::vector ss) { + addFlag({ + .longName = "option", + .description = "set a Nix configuration option (overriding nix.conf)", + .labels = {"name", "value"}, + .handler = {[](std::string name, std::string value) { try { - globalConfig.set(ss[0], ss[1]); + globalConfig.set(name, value); } catch (UsageError & e) { warn(e.what()); } - }); + }}, + }); - mkFlag() - .longName("max-jobs") - .shortName('j') - .label("jobs") - .description("maximum number of parallel builds") - .handler([=](std::string s) { + addFlag({ + .longName = "max-jobs", + .shortName = 'j', + .description = "maximum number of parallel builds", + .labels = Strings{"jobs"}, + .handler = {[=](std::string s) { settings.set("max-jobs", s); - }); + }} + }); std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 8586257bf..70d1f0186 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -165,28 +165,32 @@ LegacyArgs::LegacyArgs(const std::string & programName, std::function parseArg) : MixCommonArgs(programName), parseArg(parseArg) { - mkFlag() - .longName("no-build-output") - .shortName('Q') - .description("do not show build output") - .set(&settings.verboseBuild, false); + addFlag({ + .longName = "no-build-output", + .shortName = 'Q', + .description = "do not show build output", + .handler = {&settings.verboseBuild, false}, + }); - mkFlag() - .longName("keep-failed") - .shortName('K') - .description("keep temporary directories of failed builds") - .set(&(bool&) settings.keepFailed, true); + addFlag({ + .longName = "keep-failed", + .shortName ='K', + .description = "keep temporary directories of failed builds", + .handler = {&(bool&) settings.keepFailed, true}, + }); - mkFlag() - .longName("keep-going") - .shortName('k') - .description("keep going after a build fails") - .set(&(bool&) settings.keepGoing, true); + addFlag({ + .longName = "keep-going", + .shortName ='k', + .description = "keep going after a build fails", + .handler = {&(bool&) settings.keepGoing, true}, + }); - mkFlag() - .longName("fallback") - .description("build from source if substitution fails") - .set(&(bool&) settings.tryFallback, true); + addFlag({ + .longName = "fallback", + .description = "build from source if substitution fails", + .handler = {&(bool&) settings.tryFallback, true}, + }); auto intSettingAlias = [&](char shortName, const std::string & longName, const std::string & description, const std::string & dest) { @@ -205,11 +209,12 @@ LegacyArgs::LegacyArgs(const std::string & programName, mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", &gcWarning, false); - mkFlag() - .longName("store") - .label("store-uri") - .description("URI of the Nix store to use") - .dest(&(std::string&) settings.storeUri); + addFlag({ + .longName = "store", + .description = "URI of the Nix store to use", + .labels = {"store-uri"}, + .handler = {&(std::string&) settings.storeUri}, + }); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d6ea0318e..a0a2d850e 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -167,21 +167,24 @@ template<> void BaseSetting::toJSON(JSONPlaceholder & out) template<> void BaseSetting::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description("Enable sandboxing.") - .handler([=](std::vector ss) { override(smEnabled); }) - .category(category); - args.mkFlag() - .longName("no-" + name) - .description("Disable sandboxing.") - .handler([=](std::vector ss) { override(smDisabled); }) - .category(category); - args.mkFlag() - .longName("relaxed-" + name) - .description("Enable sandboxing, but allow builds to disable it.") - .handler([=](std::vector ss) { override(smRelaxed); }) - .category(category); + args.addFlag({ + .longName = name, + .description = "Enable sandboxing.", + .category = category, + .handler = {[=]() { override(smEnabled); }} + }); + args.addFlag({ + .longName = "no-" + name, + .description = "Disable sandboxing.", + .category = category, + .handler = {[=]() { override(smDisabled); }} + }); + args.addFlag({ + .longName = "relaxed-" + name, + .description = "Enable sandboxing, but allow builds to disable it.", + .category = category, + .handler = {[=]() { override(smRelaxed); }} + }); } void MaxBuildJobsSetting::set(const std::string & str) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index ba15ea571..91fc2f581 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -3,16 +3,14 @@ namespace nix { -Args::FlagMaker Args::mkFlag() -{ - return FlagMaker(*this); -} - -Args::FlagMaker::~FlagMaker() +void Args::addFlag(Flag && flag_) { + auto flag = std::make_shared(std::move(flag_)); + if (flag->handler.arity != ArityAny) + assert(flag->handler.arity == flag->labels.size()); assert(flag->longName != ""); - args.longFlags[flag->longName] = flag; - if (flag->shortName) args.shortFlags[flag->shortName] = flag; + longFlags[flag->longName] = flag; + if (flag->shortName) shortFlags[flag->shortName] = flag; } void Args::parseCmdline(const Strings & _cmdline) @@ -101,15 +99,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; std::vector args; - for (size_t n = 0 ; n < flag.arity; ++n) { + for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { - if (flag.arity == ArityAny) break; - throw UsageError(format("flag '%1%' requires %2% argument(s)") - % name % flag.arity); + if (flag.handler.arity == ArityAny) break; + throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } args.push_back(*pos++); } - flag.handler(std::move(args)); + flag.handler.fun(std::move(args)); return true; }; @@ -157,17 +154,18 @@ bool Args::processArgs(const Strings & args, bool finish) return res; } -Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht) +Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) { - arity(1); - label("type"); - description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); - handler([ht](std::string s) { - *ht = parseHashType(s); - if (*ht == htUnknown) - throw UsageError("unknown hash type '%1%'", s); - }); - return *this; + return Flag { + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", + .labels = {"hash-algo"}, + .handler = {[ht](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError("unknown hash type '%1%'", s); + }} + }; } Strings argvToStrings(int argc, char * * argv) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 967efbe1c..9b5e316a5 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -32,13 +32,59 @@ protected: struct Flag { typedef std::shared_ptr ptr; + + struct Handler + { + std::function)> fun; + size_t arity; + + Handler() {} + + Handler(std::function)> && fun) + : fun(std::move(fun)) + , arity(ArityAny) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector) { handler(); }) + , arity(0) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector ss) { + handler(std::move(ss[0])); + }) + , arity(1) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector ss) { + handler(std::move(ss[0]), std::move(ss[1])); + }) + , arity(2) + { } + + template + Handler(T * dest) + : fun([=](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + template + Handler(T * dest, const T & val) + : fun([=](std::vector ss) { *dest = val; }) + , arity(0) + { } + }; + std::string longName; char shortName = 0; std::string description; - Strings labels; - size_t arity = 0; - std::function)> handler; std::string category; + Strings labels; + Handler handler; + + static Flag mkHashTypeFlag(std::string && longName, HashType * ht); }; std::map longFlags; @@ -65,49 +111,7 @@ protected: public: - class FlagMaker - { - Args & args; - Flag::ptr flag; - friend class Args; - FlagMaker(Args & args) : args(args), flag(std::make_shared()) { } - public: - ~FlagMaker(); - FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; } - FlagMaker & shortName(char s) { flag->shortName = s; return *this; } - FlagMaker & description(const std::string & s) { flag->description = s; return *this; } - FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; } - FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; } - FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; } - FlagMaker & handler(std::function)> handler) { flag->handler = handler; return *this; } - FlagMaker & handler(std::function handler) { flag->handler = [handler](std::vector) { handler(); }; return *this; } - FlagMaker & handler(std::function handler) { - flag->arity = 1; - flag->handler = [handler](std::vector ss) { handler(std::move(ss[0])); }; - return *this; - } - FlagMaker & category(const std::string & s) { flag->category = s; return *this; } - - template - FlagMaker & dest(T * dest) - { - flag->arity = 1; - flag->handler = [=](std::vector ss) { *dest = ss[0]; }; - return *this; - } - - template - FlagMaker & set(T * dest, const T & val) - { - flag->arity = 0; - flag->handler = [=](std::vector ss) { *dest = val; }; - return *this; - } - - FlagMaker & mkHashTypeFlag(HashType * ht); - }; - - FlagMaker mkFlag(); + void addFlag(Flag && flag); /* Helper functions for constructing flags / positional arguments. */ @@ -116,13 +120,13 @@ public: const std::string & label, const std::string & description, std::function fun) { - mkFlag() - .shortName(shortName) - .longName(longName) - .labels({label}) - .description(description) - .arity(1) - .handler([=](std::vector ss) { fun(ss[0]); }); + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {label}, + .handler = {[=](std::string s) { fun(s); }} + }); } void mkFlag(char shortName, const std::string & name, @@ -135,11 +139,12 @@ public: void mkFlag(char shortName, const std::string & longName, const std::string & description, T * dest, const T & value) { - mkFlag() - .shortName(shortName) - .longName(longName) - .description(description) - .handler([=](std::vector ss) { *dest = value; }); + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .handler = {[=]() { *dest = value; }} + }); } template @@ -155,18 +160,18 @@ public: void mkFlag(char shortName, const std::string & longName, const std::string & description, std::function fun) { - mkFlag() - .shortName(shortName) - .longName(longName) - .labels({"N"}) - .description(description) - .arity(1) - .handler([=](std::vector ss) { + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {"N"}, + .handler = {[=](std::string s) { I n; - if (!string2Int(ss[0], n)) + if (!string2Int(s, n)) throw UsageError("flag '--%s' requires a integer argument", longName); fun(n); - }); + }} + }); } /* Expect a string argument. */ diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7551d97d1..f03e444ec 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -177,12 +177,13 @@ void BaseSetting::toJSON(JSONPlaceholder & out) template void BaseSetting::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description(description) - .arity(1) - .handler([=](std::vector ss) { overriden = true; set(ss[0]); }) - .category(category); + args.addFlag({ + .longName = name, + .description = description, + .category = category, + .labels = {"value"}, + .handler = {[=](std::string s) { overriden = true; set(s); }}, + }); } template<> void BaseSetting::set(const std::string & str) @@ -227,16 +228,18 @@ template<> std::string BaseSetting::to_string() const template<> void BaseSetting::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description(description) - .handler([=](std::vector ss) { override(true); }) - .category(category); - args.mkFlag() - .longName("no-" + name) - .description(description) - .handler([=](std::vector ss) { override(false); }) - .category(category); + args.addFlag({ + .longName = name, + .description = description, + .category = category, + .handler = {[=]() { override(true); }} + }); + args.addFlag({ + .longName = "no-" + name, + .description = description, + .category = category, + .handler = {[=]() { override(false); }} + }); } template<> void BaseSetting::set(const std::string & str) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ed02227db..00da01f7e 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -14,12 +14,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand { expectArg("path", &path); - mkFlag() - .longName("name") - .shortName('n') - .description("name component of the store path") - .labels({"name"}) - .dest(&namePart); + addFlag({ + .longName = "name", + .shortName = 'n', + .description = "name component of the store path", + .labels = {"name"}, + .handler = {&namePart}, + }); } std::string description() override diff --git a/src/nix/build.cc b/src/nix/build.cc index 0b0762836..850e09ce8 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -11,17 +11,19 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile CmdBuild() { - mkFlag() - .longName("out-link") - .shortName('o') - .description("path of the symlink to the build result") - .labels({"path"}) - .dest(&outLink); + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "path of the symlink to the build result", + .labels = {"path"}, + .handler = {&outLink}, + }); - mkFlag() - .longName("no-link") - .description("do not create a symlink to the build result") - .set(&outLink, Path("")); + addFlag({ + .longName = "no-link", + .description = "do not create a symlink to the build result", + .handler = {&outLink, Path("")}, + }); } std::string description() override diff --git a/src/nix/command.cc b/src/nix/command.cc index 99b24d2a2..71b027719 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -35,16 +35,18 @@ StorePathsCommand::StorePathsCommand(bool recursive) : recursive(recursive) { if (recursive) - mkFlag() - .longName("no-recursive") - .description("apply operation to specified paths only") - .set(&this->recursive, false); + addFlag({ + .longName = "no-recursive", + .description = "apply operation to specified paths only", + .handler = {&this->recursive, false}, + }); else - mkFlag() - .longName("recursive") - .shortName('r') - .description("apply operation to closure of the specified paths") - .set(&this->recursive, true); + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "apply operation to closure of the specified paths", + .handler = {&this->recursive, true}, + }); mkFlag(0, "all", "apply operation to the entire store", &all); } @@ -101,11 +103,12 @@ Strings editorFor(const Pos & pos) MixProfile::MixProfile() { - mkFlag() - .longName("profile") - .description("profile to update") - .labels({"path"}) - .dest(&profile); + addFlag({ + .longName = "profile", + .description = "profile to update", + .labels = {"path"}, + .handler = {&profile}, + }); } void MixProfile::updateProfile(const StorePath & storePath) @@ -145,28 +148,30 @@ MixDefaultProfile::MixDefaultProfile() profile = getDefaultProfile(); } -MixEnvironment::MixEnvironment() : ignoreEnvironment(false) { - mkFlag() - .longName("ignore-environment") - .shortName('i') - .description("clear the entire environment (except those specified with --keep)") - .set(&ignoreEnvironment, true); +MixEnvironment::MixEnvironment() : ignoreEnvironment(false) +{ + addFlag({ + .longName = "ignore-environment", + .shortName = 'i', + .description = "clear the entire environment (except those specified with --keep)", + .handler = {&ignoreEnvironment, true}, + }); - mkFlag() - .longName("keep") - .shortName('k') - .description("keep specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector ss) { keep.insert(ss.front()); }); + addFlag({ + .longName = "keep", + .shortName = 'k', + .description = "keep specified environment variable", + .labels = {"name"}, + .handler = {[&](std::string s) { keep.insert(s); }}, + }); - mkFlag() - .longName("unset") - .shortName('u') - .description("unset specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector ss) { unset.insert(ss.front()); }); + addFlag({ + .longName = "unset", + .shortName = 'u', + .description = "unset specified environment variable", + .labels = {"name"}, + .handler = {[&](std::string s) { unset.insert(s); }}, + }); } void MixEnvironment::setEnviron() { diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 85c777d38..77673a1c2 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -19,27 +19,32 @@ struct CmdCopy : StorePathsCommand CmdCopy() : StorePathsCommand(true) { - mkFlag() - .longName("from") - .labels({"store-uri"}) - .description("URI of the source Nix store") - .dest(&srcUri); - mkFlag() - .longName("to") - .labels({"store-uri"}) - .description("URI of the destination Nix store") - .dest(&dstUri); + addFlag({ + .longName = "from", + .description = "URI of the source Nix store", + .labels = {"store-uri"}, + .handler = {&srcUri}, + }); - mkFlag() - .longName("no-check-sigs") - .description("do not require that paths are signed by trusted keys") - .set(&checkSigs, NoCheckSigs); + addFlag({ + .longName = "to", + .description = "URI of the destination Nix store", + .labels = {"store-uri"}, + .handler = {&dstUri}, + }); - mkFlag() - .longName("substitute-on-destination") - .shortName('s') - .description("whether to try substitutes on the destination store (only supported by SSH)") - .set(&substitute, Substitute); + addFlag({ + .longName = "no-check-sigs", + .description = "do not require that paths are signed by trusted keys", + .handler = {&checkSigs, NoCheckSigs}, + }); + + addFlag({ + .longName = "substitute-on-destination", + .shortName = 's', + .description = "whether to try substitutes on the destination store (only supported by SSH)", + .handler = {&substitute, Substitute}, + }); } std::string description() override diff --git a/src/nix/dev-shell.cc b/src/nix/dev-shell.cc index 9c45a935d..2bdf59839 100644 --- a/src/nix/dev-shell.cc +++ b/src/nix/dev-shell.cc @@ -237,16 +237,16 @@ struct CmdDevShell : Common, MixEnvironment CmdDevShell() { - mkFlag() - .longName("command") - .shortName('c') - .description("command and arguments to be executed insted of an interactive shell") - .labels({"command", "args"}) - .arity(ArityAny) - .handler([&](std::vector ss) { + addFlag({ + .longName = "command", + .shortName = 'c', + .description = "command and arguments to be executed insted of an interactive shell", + .labels = {"command", "args"}, + .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; - }); + }} + }); } std::string description() override diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 01628cf6c..9b9509d3a 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -23,9 +23,7 @@ struct CmdHash : Command mkFlag(0, "base64", "print hash in base-64", &base, Base64); mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); mkFlag(0, "base16", "print hash in base-16", &base, Base16); - mkFlag() - .longName("type") - .mkHashTypeFlag(&ht); + addFlag(Flag::mkHashTypeFlag("type", &ht)); #if 0 mkFlag() .longName("modulo") @@ -76,9 +74,7 @@ struct CmdToBase : Command CmdToBase(Base base) : base(base) { - mkFlag() - .longName("type") - .mkHashTypeFlag(&ht); + addFlag(Flag::mkHashTypeFlag("type", &ht)); expectArgs("strings", &args); } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 1d70ad3d5..937d69206 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -15,12 +15,13 @@ namespace nix { SourceExprCommand::SourceExprCommand() { - mkFlag() - .shortName('f') - .longName("file") - .label("file") - .description("evaluate FILE rather than the default") - .dest(&file); + addFlag({ + .longName = "file", + .shortName = 'f', + .description = "evaluate FILE rather than the default", + .labels = {"file"}, + .handler = {&file} + }); } Value * SourceExprCommand::getSourceExpr(EvalState & state) diff --git a/src/nix/main.cc b/src/nix/main.cc index 2c64c7476..57b8bed9f 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -59,15 +59,16 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { - mkFlag() - .longName("help") - .description("show usage information") - .handler([&]() { showHelpAndExit(); }); + addFlag({ + .longName = "help", + .description = "show usage information", + .handler = {[&]() { showHelpAndExit(); }}, + }); - mkFlag() - .longName("help-config") - .description("show configuration options") - .handler([&]() { + addFlag({ + .longName = "help-config", + .description = "show configuration options", + .handler = {[&]() { std::cout << "The following configuration options are available:\n\n"; Table2 tbl; std::map settings; @@ -76,28 +77,33 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs tbl.emplace_back(s.first, s.second.description); printTable(std::cout, tbl); throw Exit(); - }); + }}, + }); - mkFlag() - .longName("print-build-logs") - .shortName('L') - .description("print full build logs on stderr") - .set(&printBuildLogs, true); + addFlag({ + .longName = "print-build-logs", + .shortName = 'L', + .description = "print full build logs on stderr", + .handler = {&printBuildLogs, true}, + }); - mkFlag() - .longName("version") - .description("show version information") - .handler([&]() { printVersion(programName); }); + addFlag({ + .longName = "version", + .description = "show version information", + .handler = {[&]() { printVersion(programName); }}, + }); - mkFlag() - .longName("no-net") - .description("disable substituters and consider all previously downloaded files up-to-date") - .handler([&]() { useNet = false; }); + addFlag({ + .longName = "no-net", + .description = "disable substituters and consider all previously downloaded files up-to-date", + .handler = {[&]() { useNet = false; }}, + }); - mkFlag() - .longName("refresh") - .description("consider all previously downloaded files out-of-date") - .handler([&]() { refresh = true; }); + addFlag({ + .longName = "refresh", + .description = "consider all previously downloaded files out-of-date", + .handler = {[&]() { refresh = true; }}, + }); } void printFlags(std::ostream & out) override diff --git a/src/nix/run.cc b/src/nix/run.cc index ebfec36d9..b888281a5 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -63,16 +63,16 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment CmdShell() { - mkFlag() - .longName("command") - .shortName('c') - .description("command and arguments to be executed; defaults to '$SHELL'") - .labels({"command", "args"}) - .arity(ArityAny) - .handler([&](std::vector ss) { + addFlag({ + .longName = "command", + .shortName = 'c', + .description = "command and arguments to be executed; defaults to '$SHELL'", + .labels = {"command", "args"}, + .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; - }); + }} + }); } std::string description() override diff --git a/src/nix/search.cc b/src/nix/search.cc index 769274543..fcad6be84 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -40,16 +40,18 @@ struct CmdSearch : SourceExprCommand, MixJSON { expectArgs("regex", &res); - mkFlag() - .longName("update-cache") - .shortName('u') - .description("update the package search cache") - .handler([&]() { writeCache = true; useCache = false; }); + addFlag({ + .longName = "update-cache", + .shortName = 'u', + .description = "update the package search cache", + .handler = {[&]() { writeCache = true; useCache = false; }} + }); - mkFlag() - .longName("no-cache") - .description("do not use or update the package search cache") - .handler([&]() { writeCache = false; useCache = false; }); + addFlag({ + .longName = "no-cache", + .description = "do not use or update the package search cache", + .handler = {[&]() { writeCache = false; useCache = false; }} + }); } std::string description() override diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 0ede7b468..b6f24599f 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -15,11 +15,12 @@ struct CmdShowDerivation : InstallablesCommand CmdShowDerivation() { - mkFlag() - .longName("recursive") - .shortName('r') - .description("include the dependencies of the specified derivations") - .set(&recursive, true); + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "include the dependencies of the specified derivations", + .handler = {&recursive, true} + }); } std::string description() override diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 5f07448e0..a91465c2a 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -13,13 +13,13 @@ struct CmdCopySigs : StorePathsCommand CmdCopySigs() { - mkFlag() - .longName("substituter") - .shortName('s') - .labels({"store-uri"}) - .description("use signatures from specified store") - .arity(1) - .handler([&](std::vector ss) { substituterUris.push_back(ss[0]); }); + addFlag({ + .longName = "substituter", + .shortName = 's', + .description = "use signatures from specified store", + .labels = {"store-uri"}, + .handler = {[&](std::string s) { substituterUris.push_back(s); }}, + }); } std::string description() override @@ -98,12 +98,13 @@ struct CmdSignPaths : StorePathsCommand CmdSignPaths() { - mkFlag() - .shortName('k') - .longName("key-file") - .label("file") - .description("file containing the secret signing key") - .dest(&secretKeyFile); + addFlag({ + .longName = "key-file", + .shortName = 'k', + .description = "file containing the secret signing key", + .labels = {"file"}, + .handler = {&secretKeyFile} + }); } std::string description() override diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 4fcc6a721..32efcc3a7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -16,18 +16,20 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand CmdUpgradeNix() { - mkFlag() - .longName("profile") - .shortName('p') - .labels({"profile-dir"}) - .description("the Nix profile to upgrade") - .dest(&profileDir); + addFlag({ + .longName = "profile", + .shortName = 'p', + .description = "the Nix profile to upgrade", + .labels = {"profile-dir"}, + .handler = {&profileDir} + }); - mkFlag() - .longName("nix-store-paths-url") - .labels({"url"}) - .description("URL of the file that contains the store paths of the latest Nix release") - .dest(&storePathsUrl); + addFlag({ + .longName = "nix-store-paths-url", + .description = "URL of the file that contains the store paths of the latest Nix release", + .labels = {"url"}, + .handler = {&storePathsUrl} + }); } std::string description() override diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 9b0658803..08a36ac50 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -20,13 +20,13 @@ struct CmdVerify : StorePathsCommand { mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); - mkFlag() - .longName("substituter") - .shortName('s') - .labels({"store-uri"}) - .description("use signatures from specified store") - .arity(1) - .handler([&](std::vector ss) { substituterUris.push_back(ss[0]); }); + addFlag({ + .longName = "substituter", + .shortName = 's', + .description = "use signatures from specified store", + .labels = {"store-uri"}, + .handler = {[&](std::string s) { substituterUris.push_back(s); }} + }); mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index f9acc7f13..36a3ee863 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -37,11 +37,12 @@ struct CmdWhyDepends : SourceExprCommand expectArg("package", &_package); expectArg("dependency", &_dependency); - mkFlag() - .longName("all") - .shortName('a') - .description("show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path") - .set(&all, true); + addFlag({ + .longName = "all", + .shortName = 'a', + .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .handler = {&all, true}, + }); } std::string description() override From e2fc575c6122b40f5d660ae04f269e118354c101 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Mon, 4 May 2020 14:42:06 -0700 Subject: [PATCH 095/118] nix auto-gc: use fragment size --- src/libstore/gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 6bab1e37c..d210defe5 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -889,7 +889,7 @@ void LocalStore::autoGC(bool sync) if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); - return (uint64_t) st.f_bavail * st.f_bsize; + return (uint64_t) st.f_bavail * st.f_frsize; }; std::shared_future future; From f132d82a796c91fcb741c127f37c963622b4cae4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2020 15:18:23 +0200 Subject: [PATCH 096/118] nix --help: Group commands --- src/libutil/ansicolor.hh | 24 ++++++------ src/libutil/args.cc | 57 ++++++++++++++++------------- src/libutil/args.hh | 18 +++++---- src/nix/add-to-store.cc | 2 + src/nix/cat.cc | 8 +++- src/nix/command.hh | 4 ++ src/nix/copy.cc | 2 + src/nix/dev-shell.cc | 2 + src/nix/doctor.cc | 2 + src/nix/dump-path.cc | 2 + src/nix/edit.cc | 2 + src/nix/eval.cc | 2 + src/nix/hash.cc | 4 ++ src/nix/log.cc | 2 + src/nix/ls.cc | 8 +++- src/nix/main.cc | 18 ++++++--- src/nix/make-content-addressable.cc | 3 ++ src/nix/optimise-store.cc | 2 + src/nix/path-info.cc | 2 + src/nix/ping-store.cc | 2 + src/nix/show-config.cc | 2 + src/nix/show-derivation.cc | 2 + src/nix/sigs.cc | 4 ++ src/nix/upgrade-nix.cc | 2 + src/nix/verify.cc | 2 + src/nix/why-depends.cc | 2 + 26 files changed, 125 insertions(+), 55 deletions(-) diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index 390bd4d17..8ae07b092 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -1,13 +1,15 @@ -#pragma once +#pragma once + +namespace nix { + +/* Some ANSI escape sequences. */ +#define ANSI_NORMAL "\e[0m" +#define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" +#define ANSI_ITALIC "\e[3m" +#define ANSI_RED "\e[31;1m" +#define ANSI_GREEN "\e[32;1m" +#define ANSI_YELLOW "\e[33;1m" +#define ANSI_BLUE "\e[34;1m" -namespace nix -{ - /* Some ANSI escape sequences. */ - #define ANSI_NORMAL "\e[0m" - #define ANSI_BOLD "\e[1m" - #define ANSI_FAINT "\e[2m" - #define ANSI_RED "\e[31;1m" - #define ANSI_GREEN "\e[32;1m" - #define ANSI_YELLOW "\e[33;1m" - #define ANSI_BLUE "\e[34;1m" } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 91fc2f581..f829415d1 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -59,7 +59,7 @@ void Args::parseCmdline(const Strings & _cmdline) void Args::printHelp(const string & programName, std::ostream & out) { - std::cout << "Usage: " << programName << " ..."; + std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName); for (auto & exp : expectedArgs) { std::cout << renderLabels({exp.label}); // FIXME: handle arity > 1 @@ -70,11 +70,11 @@ void Args::printHelp(const string & programName, std::ostream & out) auto s = description(); if (s != "") - std::cout << "\nSummary: " << s << ".\n"; + std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n"; if (longFlags.size()) { std::cout << "\n"; - std::cout << "Flags:\n"; + std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n"; printFlags(out); } } @@ -181,7 +181,7 @@ std::string renderLabels(const Strings & labels) std::string res; for (auto label : labels) { for (auto & c : label) c = std::toupper(c); - res += " <" + label + ">"; + res += " " ANSI_ITALIC + label + ANSI_NORMAL; } return res; } @@ -190,10 +190,10 @@ void printTable(std::ostream & out, const Table2 & table) { size_t max = 0; for (auto & row : table) - max = std::max(max, row.first.size()); + max = std::max(max, filterANSIEscapes(row.first, true).size()); for (auto & row : table) { out << " " << row.first - << std::string(max - row.first.size() + 2, ' ') + << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ') << row.second << "\n"; } } @@ -204,8 +204,7 @@ void Command::printHelp(const string & programName, std::ostream & out) auto exs = examples(); if (!exs.empty()) { - out << "\n"; - out << "Examples:\n"; + out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n"; for (auto & ex : exs) out << "\n" << " " << ex.description << "\n" // FIXME: wrap @@ -221,49 +220,55 @@ MultiCommand::MultiCommand(const Commands & commands) auto i = commands.find(ss[0]); if (i == commands.end()) throw UsageError("'%s' is not a recognised command", ss[0]); - command = i->second(); - command->_name = ss[0]; + command = {ss[0], i->second()}; }}); + + categories[Command::catDefault] = "Available commands"; } void MultiCommand::printHelp(const string & programName, std::ostream & out) { if (command) { - command->printHelp(programName + " " + command->name(), out); + command->second->printHelp(programName + " " + command->first, out); return; } - out << "Usage: " << programName << " ... ...\n"; + out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName); - out << "\n"; - out << "Common flags:\n"; + out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n"; printFlags(out); - out << "\n"; - out << "Available commands:\n"; + std::map>> commandsByCategory; - Table2 table; - for (auto & i : commands) { - auto command = i.second(); - command->_name = i.first; - auto descr = command->description(); - if (!descr.empty()) - table.push_back(std::make_pair(command->name(), descr)); + for (auto & [name, commandFun] : commands) { + auto command = commandFun(); + commandsByCategory[command->category()].insert_or_assign(name, command); + } + + for (auto & [category, commands] : commandsByCategory) { + out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]); + + Table2 table; + for (auto & [name, command] : commands) { + auto descr = command->description(); + if (!descr.empty()) + table.push_back(std::make_pair(name, descr)); + } + printTable(out, table); } - printTable(out, table); } bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) { if (Args::processFlag(pos, end)) return true; - if (command && command->processFlag(pos, end)) return true; + if (command && command->second->processFlag(pos, end)) return true; return false; } bool MultiCommand::processArgs(const Strings & args, bool finish) { if (command) - return command->processArgs(args, finish); + return command->second->processArgs(args, finish); else return Args::processArgs(args, finish); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 9b5e316a5..1932e6a8a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -197,17 +197,10 @@ public: run() method. */ struct Command : virtual Args { -private: - std::string _name; - friend class MultiCommand; -public: - virtual ~Command() { } - std::string name() { return _name; } - virtual void prepare() { }; virtual void run() = 0; @@ -221,6 +214,12 @@ public: virtual Examples examples() { return Examples(); } + typedef int Category; + + static constexpr Category catDefault = 0; + + virtual Category category() { return catDefault; } + void printHelp(const string & programName, std::ostream & out) override; }; @@ -233,7 +232,10 @@ class MultiCommand : virtual Args public: Commands commands; - std::shared_ptr command; + std::map categories; + + // Selected command, if any. + std::optional>> command; MultiCommand(const Commands & commands); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 00da01f7e..1d298903b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -34,6 +34,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand }; } + Category category() override { return catUtility; } + void run(ref store) override { if (!namePart) namePart = baseNameOf(path); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 851f90abd..fd91f2036 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -30,9 +30,11 @@ struct CmdCatStore : StoreCommand, MixCat std::string description() override { - return "print the contents of a store file on stdout"; + return "print the contents of a file in the Nix store on stdout"; } + Category category() override { return catUtility; } + void run(ref store) override { cat(store->getFSAccessor()); @@ -51,9 +53,11 @@ struct CmdCatNar : StoreCommand, MixCat std::string description() override { - return "print the contents of a file inside a NAR file"; + return "print the contents of a file inside a NAR file on stdout"; } + Category category() override { return catUtility; } + void run(ref store) override { cat(makeNarAccessor(make_ref(readFile(narPath)))); diff --git a/src/nix/command.hh b/src/nix/command.hh index bf43d950f..959d5f19d 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -10,6 +10,10 @@ namespace nix { extern std::string programPath; +static constexpr Command::Category catSecondary = 100; +static constexpr Command::Category catUtility = 101; +static constexpr Command::Category catNixInstallation = 102; + /* A command that requires a Nix store. */ struct StoreCommand : virtual Command { diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 77673a1c2..c7c38709d 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -80,6 +80,8 @@ struct CmdCopy : StorePathsCommand }; } + Category category() override { return catSecondary; } + ref createStore() override { return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); diff --git a/src/nix/dev-shell.cc b/src/nix/dev-shell.cc index 2bdf59839..d300f6a23 100644 --- a/src/nix/dev-shell.cc +++ b/src/nix/dev-shell.cc @@ -328,6 +328,8 @@ struct CmdPrintDevEnv : Common }; } + Category category() override { return catUtility; } + void run(ref store) override { auto buildEnvironment = getBuildEnvironment(store).first; diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 0aa634d6e..cc36354f1 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -43,6 +43,8 @@ struct CmdDoctor : StoreCommand return "check your system for potential problems and print a PASS or FAIL for each check."; } + Category category() override { return catNixInstallation; } + void run(ref store) override { logger->log("Running checks against store uri: " + store->getUri()); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index bb741b572..e1de71bf8 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -20,6 +20,8 @@ struct CmdDumpPath : StorePathCommand }; } + Category category() override { return catUtility; } + void run(ref store, const StorePath & storePath) override { FdSink sink(STDOUT_FILENO); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 1683eada0..067d3a973 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -25,6 +25,8 @@ struct CmdEdit : InstallableCommand }; } + Category category() override { return catSecondary; } + void run(ref store) override { auto state = getEvalState(); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 86a1e8b68..26e98ac2a 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -45,6 +45,8 @@ struct CmdEval : MixJSON, InstallableCommand }; } + Category category() override { return catSecondary; } + void run(ref store) override { if (raw && json) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9b9509d3a..366314227 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -41,6 +41,8 @@ struct CmdHash : Command : "print cryptographic hash of the NAR serialisation of a path"; } + Category category() override { return catUtility; } + void run() override { for (auto path : paths) { @@ -87,6 +89,8 @@ struct CmdToBase : Command "SRI"); } + Category category() override { return catUtility; } + void run() override { for (auto s : args) diff --git a/src/nix/log.cc b/src/nix/log.cc index 795991cb7..3fe22f6c2 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -31,6 +31,8 @@ struct CmdLog : InstallableCommand }; } + Category category() override { return catSecondary; } + void run(ref store) override { settings.readOnlyMode = true; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 8590199d7..6ae4df27e 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -100,9 +100,11 @@ struct CmdLsStore : StoreCommand, MixLs std::string description() override { - return "show information about a store path"; + return "show information about a path in the Nix store"; } + Category category() override { return catUtility; } + void run(ref store) override { list(store->getFSAccessor()); @@ -131,9 +133,11 @@ struct CmdLsNar : Command, MixLs std::string description() override { - return "show information about the contents of a NAR file"; + return "show information about a path inside a NAR file"; } + Category category() override { return catUtility; } + void run() override { list(makeNarAccessor(make_ref(readFile(narPath, true)))); diff --git a/src/nix/main.cc b/src/nix/main.cc index 57b8bed9f..5cf09c4f0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -59,6 +59,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { + categories.clear(); + categories[Command::catDefault] = "Main commands"; + categories[catSecondary] = "Infrequently used commands"; + categories[catUtility] = "Utility/scripting commands"; + categories[catNixInstallation] = "Commands for upgrading or troubleshooting your Nix installation"; + addFlag({ .longName = "help", .description = "show usage information", @@ -111,8 +117,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs Args::printFlags(out); std::cout << "\n" - "In addition, most configuration settings can be overriden using '-- '.\n" - "Boolean settings can be overriden using '--' or '--no-'. See 'nix\n" + "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n" + "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n" "--help-config' for a list of configuration settings.\n"; } @@ -121,10 +127,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs MultiCommand::printHelp(programName, out); #if 0 - out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-'.\n"; + out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n"; #endif - std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n"; } void showHelpAndExit() @@ -191,8 +197,8 @@ void mainWrapped(int argc, char * * argv) if (args.refresh) settings.tarballTtl = 0; - args.command->prepare(); - args.command->run(); + args.command->second->prepare(); + args.command->second->run(); } } diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index f9c7fef3f..8803461f4 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -31,6 +31,9 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON }, }; } + + Category category() override { return catUtility; } + void run(ref store, StorePaths storePaths) override { auto paths = store->topoSortPaths(storePathsToSet(storePaths)); diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index fed012b04..b45951879 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -23,6 +23,8 @@ struct CmdOptimiseStore : StoreCommand }; } + Category category() override { return catUtility; } + void run(ref store) override { store->optimiseStore(); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 45ec297d2..88d7fffd4 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -29,6 +29,8 @@ struct CmdPathInfo : StorePathsCommand, MixJSON return "query information about store paths"; } + Category category() override { return catSecondary; } + Examples examples() override { return { diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 3a2e542a3..127397a29 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -21,6 +21,8 @@ struct CmdPingStore : StoreCommand }; } + Category category() override { return catUtility; } + void run(ref store) override { store->connect(); diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 6104b10bc..4fd8886de 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -13,6 +13,8 @@ struct CmdShowConfig : Command, MixJSON return "show the Nix configuration"; } + Category category() override { return catUtility; } + void run() override { if (json) { diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index b6f24599f..22c569f3c 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -42,6 +42,8 @@ struct CmdShowDerivation : InstallablesCommand }; } + Category category() override { return catUtility; } + void run(ref store) override { auto drvPaths = toDerivations(store, installables, true); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index a91465c2a..6c9b9a792 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -27,6 +27,8 @@ struct CmdCopySigs : StorePathsCommand return "copy path signatures from substituters (like binary caches)"; } + Category category() override { return catUtility; } + void run(ref store, StorePaths storePaths) override { if (substituterUris.empty()) @@ -112,6 +114,8 @@ struct CmdSignPaths : StorePathsCommand return "sign the specified paths"; } + Category category() override { return catUtility; } + void run(ref store, StorePaths storePaths) override { if (secretKeyFile.empty()) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 32efcc3a7..678780f33 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -51,6 +51,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }; } + Category category() override { return catNixInstallation; } + void run(ref store) override { evalSettings.pureEval = true; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 08a36ac50..cf1fa6a99 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -49,6 +49,8 @@ struct CmdVerify : StorePathsCommand }; } + Category category() override { return catSecondary; } + void run(ref store, StorePaths storePaths) override { std::vector> substituters; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 36a3ee863..6057beedb 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -68,6 +68,8 @@ struct CmdWhyDepends : SourceExprCommand }; } + Category category() override { return catSecondary; } + void run(ref store) override { auto package = parseInstallable(*this, store, _package, false); From 909b4a882057a92ae8b40b8afdab3c4ac30da915 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2020 15:27:47 +0200 Subject: [PATCH 097/118] nix doctor: Consistency --- src/nix/doctor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index cc36354f1..82e92cdd0 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -40,7 +40,7 @@ struct CmdDoctor : StoreCommand std::string description() override { - return "check your system for potential problems and print a PASS or FAIL for each check."; + return "check your system for potential problems and print a PASS or FAIL for each check"; } Category category() override { return catNixInstallation; } From fd4911269f9a9ce0f9647c5fb88658074f654eff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 May 2020 10:54:01 +0200 Subject: [PATCH 098/118] Revert "Merge pull request #3558 from LnL7/ssh-ng-stderr" This reverts commit 3ebfbecdd187002569257f7cb183bf9e0b39af1e, reversing changes made to c089c52d5f1cff888552f485775b74226dcbe618. https://github.com/NixOS/nix/pull/3558 --- src/libstore/remote-store.cc | 6 ++---- src/libutil/logging.cc | 10 ---------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 07ef79382..8c55da268 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -779,10 +779,8 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return std::make_exception_ptr(Error(status, error)); } - else if (msg == STDERR_NEXT) { - string s = chomp(readString(from)); - printMsg(lvlVomit, "stderr %s", s); - } + else if (msg == STDERR_NEXT) + printError(chomp(readString(from))); else if (msg == STDERR_START_ACTIVITY) { auto act = readNum(from); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 777650de5..3cc4ef8f1 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -63,16 +63,6 @@ public: writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); } - void result(ActivityId act, ResultType type, const std::vector & fields) override - { - if (type == resBuildLogLine || type == resPostBuildLogLine) { - assert(0 < fields.size()); - assert(fields[0].type == Logger::Field::tString); - auto lastLine = fields[0].s; - log(lvlInfo, lastLine); - } - } - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) override From 9be46859a90d5a0771e079615d324644a08efb41 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Wed, 6 May 2020 11:21:12 +0200 Subject: [PATCH 099/118] libstore/build.cc: more explicit about form of output Be more explicit about why we expect a regular file as output when outputHashMode=flat for a fixed output derivation. --- src/libstore/build.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 572634765..e0c7324c7 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3678,7 +3678,8 @@ void DerivationGoal::registerOutputs() /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( - format("output path '%1%' should be a non-executable regular file") % path); + format("output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (outputHashMode=flat).") % path); } /* Check the hash. In hash mode, move the path produced by From 85c1932c9425c15d8d16da4777ed27667874a744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 5 May 2020 12:09:46 +0100 Subject: [PATCH 100/118] nix/search: no error for empty search results if json is enabled - result list will be always empty if --json is passed - for scripts an empty search result is not really an error, we rather want to distinguish between evaluation errors and empty results --- src/nix/search.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index fcad6be84..ba72c1e79 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -265,7 +265,7 @@ struct CmdSearch : SourceExprCommand, MixJSON throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); } - if (results.size() == 0) + if (!json && results.size() == 0) throw Error("no results for the given search term(s)!"); RunPager pager; From 58ed1e6d6842688ac2d32355265e0d89049f7e36 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Tue, 5 May 2020 17:56:46 +0200 Subject: [PATCH 101/118] WIP: add unit tests for libutil This is a proof on concept to evaluate writing unit tests for Nix using google test (https://github.com/google/googletest). In order to execute tests: $ make unit-tests $ ./unit-tests The Makefile rules for `unit-tests` is a complete hack. --- Makefile | 3 +- release-common.nix | 1 + tests/unit-tests/local.mk | 4 + tests/unit-tests/test-util.cc | 594 ++++++++++++++++++++++++++++++++++ 4 files changed, 601 insertions(+), 1 deletion(-) create mode 100644 tests/unit-tests/local.mk create mode 100644 tests/unit-tests/test-util.cc diff --git a/Makefile b/Makefile index e3057c36c..06de35a27 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,8 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ tests/local.mk \ - tests/plugins/local.mk + tests/plugins/local.mk \ + tests/unit-tests/local.mk -include Makefile.config diff --git a/release-common.nix b/release-common.nix index b6c8f3d81..7e7de005d 100644 --- a/release-common.nix +++ b/release-common.nix @@ -55,6 +55,7 @@ rec { # Tests git mercurial + gmock ] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium diff --git a/tests/unit-tests/local.mk b/tests/unit-tests/local.mk new file mode 100644 index 000000000..ed11261c1 --- /dev/null +++ b/tests/unit-tests/local.mk @@ -0,0 +1,4 @@ +LIBS=-larchive -lcrypto -llzma -lbz2 -lz -lbrotlienc -lbrotlidec +unit-tests: + echo $(nix_LDFLAGS) + $(CXX) -o unit-test $(nix_CXXFLAGS) $(nix_LDFLAGS) $(LIBS) --std=c++17 $$(pkg-config --libs gtest_main) ./src/libutil/*.o tests/unit-tests/test-util.cc diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc new file mode 100644 index 000000000..05d8a0039 --- /dev/null +++ b/tests/unit-tests/test-util.cc @@ -0,0 +1,594 @@ +#include "../../src/libutil/util.hh" +#include "../../src/libutil/types.hh" + +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * absPath + * --------------------------------------------------------------------------*/ + + TEST(absPath, doesntChangeRoot) { + auto p = absPath("/"); + + ASSERT_STREQ(p.c_str(), "/"); + } + + TEST(absPath, turnsEmptyPathIntoCWD) { + char cwd[PATH_MAX+1]; + auto p = absPath(""); + + ASSERT_STREQ(p.c_str(), getcwd((char*)&cwd, PATH_MAX)); + } + + TEST(absPath, usesOptionalBasePathWhenGiven) { + char _cwd[PATH_MAX+1]; + char* cwd = getcwd((char*)&_cwd, PATH_MAX); + + auto p = absPath("", cwd); + + ASSERT_STREQ(p.c_str(), cwd); + } + + TEST(absPath, isIdempotent) { + char _cwd[PATH_MAX+1]; + char* cwd = getcwd((char*)&_cwd, PATH_MAX); + auto p1 = absPath(cwd); + auto p2 = absPath(p1); + + ASSERT_STREQ(p1.c_str(), p2.c_str()); + } + + + TEST(absPath, pathIsCanonicalised) { + auto path = "/some/path/with/trailing/dot/."; + auto p1 = absPath(path); + auto p2 = absPath(p1); + + ASSERT_STREQ(p1.c_str(), "/some/path/with/trailing/dot"); + ASSERT_STREQ(p1.c_str(), p2.c_str()); + } + + /* ---------------------------------------------------------------------------- + * canonPath + * --------------------------------------------------------------------------*/ + + TEST(canonPath, removesTrailingSlashes) { + auto path = "/this/is/a/path//"; + auto p = canonPath(path); + + ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + } + + TEST(canonPath, removesDots) { + auto path = "/this/./is/a/path/./"; + auto p = canonPath(path); + + ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + } + + TEST(canonPath, removesDots2) { + auto path = "/this/a/../is/a////path/foo/.."; + auto p = canonPath(path); + + ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + } + + TEST(canonPath, requiresAbsolutePath) { + ASSERT_ANY_THROW(canonPath(".")); + ASSERT_ANY_THROW(canonPath("..")); + ASSERT_ANY_THROW(canonPath("../")); + ASSERT_DEATH({ canonPath(""); }, "path != \"\""); + } + + /* ---------------------------------------------------------------------------- + * dirOf + * --------------------------------------------------------------------------*/ + + // XXX: according to the doc of `dirOf`, dirOf("/") and dirOf("/foo") + // should both return "" but it actually returns "/" in both cases + TEST(dirOf, DISABLED_returnsEmptyStringForRoot) { + auto p = dirOf("/"); + + ASSERT_STREQ(p.c_str(), ""); + } + + TEST(dirOf, returnsFirstPathComponent) { + auto p1 = dirOf("/dir/"); + ASSERT_STREQ(p1.c_str(), "/dir"); + auto p2 = dirOf("/dir"); + ASSERT_STREQ(p2.c_str(), "/"); + auto p3 = dirOf("/dir/.."); + ASSERT_STREQ(p3.c_str(), "/dir"); + auto p4 = dirOf("/dir/../"); + ASSERT_STREQ(p4.c_str(), "/dir/.."); + } + + /* ---------------------------------------------------------------------------- + * baseNameOf + * --------------------------------------------------------------------------*/ + + TEST(baseNameOf, emptyPath) { + auto p1 = baseNameOf(""); + ASSERT_STREQ(std::string(p1).c_str(), ""); + } + + TEST(baseNameOf, pathOnRoot) { + auto p1 = baseNameOf("/dir"); + ASSERT_STREQ(std::string(p1).c_str(), "dir"); + } + + TEST(baseNameOf, relativePath) { + auto p1 = baseNameOf("dir/foo"); + ASSERT_STREQ(std::string(p1).c_str(), "foo"); + } + + TEST(baseNameOf, pathWithTrailingSlashRoot) { + auto p1 = baseNameOf("/"); + ASSERT_STREQ(std::string(p1).c_str(), ""); + } + + // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return + // "" but it actually returns "dir" + TEST(baseNameOf, DISABLED_trailingSlash) { + auto p1 = baseNameOf("/dir/"); + ASSERT_STREQ(std::string(p1).c_str(), ""); + } + + /* ---------------------------------------------------------------------------- + * isInDir + * --------------------------------------------------------------------------*/ + + TEST(isInDir, trivialCase) { + auto p1 = isInDir("/foo/bar", "/foo"); + ASSERT_EQ(p1, true); + } + + TEST(isInDir, notInDir) { + auto p1 = isInDir("/zes/foo/bar", "/foo"); + ASSERT_EQ(p1, false); + } + + // XXX: hm, bug or feature? :) Looking at the implementation + // this might be problematic. + TEST(isInDir, emptyDir) { + auto p1 = isInDir("/zes/foo/bar", ""); + ASSERT_EQ(p1, true); + } + + /* ---------------------------------------------------------------------------- + * isDirOrInDir + * --------------------------------------------------------------------------*/ + + TEST(isDirOrInDir, trueForSameDirectory) { + ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); + ASSERT_EQ(isDirOrInDir("/", "/"), true); + } + + TEST(isDirOrInDir, trueForEmptyPaths) { + ASSERT_EQ(isDirOrInDir("", ""), true); + } + + TEST(isDirOrInDir, falseForDisjunctPaths) { + ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); + } + + TEST(isDirOrInDir, relativePaths) { + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); + } + + // XXX: while it is possible to use "." or ".." in the + // first argument this doesn't seem to work in the second. + TEST(isDirOrInDir, DISABLED_shouldWork) { + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); + + } + + /* ---------------------------------------------------------------------------- + * pathExists + * --------------------------------------------------------------------------*/ + + TEST(pathExists, rootExists) { + ASSERT_TRUE(pathExists("/")); + } + + TEST(pathExists, cwdExists) { + ASSERT_TRUE(pathExists(".")); + } + + TEST(pathExists, bogusPathDoesNotExist) { + ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes")); + } + + /* ---------------------------------------------------------------------------- + * concatStringsSep + * --------------------------------------------------------------------------*/ + + TEST(concatStringsSep, buildCommaSeparatedString) { + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); + } + + TEST(concatStringsSep, buildStringWithEmptySeparator) { + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); + } + + TEST(concatStringsSep, buildSingleString) { + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); + } + + /* ---------------------------------------------------------------------------- + * hasPrefix + * --------------------------------------------------------------------------*/ + + TEST(hasPrefix, emptyStringHasNoPrefix) { + ASSERT_FALSE(hasPrefix("", "foo")); + } + + TEST(hasPrefix, emptyStringIsAlwaysPrefix) { + ASSERT_TRUE(hasPrefix("foo", "")); + ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); + } + + TEST(hasPrefix, trivialCase) { + ASSERT_TRUE(hasPrefix("foobar", "foo")); + } + + /* ---------------------------------------------------------------------------- + * hasSuffix + * --------------------------------------------------------------------------*/ + + TEST(hasSuffix, emptyStringHasNoSuffix) { + ASSERT_FALSE(hasSuffix("", "foo")); + } + + TEST(hasSuffix, trivialCase) { + ASSERT_TRUE(hasSuffix("foo", "foo")); + ASSERT_TRUE(hasSuffix("foobar", "bar")); + } + + /* ---------------------------------------------------------------------------- + * base64Encode + * --------------------------------------------------------------------------*/ + + TEST(base64Encode, emptyString) { + ASSERT_STREQ(base64Encode("").c_str(), ""); + } + + TEST(base64Encode, encodesAString) { + ASSERT_STREQ(base64Encode("quod erat demonstrandum").c_str(), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); + } + + TEST(base64Encode, encodeAndDecode) { + auto s = "quod erat demonstrandum"; + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + ASSERT_STREQ(decoded.c_str(), s); + } + + /* ---------------------------------------------------------------------------- + * base64Decode + * --------------------------------------------------------------------------*/ + + TEST(base64Decode, emptyString) { + ASSERT_STREQ(base64Decode("").c_str(), ""); + } + + TEST(base64Decode, decodeAString) { + ASSERT_STREQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=").c_str(), "quod erat demonstrandum"); + } + + /* ---------------------------------------------------------------------------- + * toLower + * --------------------------------------------------------------------------*/ + + TEST(toLower, emptyString) { + ASSERT_STREQ(toLower("").c_str(), ""); + } + + TEST(toLower, nonLetters) { + auto s = "!@(*$#)(@#=\\234_"; + ASSERT_STREQ(toLower(s).c_str(), s); + } + + // XXX: std::tolower() doesn't cover this. This probably doesn't matter + // since the context is paths and store paths specifically where such + // characters are not allowed. + TEST(toLower, DISABLED_umlauts) { + auto s = "ÄÖÜ"; + ASSERT_STREQ(toLower(s).c_str(), "äöü"); + } + + /* ---------------------------------------------------------------------------- + * string2Float + * --------------------------------------------------------------------------*/ + + TEST(string2Float, emptyString) { + double n; + ASSERT_EQ(string2Float("", n), false); + } + + TEST(string2Float, trivialConversions) { + double n; + ASSERT_EQ(string2Float("1.0", n), true); + ASSERT_EQ(n, 1.0); + + ASSERT_EQ(string2Float("0.0", n), true); + ASSERT_EQ(n, 0.0); + + ASSERT_EQ(string2Float("-100.25", n), true); + ASSERT_EQ(n, (-100.25)); + } + + /* ---------------------------------------------------------------------------- + * string2Int + * --------------------------------------------------------------------------*/ + + TEST(string2Int, emptyString) { + double n; + ASSERT_EQ(string2Int("", n), false); + } + + TEST(string2Int, trivialConversions) { + double n; + ASSERT_EQ(string2Int("1", n), true); + ASSERT_EQ(n, 1); + + ASSERT_EQ(string2Int("0", n), true); + ASSERT_EQ(n, 0); + + ASSERT_EQ(string2Int("-100", n), true); + ASSERT_EQ(n, (-100)); + } + + /* ---------------------------------------------------------------------------- + * statusOk + * --------------------------------------------------------------------------*/ + + TEST(statusOk, zeroIsOk) { + ASSERT_EQ(statusOk(0), true); + ASSERT_EQ(statusOk(1), false); + } + + /* ---------------------------------------------------------------------------- + * replaceInSet : XXX: this function isn't called anywhere! + * --------------------------------------------------------------------------*/ + + + /* ---------------------------------------------------------------------------- + * rewriteStrings + * --------------------------------------------------------------------------*/ + + TEST(rewriteStrings, emptyString) { + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_STREQ(rewriteStrings("", rewrites).c_str(), ""); + } + + TEST(rewriteStrings, emptyRewrites) { + StringMap rewrites; + + ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that"); + } + + TEST(rewriteStrings, successfulRewrite) { + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "that and that"); + } + + TEST(rewriteStrings, doesntOccur) { + StringMap rewrites; + rewrites["foo"] = "bar"; + + ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that"); + } + + /* ---------------------------------------------------------------------------- + * replaceStrings + * --------------------------------------------------------------------------*/ + + TEST(replaceStrings, emptyString) { + ASSERT_STREQ(replaceStrings("", "this", "that").c_str(), ""); + ASSERT_STREQ(replaceStrings("this and that", "", "").c_str(), "this and that"); + } + + TEST(replaceStrings, successfulReplace) { + ASSERT_STREQ(replaceStrings("this and that", "this", "that").c_str(), "that and that"); + } + + TEST(replaceStrings, doesntOccur) { + ASSERT_STREQ(replaceStrings("this and that", "foo", "bar").c_str(), "this and that"); + } + + /* ---------------------------------------------------------------------------- + * trim + * --------------------------------------------------------------------------*/ + + TEST(trim, emptyString) { + ASSERT_STREQ(trim("").c_str(), ""); + } + + TEST(trim, removesWhitespace) { + ASSERT_STREQ(trim("foo").c_str(), "foo"); + ASSERT_STREQ(trim(" foo ").c_str(), "foo"); + ASSERT_STREQ(trim(" foo bar baz").c_str(), "foo bar baz"); + ASSERT_STREQ(trim(" \t foo bar baz\n").c_str(), "foo bar baz"); + } + + /* ---------------------------------------------------------------------------- + * chomp + * --------------------------------------------------------------------------*/ + + TEST(chomp, emptyString) { + ASSERT_STREQ(chomp("").c_str(), ""); + } + + TEST(chomp, removesWhitespace) { + ASSERT_STREQ(chomp("foo").c_str(), "foo"); + ASSERT_STREQ(chomp("foo ").c_str(), "foo"); + ASSERT_STREQ(chomp(" foo ").c_str(), " foo"); + ASSERT_STREQ(chomp(" foo bar baz ").c_str(), " foo bar baz"); + ASSERT_STREQ(chomp("\t foo bar baz\n").c_str(), "\t foo bar baz"); + } + + /* ---------------------------------------------------------------------------- + * quoteStrings + * --------------------------------------------------------------------------*/ + + TEST(quoteStrings, empty) { + Strings s = { }; + Strings expected = { }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + TEST(quoteStrings, emptyStrings) { + Strings s = { "", "", "" }; + Strings expected = { "''", "''", "''" }; + ASSERT_EQ(quoteStrings(s), expected); + + } + + TEST(quoteStrings, trivialQuote) { + Strings s = { "foo", "bar", "baz" }; + Strings expected = { "'foo'", "'bar'", "'baz'" }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + TEST(quoteStrings, quotedStrings) { + Strings s = { "'foo'", "'bar'", "'baz'" }; + Strings expected = { "''foo''", "''bar''", "''baz''" }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + /* ---------------------------------------------------------------------------- + * tokenizeString + * --------------------------------------------------------------------------*/ + + TEST(tokenizeString, empty) { + auto s = ""; + Strings expected = { }; + + ASSERT_EQ(tokenizeString(""), expected); + } + + TEST(tokenizeString, tokenizeSpacesWithDefaults) { + auto s = "foo bar baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsWithDefaults) { + auto s = "foo\tbar\tbaz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) { + auto s = "foo\t bar\t baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) { + auto s = "foo\t\n bar\t\n baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) { + auto s = "foo\t\n\r bar\t\n\r baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s2), expected2); + } + + TEST(tokenizeString, tokenizeWithCustomSep) { + auto s = "foo\n,bar\n,baz\n"; + Strings expected = { "foo\n", "bar\n", "baz\n" }; + + ASSERT_EQ(tokenizeString(s, ","), expected); + } + + /* ---------------------------------------------------------------------------- + * get + * --------------------------------------------------------------------------*/ + + TEST(get, emptyContainer) { + StringMap s = { }; + auto expected = std::nullopt; + + ASSERT_EQ(get(s, "one"), expected); + } + + TEST(get, getFromContainer) { + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(get(s, "one"), expected); + } + + /* ---------------------------------------------------------------------------- + * filterANSIEscapes + * --------------------------------------------------------------------------*/ + + TEST(filterANSIEscapes, emptyString) { + auto s = ""; + auto expected = ""; + + ASSERT_STREQ(filterANSIEscapes(s).c_str(), expected); + } + + TEST(filterANSIEscapes, doesntChangePrintableChars) { + auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; + + ASSERT_STREQ(filterANSIEscapes(s).c_str(), s); + } + + TEST(filterANSIEscapes, filtersColorCodes) { + auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; + + ASSERT_STREQ(filterANSIEscapes(s, true, 2).c_str(), " A" ); + ASSERT_STREQ(filterANSIEscapes(s, true, 3).c_str(), " A " ); + ASSERT_STREQ(filterANSIEscapes(s, true, 4).c_str(), " A " ); + ASSERT_STREQ(filterANSIEscapes(s, true, 5).c_str(), " A B" ); + ASSERT_STREQ(filterANSIEscapes(s, true, 8).c_str(), " A B C" ); + } + + TEST(filterANSIEscapes, expandsTabs) { + auto s = "foo\tbar\tbaz"; + + ASSERT_STREQ(filterANSIEscapes(s, true).c_str(), "foo bar baz" ); + } + +} From 479e8bf00b680343231d2b618e130141079a2097 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 May 2020 16:08:15 +0200 Subject: [PATCH 102/118] Manual: Fix typo --- doc/manual/advanced-topics/post-build-hook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index 08a7a772f..57ee99e84 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -139,7 +139,7 @@ $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example Now, copy the path back from the cache: -$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... warning: you did not specify '--add-root'; the result might be removed by the garbage collector /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example From 41caaaad365b8e5260069c20458a2043254a6989 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 May 2020 16:37:33 +0200 Subject: [PATCH 103/118] Manual: Typo --- doc/manual/advanced-topics/post-build-hook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index 57ee99e84..acfe9e3cc 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -61,7 +61,7 @@ substituters = https://cache.nixos.org/ s3://example-nix-cache trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= -we will restart the Nix daemon a later step. +We will restart the Nix daemon in a later step.
From 987b3d6469f03bff59614bd3b9097df858d07263 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Thu, 7 May 2020 18:10:07 +0200 Subject: [PATCH 104/118] Use ASSERT_EQ instead of ASSERT_STREQ No need to use `c_str()` in combination with `ASSERT_STREQ`. It's possible to just use ASSERT_EQ on std::string --- tests/unit-tests/test-util.cc | 108 +++++++++++++++++----------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc index 05d8a0039..62ca8f11c 100644 --- a/tests/unit-tests/test-util.cc +++ b/tests/unit-tests/test-util.cc @@ -12,14 +12,14 @@ namespace nix { TEST(absPath, doesntChangeRoot) { auto p = absPath("/"); - ASSERT_STREQ(p.c_str(), "/"); + ASSERT_EQ(p, "/"); } TEST(absPath, turnsEmptyPathIntoCWD) { char cwd[PATH_MAX+1]; auto p = absPath(""); - ASSERT_STREQ(p.c_str(), getcwd((char*)&cwd, PATH_MAX)); + ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); } TEST(absPath, usesOptionalBasePathWhenGiven) { @@ -28,7 +28,7 @@ namespace nix { auto p = absPath("", cwd); - ASSERT_STREQ(p.c_str(), cwd); + ASSERT_EQ(p, cwd); } TEST(absPath, isIdempotent) { @@ -37,7 +37,7 @@ namespace nix { auto p1 = absPath(cwd); auto p2 = absPath(p1); - ASSERT_STREQ(p1.c_str(), p2.c_str()); + ASSERT_EQ(p1, p2); } @@ -46,8 +46,8 @@ namespace nix { auto p1 = absPath(path); auto p2 = absPath(p1); - ASSERT_STREQ(p1.c_str(), "/some/path/with/trailing/dot"); - ASSERT_STREQ(p1.c_str(), p2.c_str()); + ASSERT_EQ(p1, "/some/path/with/trailing/dot"); + ASSERT_EQ(p1, p2); } /* ---------------------------------------------------------------------------- @@ -58,21 +58,21 @@ namespace nix { auto path = "/this/is/a/path//"; auto p = canonPath(path); - ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + ASSERT_EQ(p, "/this/is/a/path"); } TEST(canonPath, removesDots) { auto path = "/this/./is/a/path/./"; auto p = canonPath(path); - ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + ASSERT_EQ(p, "/this/is/a/path"); } TEST(canonPath, removesDots2) { auto path = "/this/a/../is/a////path/foo/.."; auto p = canonPath(path); - ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + ASSERT_EQ(p, "/this/is/a/path"); } TEST(canonPath, requiresAbsolutePath) { @@ -91,18 +91,18 @@ namespace nix { TEST(dirOf, DISABLED_returnsEmptyStringForRoot) { auto p = dirOf("/"); - ASSERT_STREQ(p.c_str(), ""); + ASSERT_EQ(p, ""); } TEST(dirOf, returnsFirstPathComponent) { auto p1 = dirOf("/dir/"); - ASSERT_STREQ(p1.c_str(), "/dir"); + ASSERT_EQ(p1, "/dir"); auto p2 = dirOf("/dir"); - ASSERT_STREQ(p2.c_str(), "/"); + ASSERT_EQ(p2, "/"); auto p3 = dirOf("/dir/.."); - ASSERT_STREQ(p3.c_str(), "/dir"); + ASSERT_EQ(p3, "/dir"); auto p4 = dirOf("/dir/../"); - ASSERT_STREQ(p4.c_str(), "/dir/.."); + ASSERT_EQ(p4, "/dir/.."); } /* ---------------------------------------------------------------------------- @@ -111,29 +111,29 @@ namespace nix { TEST(baseNameOf, emptyPath) { auto p1 = baseNameOf(""); - ASSERT_STREQ(std::string(p1).c_str(), ""); + ASSERT_EQ(std::string(p1), ""); } TEST(baseNameOf, pathOnRoot) { auto p1 = baseNameOf("/dir"); - ASSERT_STREQ(std::string(p1).c_str(), "dir"); + ASSERT_EQ(std::string(p1), "dir"); } TEST(baseNameOf, relativePath) { auto p1 = baseNameOf("dir/foo"); - ASSERT_STREQ(std::string(p1).c_str(), "foo"); + ASSERT_EQ(std::string(p1), "foo"); } TEST(baseNameOf, pathWithTrailingSlashRoot) { auto p1 = baseNameOf("/"); - ASSERT_STREQ(std::string(p1).c_str(), ""); + ASSERT_EQ(std::string(p1), ""); } // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return // "" but it actually returns "dir" TEST(baseNameOf, DISABLED_trailingSlash) { auto p1 = baseNameOf("/dir/"); - ASSERT_STREQ(std::string(p1).c_str(), ""); + ASSERT_EQ(std::string(p1), ""); } /* ---------------------------------------------------------------------------- @@ -265,11 +265,11 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(base64Encode, emptyString) { - ASSERT_STREQ(base64Encode("").c_str(), ""); + ASSERT_EQ(base64Encode(""), ""); } TEST(base64Encode, encodesAString) { - ASSERT_STREQ(base64Encode("quod erat demonstrandum").c_str(), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); + ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); } TEST(base64Encode, encodeAndDecode) { @@ -277,7 +277,7 @@ namespace nix { auto encoded = base64Encode(s); auto decoded = base64Decode(encoded); - ASSERT_STREQ(decoded.c_str(), s); + ASSERT_EQ(decoded, s); } /* ---------------------------------------------------------------------------- @@ -285,11 +285,11 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(base64Decode, emptyString) { - ASSERT_STREQ(base64Decode("").c_str(), ""); + ASSERT_EQ(base64Decode(""), ""); } TEST(base64Decode, decodeAString) { - ASSERT_STREQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=").c_str(), "quod erat demonstrandum"); + ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); } /* ---------------------------------------------------------------------------- @@ -297,12 +297,12 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(toLower, emptyString) { - ASSERT_STREQ(toLower("").c_str(), ""); + ASSERT_EQ(toLower(""), ""); } TEST(toLower, nonLetters) { auto s = "!@(*$#)(@#=\\234_"; - ASSERT_STREQ(toLower(s).c_str(), s); + ASSERT_EQ(toLower(s), s); } // XXX: std::tolower() doesn't cover this. This probably doesn't matter @@ -310,7 +310,7 @@ namespace nix { // characters are not allowed. TEST(toLower, DISABLED_umlauts) { auto s = "ÄÖÜ"; - ASSERT_STREQ(toLower(s).c_str(), "äöü"); + ASSERT_EQ(toLower(s), "äöü"); } /* ---------------------------------------------------------------------------- @@ -377,27 +377,27 @@ namespace nix { StringMap rewrites; rewrites["this"] = "that"; - ASSERT_STREQ(rewriteStrings("", rewrites).c_str(), ""); + ASSERT_EQ(rewriteStrings("", rewrites), ""); } TEST(rewriteStrings, emptyRewrites) { StringMap rewrites; - ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that"); + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); } TEST(rewriteStrings, successfulRewrite) { StringMap rewrites; rewrites["this"] = "that"; - ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "that and that"); + ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); } TEST(rewriteStrings, doesntOccur) { StringMap rewrites; rewrites["foo"] = "bar"; - ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that"); + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); } /* ---------------------------------------------------------------------------- @@ -405,16 +405,16 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(replaceStrings, emptyString) { - ASSERT_STREQ(replaceStrings("", "this", "that").c_str(), ""); - ASSERT_STREQ(replaceStrings("this and that", "", "").c_str(), "this and that"); + ASSERT_EQ(replaceStrings("", "this", "that"), ""); + ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); } TEST(replaceStrings, successfulReplace) { - ASSERT_STREQ(replaceStrings("this and that", "this", "that").c_str(), "that and that"); + ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); } TEST(replaceStrings, doesntOccur) { - ASSERT_STREQ(replaceStrings("this and that", "foo", "bar").c_str(), "this and that"); + ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); } /* ---------------------------------------------------------------------------- @@ -422,14 +422,14 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(trim, emptyString) { - ASSERT_STREQ(trim("").c_str(), ""); + ASSERT_EQ(trim(""), ""); } TEST(trim, removesWhitespace) { - ASSERT_STREQ(trim("foo").c_str(), "foo"); - ASSERT_STREQ(trim(" foo ").c_str(), "foo"); - ASSERT_STREQ(trim(" foo bar baz").c_str(), "foo bar baz"); - ASSERT_STREQ(trim(" \t foo bar baz\n").c_str(), "foo bar baz"); + ASSERT_EQ(trim("foo"), "foo"); + ASSERT_EQ(trim(" foo "), "foo"); + ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); + ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); } /* ---------------------------------------------------------------------------- @@ -437,15 +437,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(chomp, emptyString) { - ASSERT_STREQ(chomp("").c_str(), ""); + ASSERT_EQ(chomp(""), ""); } TEST(chomp, removesWhitespace) { - ASSERT_STREQ(chomp("foo").c_str(), "foo"); - ASSERT_STREQ(chomp("foo ").c_str(), "foo"); - ASSERT_STREQ(chomp(" foo ").c_str(), " foo"); - ASSERT_STREQ(chomp(" foo bar baz ").c_str(), " foo bar baz"); - ASSERT_STREQ(chomp("\t foo bar baz\n").c_str(), "\t foo bar baz"); + ASSERT_EQ(chomp("foo"), "foo"); + ASSERT_EQ(chomp("foo "), "foo"); + ASSERT_EQ(chomp(" foo "), " foo"); + ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); + ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); } /* ---------------------------------------------------------------------------- @@ -566,29 +566,29 @@ namespace nix { auto s = ""; auto expected = ""; - ASSERT_STREQ(filterANSIEscapes(s).c_str(), expected); + ASSERT_EQ(filterANSIEscapes(s), expected); } TEST(filterANSIEscapes, doesntChangePrintableChars) { auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; - ASSERT_STREQ(filterANSIEscapes(s).c_str(), s); + ASSERT_EQ(filterANSIEscapes(s), s); } TEST(filterANSIEscapes, filtersColorCodes) { auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; - ASSERT_STREQ(filterANSIEscapes(s, true, 2).c_str(), " A" ); - ASSERT_STREQ(filterANSIEscapes(s, true, 3).c_str(), " A " ); - ASSERT_STREQ(filterANSIEscapes(s, true, 4).c_str(), " A " ); - ASSERT_STREQ(filterANSIEscapes(s, true, 5).c_str(), " A B" ); - ASSERT_STREQ(filterANSIEscapes(s, true, 8).c_str(), " A B C" ); + ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" ); + ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " ); + ASSERT_EQ(filterANSIEscapes(s, true, 4), " A " ); + ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B" ); + ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C" ); } TEST(filterANSIEscapes, expandsTabs) { auto s = "foo\tbar\tbaz"; - ASSERT_STREQ(filterANSIEscapes(s, true).c_str(), "foo bar baz" ); + ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); } } From 1f3602a2c9c3bfeea7cc8ee4d478c77dec349206 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Thu, 7 May 2020 18:15:13 +0200 Subject: [PATCH 105/118] Remove replaceInSet The function isn't being used anywhere so it seems safe to remove --- src/libutil/util.hh | 11 ----------- tests/unit-tests/test-util.cc | 4 ---- 2 files changed, 15 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 1b263abcc..8770add64 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -389,17 +389,6 @@ string replaceStrings(const std::string & s, std::string rewriteStrings(const std::string & s, const StringMap & rewrites); -/* If a set contains 'from', remove it and insert 'to'. */ -template -void replaceInSet(std::set & set, const T & from, const T & to) -{ - auto i = set.find(from); - if (i == set.end()) return; - set.erase(i); - set.insert(to); -} - - /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc index 62ca8f11c..e0738614e 100644 --- a/tests/unit-tests/test-util.cc +++ b/tests/unit-tests/test-util.cc @@ -364,10 +364,6 @@ namespace nix { ASSERT_EQ(statusOk(1), false); } - /* ---------------------------------------------------------------------------- - * replaceInSet : XXX: this function isn't called anywhere! - * --------------------------------------------------------------------------*/ - /* ---------------------------------------------------------------------------- * rewriteStrings From 73d0b5d807edee05910cff4a30549393d5375d89 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Thu, 7 May 2020 19:29:10 +0200 Subject: [PATCH 106/118] Drop unnecessary std::string --- tests/unit-tests/test-util.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc index e0738614e..a0fa7c109 100644 --- a/tests/unit-tests/test-util.cc +++ b/tests/unit-tests/test-util.cc @@ -111,29 +111,29 @@ namespace nix { TEST(baseNameOf, emptyPath) { auto p1 = baseNameOf(""); - ASSERT_EQ(std::string(p1), ""); + ASSERT_EQ(p1, ""); } TEST(baseNameOf, pathOnRoot) { auto p1 = baseNameOf("/dir"); - ASSERT_EQ(std::string(p1), "dir"); + ASSERT_EQ(p1, "dir"); } TEST(baseNameOf, relativePath) { auto p1 = baseNameOf("dir/foo"); - ASSERT_EQ(std::string(p1), "foo"); + ASSERT_EQ(p1, "foo"); } TEST(baseNameOf, pathWithTrailingSlashRoot) { auto p1 = baseNameOf("/"); - ASSERT_EQ(std::string(p1), ""); + ASSERT_EQ(p1, ""); } // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return // "" but it actually returns "dir" TEST(baseNameOf, DISABLED_trailingSlash) { auto p1 = baseNameOf("/dir/"); - ASSERT_EQ(std::string(p1), ""); + ASSERT_EQ(p1, ""); } /* ---------------------------------------------------------------------------- From 7cc7cef9502dc388706561cbdf275e070520e65d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 11:34:09 +0200 Subject: [PATCH 107/118] Move unit tests to sr/libutil/tests, use mk make rules --- Makefile | 4 ++-- src/libutil/tests/local.mk | 11 +++++++++++ .../test-util.cc => src/libutil/tests/tests.cc | 4 ++-- tests/unit-tests/local.mk | 4 ---- 4 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 src/libutil/tests/local.mk rename tests/unit-tests/test-util.cc => src/libutil/tests/tests.cc (99%) delete mode 100644 tests/unit-tests/local.mk diff --git a/Makefile b/Makefile index 06de35a27..0ba011e2a 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ makefiles = \ local.mk \ nix-rust/local.mk \ src/libutil/local.mk \ + src/libutil/tests/local.mk \ src/libstore/local.mk \ src/libfetchers/local.mk \ src/libmain/local.mk \ @@ -16,8 +17,7 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ tests/local.mk \ - tests/plugins/local.mk \ - tests/unit-tests/local.mk + tests/plugins/local.mk -include Makefile.config diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk new file mode 100644 index 000000000..9c2d2a7bb --- /dev/null +++ b/src/libutil/tests/local.mk @@ -0,0 +1,11 @@ +programs += libutil-tests + +libutil-tests_DIR := $(d) + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_CXXFLAGS += -I src/libutil + +libutil-tests_LIBS = libutil + +libutil-tests_LDFLAGS := $$(pkg-config --libs gtest_main) diff --git a/tests/unit-tests/test-util.cc b/src/libutil/tests/tests.cc similarity index 99% rename from tests/unit-tests/test-util.cc rename to src/libutil/tests/tests.cc index a0fa7c109..fb593b5e6 100644 --- a/tests/unit-tests/test-util.cc +++ b/src/libutil/tests/tests.cc @@ -1,5 +1,5 @@ -#include "../../src/libutil/util.hh" -#include "../../src/libutil/types.hh" +#include "util.hh" +#include "types.hh" #include diff --git a/tests/unit-tests/local.mk b/tests/unit-tests/local.mk deleted file mode 100644 index ed11261c1..000000000 --- a/tests/unit-tests/local.mk +++ /dev/null @@ -1,4 +0,0 @@ -LIBS=-larchive -lcrypto -llzma -lbz2 -lz -lbrotlienc -lbrotlidec -unit-tests: - echo $(nix_LDFLAGS) - $(CXX) -o unit-test $(nix_CXXFLAGS) $(nix_LDFLAGS) $(LIBS) --std=c++17 $$(pkg-config --libs gtest_main) ./src/libutil/*.o tests/unit-tests/test-util.cc From 72b9d971bc4ff5b5dcb9349d03526a5ef3e4300e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 11:35:57 +0200 Subject: [PATCH 108/118] Fix warning --- src/libutil/tests/tests.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index fb593b5e6..0fb4411e8 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -481,8 +481,7 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(tokenizeString, empty) { - auto s = ""; - Strings expected = { }; + Strings expected = { }; ASSERT_EQ(tokenizeString(""), expected); } From 7898cdb75a721cc21867ccdbcc523e95f2db0354 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 11:49:40 +0200 Subject: [PATCH 109/118] make check: Run unit tests --- mk/programs.mk | 6 ++++++ mk/tracing.mk | 1 + nix-rust/local.mk | 2 +- src/libutil/tests/local.mk | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mk/programs.mk b/mk/programs.mk index d93df4468..2d0c988d4 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -76,4 +76,10 @@ define build-program programs-list += $$($(1)_PATH) clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) dist-files += $$(_srcs) + + # Phony target to run this program (typically as a dependency of 'check'). + .PHONY: $(1)_RUN + $(1)_RUN: $$($(1)_PATH) + $(trace-test) $$($(1)_PATH) + endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 13912d3c7..54c77ab60 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -11,6 +11,7 @@ ifeq ($(V), 0) trace-javac = @echo " JAVAC " $@; trace-jar = @echo " JAR " $@; trace-mkdir = @echo " MKDIR " $@; + trace-test = @echo " TEST " $@; suppress = @ diff --git a/nix-rust/local.mk b/nix-rust/local.mk index 1e006e500..e4bfde31b 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -41,5 +41,5 @@ ifneq ($(OS), Darwin) check: rust-tests rust-tests: - cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi) + $(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi) endif diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 9c2d2a7bb..a188d5093 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -1,3 +1,5 @@ +check: libutil-tests_RUN + programs += libutil-tests libutil-tests_DIR := $(d) From ca657525b88e788b72ce53ecc6d140a573476644 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 12:03:27 +0200 Subject: [PATCH 110/118] Don't install unit tests --- mk/programs.mk | 20 ++++++++++++-------- src/libutil/tests/local.mk | 2 ++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/mk/programs.mk b/mk/programs.mk index 2d0c988d4..3fa9685c3 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -35,24 +35,28 @@ define build-program $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) - $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) + ifdef $(1)_INSTALL_DIR - install: $(DESTDIR)$$($(1)_INSTALL_PATH) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) - ifeq ($(BUILD_SHARED_LIBS), 1) + $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) - _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) + install: $(DESTDIR)$$($(1)_INSTALL_PATH) - $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + ifeq ($(BUILD_SHARED_LIBS), 1) + + _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) + + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) - else + else - $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< + endif endif # Propagate CFLAGS and CXXFLAGS to the individual object files. diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index a188d5093..5f4b3edc5 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -4,6 +4,8 @@ programs += libutil-tests libutil-tests_DIR := $(d) +libutil-tests_INSTALL_DIR := + libutil-tests_SOURCES := $(wildcard $(d)/*.cc) libutil-tests_CXXFLAGS += -I src/libutil From 5b8883faac0cff928a4359b4260b5be09a311d4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 12:09:00 +0200 Subject: [PATCH 111/118] configure: Look for gtest --- Makefile.config.in | 19 ++++++++++--------- configure.ac | 4 ++++ src/libutil/tests/local.mk | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index e7a12089a..b632444e8 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,36 +1,38 @@ AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ +BOOST_LDFLAGS = @BOOST_LDFLAGS@ BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ -LDFLAGS = @LDFLAGS@ +EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ -HAVE_SODIUM = @HAVE_SODIUM@ +GTEST_LIBS = @GTEST_LIBS@ HAVE_SECCOMP = @HAVE_SECCOMP@ -BOOST_LDFLAGS = @BOOST_LDFLAGS@ +HAVE_SODIUM = @HAVE_SODIUM@ +LDFLAGS = @LDFLAGS@ +LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ +LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBLZMA_LIBS = @LIBLZMA_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SODIUM_LIBS = @SODIUM_LIBS@ -LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ -LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ -LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ -EDITLINE_LIBS = @EDITLINE_LIBS@ bash = @bash@ bindir = @bindir@ -lsof = @lsof@ datadir = @datadir@ datarootdir = @datarootdir@ +doc_generate = @doc_generate@ docdir = @docdir@ exec_prefix = @exec_prefix@ includedir = @includedir@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ +lsof = @lsof@ mandir = @mandir@ pkglibdir = $(libdir)/$(PACKAGE_NAME) prefix = @prefix@ @@ -38,6 +40,5 @@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ system = @system@ -doc_generate = @doc_generate@ xmllint = @xmllint@ xsltproc = @xsltproc@ diff --git a/configure.ac b/configure.ac index b868343d2..c3007b4b6 100644 --- a/configure.ac +++ b/configure.ac @@ -266,6 +266,10 @@ if test "$gc" = yes; then fi +# Look for gtest. +PKG_CHECK_MODULES([GTEST], [gtest_main]) + + # documentation generation switch AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], [disable documentation generation]), diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 5f4b3edc5..a297edb64 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -12,4 +12,4 @@ libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil -libutil-tests_LDFLAGS := $$(pkg-config --libs gtest_main) +libutil-tests_LDFLAGS := $(GTEST_LIBS) From e3df9c2a6e867d0e038cb26af40501511607e007 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Fri, 8 May 2020 15:03:44 +0200 Subject: [PATCH 112/118] Enable `dirOf` test Adjusted the doc comment for `dirOf` to reflect the implementation behavior. --- src/libutil/tests/tests.cc | 6 ++---- src/libutil/util.hh | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 0fb4411e8..1b016430d 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -86,12 +86,10 @@ namespace nix { * dirOf * --------------------------------------------------------------------------*/ - // XXX: according to the doc of `dirOf`, dirOf("/") and dirOf("/foo") - // should both return "" but it actually returns "/" in both cases - TEST(dirOf, DISABLED_returnsEmptyStringForRoot) { + TEST(dirOf, returnsEmptyStringForRoot) { auto p = dirOf("/"); - ASSERT_EQ(p, ""); + ASSERT_EQ(p, "/"); } TEST(dirOf, returnsFirstPathComponent) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8770add64..38a0f7a5c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -58,8 +58,8 @@ Path canonPath(const Path & path, bool resolveSymlinks = false); /* Return the directory part of the given canonical path, i.e., everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means an empty string - is returned. */ + immediate child thereof (e.g., `/foo'), this means `/' + is returned.*/ Path dirOf(const Path & path); /* Return the base name of the given canonical path, i.e., everything From 2191141274cfefcb9ffbafbbfcdc58414fb28f1a Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Fri, 8 May 2020 15:07:40 +0200 Subject: [PATCH 113/118] Enable `baseNameOf` test Add note about removal of trailing slashes in the doc comment of baseNameOf and enabled the test. --- src/libutil/tests/tests.cc | 6 ++---- src/libutil/util.hh | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 1b016430d..d46f6a5a4 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -127,11 +127,9 @@ namespace nix { ASSERT_EQ(p1, ""); } - // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return - // "" but it actually returns "dir" - TEST(baseNameOf, DISABLED_trailingSlash) { + TEST(baseNameOf, trailingSlash) { auto p1 = baseNameOf("/dir/"); - ASSERT_EQ(p1, ""); + ASSERT_EQ(p1, "dir"); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 38a0f7a5c..a63ee05b3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -63,7 +63,7 @@ Path canonPath(const Path & path, bool resolveSymlinks = false); Path dirOf(const Path & path); /* Return the base name of the given canonical path, i.e., everything - following the final `/'. */ + following the final `/' (trailing slashes are removed). */ std::string_view baseNameOf(std::string_view path); /* Check whether 'path' is a descendant of 'dir'. */ From 181a47d8845e06edc9653e99de589376a1a96bb6 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Fri, 8 May 2020 15:13:55 +0200 Subject: [PATCH 114/118] Enable toLower umlauts test Update comment and enable the test --- src/libutil/tests/tests.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index d46f6a5a4..5abfe2c9c 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -301,12 +301,12 @@ namespace nix { ASSERT_EQ(toLower(s), s); } - // XXX: std::tolower() doesn't cover this. This probably doesn't matter - // since the context is paths and store paths specifically where such - // characters are not allowed. - TEST(toLower, DISABLED_umlauts) { + // std::tolower() doesn't handle unicode characters. In the context of + // store paths this isn't relevant but doesn't hurt to record this behavior + // here. + TEST(toLower, umlauts) { auto s = "ÄÖÜ"; - ASSERT_EQ(toLower(s), "äöü"); + ASSERT_EQ(toLower(s), "ÄÖÜ"); } /* ---------------------------------------------------------------------------- From 52cffafd2462c03466db15909dde250f07677d35 Mon Sep 17 00:00:00 2001 From: Dani Date: Sat, 9 May 2020 13:48:31 +0200 Subject: [PATCH 115/118] Fix typo --- doc/manual/release-notes/rl-0.8.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/release-notes/rl-0.8.xml b/doc/manual/release-notes/rl-0.8.xml index 784b26c6b..825798fa9 100644 --- a/doc/manual/release-notes/rl-0.8.xml +++ b/doc/manual/release-notes/rl-0.8.xml @@ -8,7 +8,7 @@ NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). As a result, nix-pull manifests and channels built -for Nix 0.7 and below will now work anymore. However, the Nix +for Nix 0.7 and below will not work anymore. However, the Nix expression language has not changed, so you can still build from source. Also, existing user environments continue to work. Nix 0.8 will automatically upgrade the database schema of previous From 446649e5403f9c41601eee6438eaeed5212b9190 Mon Sep 17 00:00:00 2001 From: Shao Cheng Date: Sat, 9 May 2020 15:59:39 +0200 Subject: [PATCH 116/118] Update "Upgrading Nix" documentation This PR proposes two changes to the "Upgrading Nix" documentation: * Besides updating `nixpkgs.nix`, we also update `nixpkgs.cacert`, so that the certificates are up-to-date as well. * Add the instructions for multi-user mode on Linux. --- doc/manual/installation/upgrading.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/manual/installation/upgrading.xml b/doc/manual/installation/upgrading.xml index 30670d7fe..592f63895 100644 --- a/doc/manual/installation/upgrading.xml +++ b/doc/manual/installation/upgrading.xml @@ -17,6 +17,11 @@ Single-user installations of Nix should run this: - nix-channel --update; nix-env -iA nixpkgs.nix + nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert + + + + Multi-user Nix users on Linux should run this with sudo: + nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon From 146f9c114ff347d372b5ab6c5fc00fdd3f2895da Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sat, 9 May 2020 10:58:43 -0400 Subject: [PATCH 117/118] doc: consistently refer to 'fixed-output' with a dash General cleanup that makes it easier to search for the term. --- doc/manual/command-ref/nix-store.xml | 2 +- src/libstore/build.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 1ddb5408d..d71f9d8e4 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -936,7 +936,7 @@ $ nix-store --add ./foo.c The operation adds the specified paths to the Nix store. Unlike paths are registered using the -specified hashing algorithm, resulting in the same output path as a fixed output +specified hashing algorithm, resulting in the same output path as a fixed-output derivation. This can be used for sources that are not available from a public url or broke since the download expression was written. diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 147093fae..6258bcb33 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3155,7 +3155,7 @@ void DerivationGoal::runChild() // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the - // potential impurities introduced in fixed outputs. + // potential impurities introduced in fixed-outputs. writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); ss.push_back("/etc/services"); From 1d8144e36b18d3467f1b29b80800ed3eb9b876c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 11 May 2020 18:14:23 +0200 Subject: [PATCH 118/118] Update src/libstore/build.cc --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e0c7324c7..f9b701910 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3679,7 +3679,7 @@ void DerivationGoal::registerOutputs() if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( format("output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat).") % path); + "since recursive hashing is not enabled (outputHashMode=flat)") % path); } /* Check the hash. In hash mode, move the path produced by