From c38c726eb5d447c7e9d894d57cd05ac46c173ddd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 21:15:45 +0200 Subject: [PATCH 1/5] Fix test failures when $TMPDIR changes --- tests/build-dry.sh | 6 +++--- tests/nix-copy-ssh.sh | 2 +- tests/nix-shell.sh | 8 ++++---- tests/placeholders.sh | 2 -- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/build-dry.sh b/tests/build-dry.sh index 610e6070c..e72533e70 100644 --- a/tests/build-dry.sh +++ b/tests/build-dry.sh @@ -8,13 +8,13 @@ clearStore clearCache # Ensure this builds successfully first -nix build -f dependencies.nix +nix build --no-link -f dependencies.nix clearStore clearCache # Try --dry-run using old command first -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" # Now new command: nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" @@ -27,7 +27,7 @@ clearCache # Try --dry-run using new command first nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" # Now old command: -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" fi ################################################### diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh index 6aba667a4..eb801548d 100644 --- a/tests/nix-copy-ssh.sh +++ b/tests/nix-copy-ssh.sh @@ -7,7 +7,7 @@ remoteRoot=$TEST_ROOT/store2 chmod -R u+w "$remoteRoot" || true rm -rf "$remoteRoot" -outPath=$(nix-build dependencies.nix) +outPath=$(nix-build --no-out-link dependencies.nix) nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 6024ea399..ee502dddb 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -27,13 +27,13 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run # Test nix-shell on a .drv symlink # Legacy: absolute path and .drv extension required -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv -[[ $(nix-shell --pure $PWD/shell.drv --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv +[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # New behaviour: just needs to resolve to a derivation in the store -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell -[[ $(nix-shell --pure shell --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell +[[ $(nix-shell --pure $TEST_ROOT/shell --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p diff --git a/tests/placeholders.sh b/tests/placeholders.sh index 071cfe2dc..cd1bb7bc2 100644 --- a/tests/placeholders.sh +++ b/tests/placeholders.sh @@ -18,5 +18,3 @@ nix-build --no-out-link -E ' "; } ' - -echo XYZZY From 3c171851a8aed22e977a1798dc5d306faa6e5b63 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 22:29:16 +0200 Subject: [PATCH 2/5] Make the URL/path of the global flake registry configurable --- src/libexpr/eval.cc | 8 ++++++++ src/libexpr/eval.hh | 5 +++++ src/libexpr/primops/flake.cc | 3 +-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4bd1280ad..2d83af983 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1972,6 +1972,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { EvalSettings evalSettings; +EvalSettings::EvalSettings() +{ + if (flakeRegistry == "") + // FIXME: static initialization order fiasco. But this will go + // away when we switch to an online registry. + flakeRegistry = settings.nixDataDir + "/nix/flake-registry.json"; +} + static GlobalConfig::Register r1(&evalSettings); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 44988cd70..b0bf777fc 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -367,6 +367,11 @@ struct EvalSettings : Config Setting allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting flakeRegistry{this, "", "flake-registry", + "Path or URI of the global flake registry."}; + + EvalSettings(); }; extern EvalSettings evalSettings; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 89267e2b1..f73ab4424 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -126,8 +126,7 @@ void writeLockFile(const LockFile & lockFile, const Path & path) std::shared_ptr getGlobalRegistry() { - Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; - return readRegistry(registryFile); + return readRegistry(evalSettings.flakeRegistry); } Path getUserRegistryPath() From 2a41a567e29846cf32d38f338b992069592770c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 23:06:15 +0200 Subject: [PATCH 3/5] Improve FlakeRef::to_string() We were incorrectly using path syntax (i.e. //) for Git repositories. This is only valid for GitHub flakerefs. --- src/libexpr/primops/flakeref.cc | 42 +++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 141d61c0d..784a0868e 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -147,31 +147,53 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::string FlakeRef::to_string() const { std::string string; + bool first = true; - if (auto refData = std::get_if(&data)) + auto addParam = + [&](const std::string & name, std::string value) { + string += first ? '?' : '&'; + first = false; + string += name; + string += '='; + string += value; // FIXME: escaping + }; + + if (auto refData = std::get_if(&data)) { string = refData->alias; + if (ref) string += '/' + *ref; + if (rev) string += '/' + rev->to_string(Base16, false); + } + + else if (auto refData = std::get_if(&data)) { + assert(subdir == ""); + assert(!rev); + assert(!ref); + return refData->path; + } else if (auto refData = std::get_if(&data)) { assert(!(ref && rev)); string = "github:" + refData->owner + "/" + refData->repo; + if (ref) { string += '/'; string += *ref; } + if (rev) { string += '/'; string += rev->to_string(Base16, false); } + if (subdir != "") addParam("dir", subdir); } else if (auto refData = std::get_if(&data)) { assert(!rev || ref); string = refData->uri; + + if (ref) { + addParam("ref", *ref); + if (rev) + addParam("rev", rev->to_string(Base16, false)); + } + + if (subdir != "") addParam("dir", subdir); } - else if (auto refData = std::get_if(&data)) - return refData->path; - else abort(); - // FIXME: need to use ?rev etc. for IsGit URIs. - string += (ref ? "/" + *ref : "") + - (rev ? "/" + rev->to_string(Base16, false) : ""); - - if (subdir != "") string += "?dir=" + subdir; - return string; } From ddd42b7e948acc1b541741250c9a77e86aaada04 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 23:20:42 +0200 Subject: [PATCH 4/5] Fix immutable flakeref construction We were appending ref/revs incorrectly for the IsGit case (by appending // rather than ?ref=&rev=size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); - std::string rev = std::string(*result.etag, 1, result.etag->size() - 2); - const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + rev); + FlakeRef ref(resolvedRef.baseRef()); + ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); SourceInfo info(ref); info.storePath = result.path; @@ -247,7 +247,9 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool // This downloads the entire git history else if (auto refData = std::get_if(&resolvedRef.data)) { auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"); - const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false)); + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; SourceInfo info(ref); info.storePath = gitInfo.storePath; info.revCount = gitInfo.revCount; @@ -258,7 +260,9 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source"); - const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false)); + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; SourceInfo info(ref); info.storePath = gitInfo.storePath; info.revCount = gitInfo.revCount; From 2d5a21968842c88b425e1a591bd413c484a470e7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 23:22:47 +0200 Subject: [PATCH 5/5] Add basic flake tests --- Makefile.config.in | 1 + configure.ac | 1 + release-common.nix | 1 + tests/{config.nix => config.nix.in} | 2 +- tests/flakes.sh | 117 ++++++++++++++++++++++++++++ tests/local.mk | 8 +- 6 files changed, 126 insertions(+), 4 deletions(-) rename tests/{config.nix => config.nix.in} (92%) create mode 100644 tests/flakes.sh diff --git a/Makefile.config.in b/Makefile.config.in index 59730b646..f2273eaed 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -35,6 +35,7 @@ prefix = @prefix@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ +system = @system@ doc_generate = @doc_generate@ xmllint = @xmllint@ xsltproc = @xsltproc@ diff --git a/configure.ac b/configure.ac index f5b1614f1..8087f308b 100644 --- a/configure.ac +++ b/configure.ac @@ -129,6 +129,7 @@ NEED_PROG(gzip, gzip) NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) +NEED_PROG(jq, jq) NEED_PROG(cat, cat) diff --git a/release-common.nix b/release-common.nix index 4c5565985..b745932a7 100644 --- a/release-common.nix +++ b/release-common.nix @@ -56,6 +56,7 @@ rec { # Tests git mercurial + jq ] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium diff --git a/tests/config.nix b/tests/config.nix.in similarity index 92% rename from tests/config.nix rename to tests/config.nix.in index 6ba91065b..ff5aeb31a 100644 --- a/tests/config.nix +++ b/tests/config.nix.in @@ -5,7 +5,7 @@ rec { path = coreutils; - system = builtins.currentSystem; + system = "@system@"; shared = builtins.getEnv "_NIX_TEST_SHARED"; diff --git a/tests/flakes.sh b/tests/flakes.sh new file mode 100644 index 000000000..26ffd6a9f --- /dev/null +++ b/tests/flakes.sh @@ -0,0 +1,117 @@ +source common.sh + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping flake tests" + exit 99 +fi + +clearStore + +registry=$TEST_ROOT/registry.json + +flake1=$TEST_ROOT/flake1 +flake2=$TEST_ROOT/flake2 +flake3=$TEST_ROOT/flake3 + +for repo in $flake1 $flake2 $flake3; do + rm -rf $repo + mkdir $repo + git -C $repo init + git -C $repo config user.email "foobar@example.com" + git -C $repo config user.name "Foobar" +done + +cat > $flake1/flake.nix < $flake2/flake.nix < $registry <