Merge remote-tracking branch 'tweag/flake-test' into flakes

Makefile.config.in

@@ -35,6 +35,7 @@ prefix = @prefix@
 sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
+system = @system@
 doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

configure.ac

@@ -129,6 +129,7 @@ NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
+NEED_PROG(jq, jq)
 
 
 NEED_PROG(cat, cat)

release-common.nix

@@ -56,6 +56,7 @@ rec {
       # Tests
       git
       mercurial
+      jq
     ]
     ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
     ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium

src/libexpr/eval.cc

@@ -1972,6 +1972,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 
 EvalSettings evalSettings;
 
+EvalSettings::EvalSettings()
+{
+    if (flakeRegistry == "")
+        // FIXME: static initialization order fiasco. But this will go
+        // away when we switch to an online registry.
+        flakeRegistry = settings.nixDataDir + "/nix/flake-registry.json";
+}
+
 static GlobalConfig::Register r1(&evalSettings);
 
 

src/libexpr/eval.hh

@@ -367,6 +367,11 @@ struct EvalSettings : Config
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+
+    Setting<std::string> flakeRegistry{this, "", "flake-registry",
+        "Path or URI of the global flake registry."};
+
+    EvalSettings();
 };
 
 extern EvalSettings evalSettings;

src/libexpr/primops/flake.cc

@@ -48,7 +48,7 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json)
 {
     FlakeRef flakeRef(json["uri"]);
     if (!flakeRef.isImmutable())
-        throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef);
+        throw Error("cannot use mutable flake '%s' in pure mode", flakeRef);
 
     LockFile::FlakeEntry entry(flakeRef);
 
@@ -126,8 +126,7 @@ void writeLockFile(const LockFile & lockFile, const Path & path)
 
 std::shared_ptr<FlakeRegistry> getGlobalRegistry()
 {
-    Path registryFile = settings.nixDataDir + "/nix/flake-registry.json";
+    return readRegistry(evalSettings.flakeRegistry);
-    return readRegistry(registryFile);
 }
 
 Path getUserRegistryPath()
@@ -237,8 +236,8 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
         if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"')
             throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url);
 
-        std::string rev = std::string(*result.etag, 1, result.etag->size() - 2);
+        FlakeRef ref(resolvedRef.baseRef());
-        const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + rev);
+        ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1);
         SourceInfo info(ref);
         info.storePath = result.path;
 
@@ -248,7 +247,9 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
     // This downloads the entire git history
     else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) {
         auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source");
-        const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false));
+        FlakeRef ref(resolvedRef.baseRef());
+        ref.ref = gitInfo.ref;
+        ref.rev = gitInfo.rev;
         SourceInfo info(ref);
         info.storePath = gitInfo.storePath;
         info.revCount = gitInfo.revCount;
@@ -259,7 +260,9 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
         if (!pathExists(refData->path + "/.git"))
             throw Error("flake '%s' does not reference a Git repository", refData->path);
         auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source");
-        const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false));
+        FlakeRef ref(resolvedRef.baseRef());
+        ref.ref = gitInfo.ref;
+        ref.rev = gitInfo.rev;
         SourceInfo info(ref);
         info.storePath = gitInfo.storePath;
         info.revCount = gitInfo.revCount;

src/libexpr/primops/flakeref.cc

@@ -147,31 +147,53 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative)
 std::string FlakeRef::to_string() const
 {
     std::string string;
+    bool first = true;
 
-    if (auto refData = std::get_if<FlakeRef::IsAlias>(&data))
+    auto addParam =
+        [&](const std::string & name, std::string value) {
+            string += first ? '?' : '&';
+            first = false;
+            string += name;
+            string += '=';
+            string += value; // FIXME: escaping
+        };
+
+    if (auto refData = std::get_if<FlakeRef::IsAlias>(&data)) {
         string = refData->alias;
+        if (ref) string += '/' + *ref;
+        if (rev) string += '/' + rev->to_string(Base16, false);
+    }
+
+    else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) {
+        assert(subdir == "");
+        assert(!rev);
+        assert(!ref);
+        return refData->path;
+    }
 
     else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) {
         assert(!(ref && rev));
         string = "github:" + refData->owner + "/" + refData->repo;
+        if (ref) { string += '/'; string += *ref; }
+        if (rev) { string += '/'; string += rev->to_string(Base16, false); }
+        if (subdir != "") addParam("dir", subdir);
     }
 
     else if (auto refData = std::get_if<FlakeRef::IsGit>(&data)) {
         assert(!rev || ref);
         string = refData->uri;
+
+        if (ref) {
+            addParam("ref", *ref);
+            if (rev)
+                addParam("rev", rev->to_string(Base16, false));
+        }
+
+        if (subdir != "") addParam("dir", subdir);
     }
 
-    else if (auto refData = std::get_if<FlakeRef::IsPath>(&data))
-        return refData->path;
-
     else abort();
 
-    // FIXME: need to use ?rev etc. for IsGit URIs.
-    string += (ref ? "/" + *ref : "") +
-              (rev ? "/" + rev->to_string(Base16, false) : "");
-
-    if (subdir != "") string += "?dir=" + subdir;
-
     return string;
 }
 

tests/build-dry.sh

@@ -8,13 +8,13 @@ clearStore
 clearCache
 
 # Ensure this builds successfully first
-nix build -f dependencies.nix
+nix build --no-link -f dependencies.nix
 
 clearStore
 clearCache
 
 # Try --dry-run using old command first
-nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
+nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built"
 # Now new command:
 nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 
@@ -27,7 +27,7 @@ clearCache
 # Try --dry-run using new command first
 nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 # Now old command:
-nix-build dependencies.nix --dry-run 2>&1 | grep "will be built"
+nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built"
 fi
 
 ###################################################

tests/config.nix.in

@@ -5,7 +5,7 @@ rec {
 
   path = coreutils;
 
-  system = builtins.currentSystem;
+  system = "@system@";
 
   shared = builtins.getEnv "_NIX_TEST_SHARED";
 

tests/flakes.sh

@@ -0,0 +1,117 @@
+source common.sh
+
+if [[ -z $(type -p git) ]]; then
+    echo "Git not installed; skipping flake tests"
+    exit 99
+fi
+
+clearStore
+
+registry=$TEST_ROOT/registry.json
+
+flake1=$TEST_ROOT/flake1
+flake2=$TEST_ROOT/flake2
+flake3=$TEST_ROOT/flake3
+
+for repo in $flake1 $flake2 $flake3; do
+    rm -rf $repo
+    mkdir $repo
+    git -C $repo init
+    git -C $repo config user.email "foobar@example.com"
+    git -C $repo config user.name "Foobar"
+done
+
+cat > $flake1/flake.nix <<EOF
+{
+  name = "flake1";
+
+  epoch = 2019;
+
+  description = "Bla bla";
+
+  provides = deps: rec {
+    packages.foo = import ./simple.nix;
+    defaultPackage = packages.foo;
+  };
+}
+EOF
+
+cp ./simple.nix ./simple.builder.sh ./config.nix $flake1/
+git -C $flake1 add flake.nix simple.nix simple.builder.sh config.nix
+git -C $flake1 commit -m 'Initial'
+
+cat > $flake2/flake.nix <<EOF
+{
+  name = "flake2";
+
+  epoch = 2019;
+
+  requires = [ "flake1" ];
+
+  description = "Fnord";
+
+  provides = deps: rec {
+    packages.bar = deps.flake1.provides.packages.foo;
+  };
+}
+EOF
+
+git -C $flake2 add flake.nix
+git -C $flake2 commit -m 'Initial'
+
+cat > $registry <<EOF
+{
+    "flakes": {
+        "flake1": {
+            "uri": "file://$flake1"
+        },
+        "flake2": {
+            "uri": "file://$flake2"
+        },
+        "flake3": {
+            "uri": "file://$flake3"
+        },
+        "nixpkgs": {
+            "uri": "flake1"
+        }
+    },
+    "version": 1
+}
+EOF
+
+# Test 'nix flake list'.
+(( $(nix flake list --flake-registry $registry | wc -l) == 4 ))
+
+# Test 'nix flake info'.
+nix flake info --flake-registry $registry flake1 | grep -q 'ID: *flake1'
+
+# Test 'nix flake info --json'.
+json=$(nix flake info --flake-registry $registry flake1 --json | jq .)
+[[ $(echo "$json" | jq -r .description) = 'Bla bla' ]]
+[[ -d $(echo "$json" | jq -r .path) ]]
+
+# Test 'nix build' on a flake.
+nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo
+[[ -e $TEST_ROOT/result/hello ]]
+
+# Test defaultPackage.
+nix build -o $TEST_ROOT/result --flake-registry $registry flake1:
+[[ -e $TEST_ROOT/result/hello ]]
+
+# Building a flake with an unlocked dependency should fail in pure mode.
+(! nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar)
+
+# But should succeed in impure mode.
+# FIXME: this currently fails.
+#nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar --impure
+
+# Test automatic lock file generation.
+nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar
+[[ -e $flake2/flake.lock ]]
+git -C $flake2 commit flake.lock -m 'Add flake.lock'
+
+# Now we should be able to build the flake in pure mode.
+nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar
+
+# Or without a registry.
+nix build -o $TEST_ROOT/result file://$flake2:bar

tests/local.mk

@@ -26,13 +26,15 @@ nix_tests = \
   check.sh \
   plugins.sh \
   search.sh \
-  nix-copy-ssh.sh
+  nix-copy-ssh.sh \
+  flakes.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
 
 tests-environment = NIX_REMOTE= $(bash) -e
 
-clean-files += $(d)/common.sh
+clean-files += $(d)/common.sh $(d)/config.nix
+
+installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT) $(d)/config.nix
 
-installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT)

tests/nix-copy-ssh.sh

@@ -7,7 +7,7 @@ remoteRoot=$TEST_ROOT/store2
 chmod -R u+w "$remoteRoot" || true
 rm -rf "$remoteRoot"
 
-outPath=$(nix-build dependencies.nix)
+outPath=$(nix-build --no-out-link dependencies.nix)
 
 nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath
 

tests/nix-shell.sh

@@ -27,13 +27,13 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run
 # Test nix-shell on a .drv symlink
 
 # Legacy: absolute path and .drv extension required
-nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv
+nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv
-[[ $(nix-shell --pure $PWD/shell.drv --run \
+[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
 
 # New behaviour: just needs to resolve to a derivation in the store
-nix-instantiate shell.nix -A shellDrv --indirect --add-root shell
+nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell
-[[ $(nix-shell --pure shell --run \
+[[ $(nix-shell --pure $TEST_ROOT/shell --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
 
 # Test nix-shell -p

tests/placeholders.sh

@@ -18,5 +18,3 @@ nix-build --no-out-link -E '
     ";
   }
 '
-
-echo XYZZY