From 6e984431dd27326681a7cbb56404665c353c834a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 May 2019 23:38:40 +0200 Subject: [PATCH 1/3] fetchGit: Don't barf if we can't update our Git clone Instead print a warning that we're continuing with the most recently fetched version. --- src/libexpr/primops/fetchGit.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 00bbeb6d8..f6b096c4a 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -139,7 +139,12 @@ GitInfo exportGit(ref store, std::string uri, // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + try { + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", uri); + } struct timeval times[2]; times[0].tv_sec = now; From 6b77bfc28d36d5b8111e23e15fbe4513a0797f47 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 May 2019 23:39:58 +0200 Subject: [PATCH 2/3] FlakeRef::to_string(): Check round trip --- src/libexpr/primops/flakeref.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 3842b3f1a..c42616374 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -194,6 +194,8 @@ std::string FlakeRef::to_string() const else abort(); + assert(FlakeRef(string) == *this); + return string; } From 90fe1dfd2fd85f17609166c92a2163dfaa09ea99 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 May 2019 23:42:13 +0200 Subject: [PATCH 3/3] Register flake source trees as GC roots This ensures that flakes don't get garbage-collected, which is important to get nix-channel-like behaviour. For example, running $ nix build hydra: will create a GC root ~/.cache/nix/flake-closures/hydra -> /nix/store/xarfiqcwa4w8r4qpz1a769xxs8c3phgn-flake-closure where the contents/references of the linked file in the store are the flake source trees used by the 'hydra' flake: /nix/store/n6d5f5lkpfjbmkyby0nlg8y1wbkmbc7i-source /nix/store/vbkg4zy1qd29fnhflsv9k2j9jnbqd5m2-source /nix/store/z46xni7d47s5wk694359mq9ay353ar94-source Note that this in itself is not enough to allow offline use; the fetcher for the flakeref (e.g. fetchGit or downloadCached) must not fail if it cannot fetch the latest version of the file, so long as it knows a cached version. Issue #2868. --- src/libexpr/primops/flake.hh | 2 ++ src/nix/installables.cc | 45 +++++++++++++++++++++++++++++++++++- tests/flakes.sh | 19 +++++++++++++-- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index b965aa9e7..f85e62e7f 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -132,6 +132,8 @@ struct ResolvedFlake ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); +void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v); + void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); void gitCloneFlake(FlakeRef flakeRef, EvalState &, Registries, const Path & destDir); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 1a79f49fb..6cab06240 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -10,6 +10,7 @@ #include "primops/flake.hh" #include +#include namespace nix { @@ -162,6 +163,44 @@ struct InstallableAttrPath : InstallableValue } }; +void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const ResolvedFlake & resFlake) +{ + if (std::get_if(&origFlakeRef.data)) return; + + /* Get the store paths of all non-local flakes. */ + PathSet closure; + + std::queue> queue; + queue.push(resFlake); + + while (!queue.empty()) { + const ResolvedFlake & flake = queue.front(); + queue.pop(); + if (!std::get_if(&flake.flake.resolvedRef.data)) + closure.insert(flake.flake.storePath); + for (const auto & dep : flake.flakeDeps) + queue.push(dep.second); + } + + if (closure.empty()) return; + + /* Write the closure to a file in the store. */ + auto closurePath = store.addTextToStore("flake-closure", concatStringsSep(" ", closure), closure); + + Path cacheDir = getCacheDir() + "/nix/flake-closures"; + createDirs(cacheDir); + + auto s = origFlakeRef.to_string(); + assert(s[0] != '.'); + s = replaceStrings(s, "%", "%25"); + s = replaceStrings(s, "/", "%2f"); + s = replaceStrings(s, ":", "%3a"); + Path symlink = cacheDir + "/" + s; + debug("writing GC root '%s' for flake closure of '%s'", symlink, origFlakeRef); + replaceSymlink(closurePath, symlink); + store.addIndirectRoot(symlink); +} + struct InstallableFlake : InstallableValue { FlakeRef flakeRef; @@ -182,7 +221,11 @@ struct InstallableFlake : InstallableValue { auto vFlake = state.allocValue(); - makeFlakeValue(state, flakeRef, cmd.getLockFileMode(), *vFlake); + auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); + + callFlake(state, resFlake, *vFlake); + + makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; diff --git a/tests/flakes.sh b/tests/flakes.sh index 822c94e42..179fef320 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -5,7 +5,10 @@ if [[ -z $(type -p git) ]]; then exit 99 fi +export _NIX_FORCE_HTTP=1 + clearStore +rm -rf $TEST_HOME/.cache registry=$TEST_ROOT/registry.json @@ -14,7 +17,7 @@ flake2Dir=$TEST_ROOT/flake2 flake3Dir=$TEST_ROOT/flake3 for repo in $flake1Dir $flake2Dir $flake3Dir; do - rm -rf $repo + rm -rf $repo $repo.tmp mkdir $repo git -C $repo init git -C $repo config user.email "foobar@example.com" @@ -142,7 +145,7 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar # Or without a registry. # FIXME: shouldn't need '--flake-registry /no-registry'? -nix build -o $TEST_ROOT/result --flake-registry /no-registry file://$flake2Dir:bar +nix build -o $TEST_ROOT/result --flake-registry /no-registry file://$flake2Dir:bar --tarball-ttl 0 # Test whether indirect dependencies work. nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:xyzzy @@ -184,3 +187,15 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | nix flake list --flake-registry file://$registry | grep -q flake3 mv $registry $registry.tmp nix flake list --flake-registry file://$registry --tarball-ttl 0 | grep -q flake3 +mv $registry.tmp $registry + +# Test whether flakes are registered as GC roots for offline use. +rm -rf $TEST_HOME/.cache +nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar +mv $flake1Dir $flake1Dir.tmp +mv $flake2Dir $flake2Dir.tmp +nix-store --gc +nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar +nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar --tarball-ttl 0 +mv $flake1Dir.tmp $flake1Dir +mv $flake2Dir.tmp $flake2Dir