From 6dbd5c26e6c853f302cd9d3ed171d134ff24ffe1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 19:10:35 +0200 Subject: [PATCH 01/13] Make flake input fetching lazy As long as the flake input is locked, it is now only fetched when it is evaluated (e.g. "nixpkgs" is fetched when "inputs.nixpkgs." is evaluated). This required adding an "id" attribute to the members of "inputs" in lockfiles, e.g. "inputs": { "nixpkgs/release-19.03": { "id": "nixpkgs", "inputs": {}, "narHash": "sha256-eYtxncIMFVmOHaHBtTdPGcs/AnJqKqA6tHCm0UmPYQU=", "nonFlakeInputs": {}, "uri": "github:edolstra/nixpkgs/e9d5882bb861dc48f8d46960e7c820efdbe8f9c1" } } because the flake ID needs to be known beforehand to construct the "inputs" attrset. Fixes #2913. --- src/libexpr/primops/flake.cc | 292 +++++++++++++++++++---------------- src/libexpr/primops/flake.hh | 138 +++++++++++------ src/nix/flake.cc | 6 +- src/nix/installables.cc | 2 + src/nlohmann/json_fwd.hpp | 10 ++ tests/flakes.sh | 11 ++ 6 files changed, 272 insertions(+), 187 deletions(-) create mode 100644 src/nlohmann/json_fwd.hpp diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index e5035c53a..189663e51 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -43,97 +43,101 @@ std::shared_ptr readRegistry(const Path & path) void writeRegistry(const FlakeRegistry & registry, const Path & path) { nlohmann::json json; - json["version"] = 1; + json["version"] = 2; for (auto elem : registry.entries) json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; createDirs(dirOf(path)); writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } -LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) +NonFlakeDep::NonFlakeDep(const nlohmann::json & json) + : ref(json["uri"]) + , narHash(Hash((std::string) json["narHash"])) { - FlakeRef flakeRef(json["uri"]); - if (!flakeRef.isImmutable()) - throw Error("cannot use mutable flake '%s' in pure mode", flakeRef); + if (!ref.isImmutable()) + throw Error("lockfile contains mutable flakeref '%s'", ref); +} - LockFile::FlakeEntry entry(flakeRef, Hash((std::string) json["narHash"])); +nlohmann::json NonFlakeDep::toJson() const +{ + nlohmann::json json; + json["uri"] = ref.to_string(); + json["narHash"] = narHash.to_string(SRI); + return json; +} +FlakeDep::FlakeDep(const nlohmann::json & json) + : FlakeInputs(json) + , id(json["id"]) + , ref(json["uri"]) + , narHash(Hash((std::string) json["narHash"])) +{ + if (!ref.isImmutable()) + throw Error("lockfile contains mutable flakeref '%s'", ref); +} + +nlohmann::json FlakeDep::toJson() const +{ + auto json = FlakeInputs::toJson(); + json["id"] = id; + json["uri"] = ref.to_string(); + json["narHash"] = narHash.to_string(SRI); + return json; +} + +FlakeInputs::FlakeInputs(const nlohmann::json & json) +{ auto nonFlakeInputs = json["nonFlakeInputs"]; - - for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) { - FlakeRef flakeRef(i->value("uri", "")); - if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); - LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", ""))); - entry.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); - } + for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) + nonFlakeDeps.insert_or_assign(i.key(), NonFlakeDep(*i)); auto inputs = json["inputs"]; - for (auto i = inputs.begin(); i != inputs.end(); ++i) - entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + flakeDeps.insert_or_assign(i.key(), FlakeDep(*i)); +} - return entry; +nlohmann::json FlakeInputs::toJson() const +{ + nlohmann::json json; + { + auto j = nlohmann::json::object(); + for (auto & i : nonFlakeDeps) + j[i.first] = i.second.toJson(); + json["nonFlakeInputs"] = std::move(j); + } + { + auto j = nlohmann::json::object(); + for (auto & i : flakeDeps) + j[i.first.to_string()] = i.second.toJson(); + json["inputs"] = std::move(j); + } + return json; +} + +nlohmann::json LockFile::toJson() const +{ + auto json = FlakeInputs::toJson(); + json["version"] = 2; + return json; } LockFile readLockFile(const Path & path) { - LockFile lockFile; + if (pathExists(path)) { + auto json = nlohmann::json::parse(readFile(path)); - if (!pathExists(path)) - return lockFile; + auto version = json.value("version", 0); + if (version != 2) + throw Error("lock file '%s' has unsupported version %d", path, version); - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 1) - throw Error("lock file '%s' has unsupported version %d", path, version); - - auto nonFlakeInputs = json["nonFlakeInputs"]; - - for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) { - FlakeRef flakeRef(i->value("uri", "")); - LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", ""))); - if (!flakeRef.isImmutable()) - throw Error("found mutable FlakeRef '%s' in lockfile at path %s", flakeRef, path); - lockFile.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); - } - - auto inputs = json["inputs"]; - - for (auto i = inputs.begin(); i != inputs.end(); ++i) - lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); - - return lockFile; -} - -nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) -{ - nlohmann::json json; - json["uri"] = entry.ref.to_string(); - json["narHash"] = entry.narHash.to_string(SRI); - for (auto & x : entry.nonFlakeEntries) { - json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string(); - json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI); - } - for (auto & x : entry.flakeEntries) - json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); - return json; + return LockFile(json); + } else + return LockFile(); } std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) { - nlohmann::json json; - json["version"] = 1; - json["nonFlakeInputs"] = nlohmann::json::object(); - for (auto & x : lockFile.nonFlakeEntries) { - json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string(); - json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI); - } - json["inputs"] = nlohmann::json::object(); - for (auto & x : lockFile.flakeEntries) - json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); - stream << json.dump(4); // '4' = indentation in json file + stream << lockFile.toJson().dump(4); // '4' = indentation in json file return stream; } @@ -387,33 +391,6 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al return nonFlake; } -LockFile entryToLockFile(const LockFile::FlakeEntry & entry) -{ - LockFile lockFile; - lockFile.flakeEntries = entry.flakeEntries; - lockFile.nonFlakeEntries = entry.nonFlakeEntries; - return lockFile; -} - -LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) -{ - LockFile::FlakeEntry entry( - resolvedFlake.flake.sourceInfo.resolvedRef, - resolvedFlake.flake.sourceInfo.narHash); - - for (auto & info : resolvedFlake.flakeDeps) - entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second)); - - for (auto & nonFlake : resolvedFlake.nonFlakeDeps) { - LockFile::NonFlakeEntry nonEntry( - nonFlake.sourceInfo.resolvedRef, - nonFlake.sourceInfo.narHash); - entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry); - } - - return entry; -} - bool allowedToWrite(HandleLockFile handle) { return handle == UpdateLockFile || handle == RecreateLockFile; @@ -435,43 +412,50 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) else assert(false); } -ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef, - HandleLockFile handleLockFile, LockFile lockFile = {}, bool topRef = false) +static std::pair updateLocks( + EvalState & state, + const FlakeRef & flakeRef, + HandleLockFile handleLockFile, + const FlakeInputs & oldEntry, + bool topRef) { - Flake flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); + auto flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); - ResolvedFlake deps(flake); + FlakeDep newEntry( + flake.id, + flake.sourceInfo.resolvedRef, + flake.sourceInfo.narHash); - for (auto & nonFlakeInfo : flake.nonFlakeInputs) { - FlakeRef ref = nonFlakeInfo.second; - auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first); - if (i != lockFile.nonFlakeEntries.end()) { - NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first); - if (nonFlake.sourceInfo.narHash != i->second.narHash) - throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); - deps.nonFlakeDeps.push_back(nonFlake); + for (auto & input : flake.nonFlakeInputs) { + auto & id = input.first; + auto & ref = input.second; + auto i = oldEntry.nonFlakeDeps.find(id); + if (i != oldEntry.nonFlakeDeps.end()) { + newEntry.nonFlakeDeps.insert_or_assign(i->first, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update non-flake dependency '%s' in pure mode", nonFlakeInfo.first); - deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first, allowedToUseRegistries(handleLockFile, false))); + throw Error("cannot update non-flake dependency '%s' in pure mode", id); + auto nonFlake = getNonFlake(state, ref, id, allowedToUseRegistries(handleLockFile, false)); + newEntry.nonFlakeDeps.insert_or_assign(id, + NonFlakeDep( + nonFlake.sourceInfo.resolvedRef, + nonFlake.sourceInfo.narHash)); } } - for (auto newFlakeRef : flake.inputs) { - auto i = lockFile.flakeEntries.find(newFlakeRef); - if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible - ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second)); - if (newResFlake.flake.sourceInfo.narHash != i->second.narHash) - throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); - deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake); + for (auto & inputRef : flake.inputs) { + auto i = oldEntry.flakeDeps.find(inputRef); + if (i != oldEntry.flakeDeps.end()) { + newEntry.flakeDeps.insert_or_assign(inputRef, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update flake dependency '%s' in pure mode", newFlakeRef.to_string()); - deps.flakeDeps.insert_or_assign(newFlakeRef, resolveFlakeFromLockFile(state, newFlakeRef, handleLockFile)); + throw Error("cannot update flake dependency '%s' in pure mode", inputRef); + newEntry.flakeDeps.insert_or_assign(inputRef, + updateLocks(state, inputRef, handleLockFile, {}, false).second); } } - return deps; + return {flake, newEntry}; } /* Given a flake reference, recursively fetch it and its dependencies. @@ -479,7 +463,8 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake */ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { - Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); + auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); + LockFile oldLockFile; if (!recreateLockFile(handleLockFile)) { @@ -490,10 +475,9 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); } - LockFile lockFile(oldLockFile); - - ResolvedFlake resFlake = resolveFlakeFromLockFile(state, topRef, handleLockFile, lockFile, true); - lockFile = entryToLockFile(dependenciesToFlakeEntry(resFlake)); + // FIXME: get rid of duplicate getFlake call + LockFile lockFile(updateLocks( + state, topRef, handleLockFile, oldLockFile, true).second); if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { @@ -509,7 +493,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc warn("using updated lockfile without writing it to file"); } - return resFlake; + return ResolvedFlake(std::move(flake), std::move(lockFile)); } void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) @@ -520,7 +504,9 @@ void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateL static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) { auto & path = sourceInfo.storePath; - state.store->isValidPath(path); + assert(state.store->isValidPath(path)); + // FIXME: turn into fetchGit etc. + // FIXME: check narHash. mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); if (sourceInfo.resolvedRef.rev) { @@ -539,42 +525,74 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); } -void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) +/* Helper primop to make callFlake (below) fetch/call its inputs + lazily. Note that this primop cannot be called by user code since + it doesn't appear in 'builtins'. */ +static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyFlake = (FlakeDep *) args[0]->attrs; + auto flake = getFlake(state, lazyFlake->ref, false); + callFlake(state, flake, *lazyFlake, v); +} + +void callFlake(EvalState & state, + const Flake & flake, + const FlakeInputs & inputs, + Value & v) { // Construct the resulting attrset '{description, outputs, // ...}'. This attrset is passed lazily as an argument to 'outputs'. - state.mkAttrs(v, resFlake.flakeDeps.size() + resFlake.nonFlakeDeps.size() + 8); + state.mkAttrs(v, + inputs.flakeDeps.size() + + inputs.nonFlakeDeps.size() + 8); - for (auto info : resFlake.flakeDeps) { - const ResolvedFlake newResFlake = info.second; - auto vFlake = state.allocAttr(v, newResFlake.flake.id); - callFlake(state, newResFlake, *vFlake); + for (auto & dep : inputs.flakeDeps) { + auto vFlake = state.allocAttr(v, dep.second.id); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new FlakeDep(dep.second); // evil! also inefficient + mkApp(*vFlake, *vPrimOp, *vArg); } - for (const NonFlake nonFlake : resFlake.nonFlakeDeps) { - auto vNonFlake = state.allocAttr(v, nonFlake.alias); + for (auto & dep : inputs.nonFlakeDeps) { + auto vNonFlake = state.allocAttr(v, dep.first); state.mkAttrs(*vNonFlake, 8); - state.store->isValidPath(nonFlake.sourceInfo.storePath); + auto nonFlake = getNonFlake(state, dep.second.ref, dep.first); + + assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); + mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); emitSourceInfoAttrs(state, nonFlake.sourceInfo, *vNonFlake); } - mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); + mkString(*state.allocAttr(v, state.sDescription), flake.description); - emitSourceInfoAttrs(state, resFlake.flake.sourceInfo, v); + emitSourceInfoAttrs(state, flake.sourceInfo, v); auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); - mkApp(*vOutputs, *resFlake.flake.vOutputs, v); + mkApp(*vOutputs, *flake.vOutputs, v); v.attrs->push_back(Attr(state.symbols.create("self"), &v)); v.attrs->sort(); } +void callFlake(EvalState & state, + const ResolvedFlake & resFlake, + Value & v) +{ + callFlake(state, resFlake.flake, resFlake.lockFile, v); +} + // This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 82b0973f6..692fa744d 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -2,6 +2,7 @@ #include "flakeref.hh" #include +#include namespace nix { @@ -19,51 +20,12 @@ struct FlakeRegistry std::map entries; }; -struct LockFile -{ - struct NonFlakeEntry - { - FlakeRef ref; - Hash narHash; - NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {}; - - bool operator ==(const NonFlakeEntry & other) const - { - return ref == other.ref && narHash == other.narHash; - } - }; - - struct FlakeEntry - { - FlakeRef ref; - Hash narHash; - std::map flakeEntries; - std::map nonFlakeEntries; - FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {}; - - bool operator ==(const FlakeEntry & other) const - { - return - ref == other.ref - && narHash == other.narHash - && flakeEntries == other.flakeEntries - && nonFlakeEntries == other.nonFlakeEntries; - } - }; - - std::map flakeEntries; - std::map nonFlakeEntries; - - bool operator ==(const LockFile & other) const - { - return - flakeEntries == other.flakeEntries - && nonFlakeEntries == other.nonFlakeEntries; - } -}; - typedef std::vector> Registries; +std::shared_ptr readRegistry(const Path &); + +void writeRegistry(const FlakeRegistry &, const Path &); + Path getUserRegistryPath(); enum HandleLockFile : unsigned int @@ -75,9 +37,80 @@ enum HandleLockFile : unsigned int , UseNewLockFile // `RecreateLockFile` without writing to file }; -std::shared_ptr readRegistry(const Path &); +struct NonFlakeDep +{ + FlakeRef ref; + Hash narHash; -void writeRegistry(const FlakeRegistry &, const Path &); + NonFlakeDep(const FlakeRef & flakeRef, const Hash & narHash) + : ref(flakeRef), narHash(narHash) {}; + + NonFlakeDep(const nlohmann::json & json); + + bool operator ==(const NonFlakeDep & other) const + { + return ref == other.ref && narHash == other.narHash; + } + + nlohmann::json toJson() const; +}; + +struct FlakeDep; + +struct FlakeInputs +{ + std::map flakeDeps; + std::map nonFlakeDeps; + + FlakeInputs() {}; + FlakeInputs(const nlohmann::json & json); + + nlohmann::json toJson() const; +}; + +struct FlakeDep : FlakeInputs +{ + FlakeId id; + FlakeRef ref; + Hash narHash; + + FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) + : id(id), ref(flakeRef), narHash(narHash) {}; + + FlakeDep(const nlohmann::json & json); + + bool operator ==(const FlakeDep & other) const + { + return + id == other.id + && ref == other.ref + && narHash == other.narHash + && flakeDeps == other.flakeDeps + && nonFlakeDeps == other.nonFlakeDeps; + } + + nlohmann::json toJson() const; +}; + +struct LockFile : FlakeInputs +{ + bool operator ==(const LockFile & other) const + { + return + flakeDeps == other.flakeDeps + && nonFlakeDeps == other.nonFlakeDeps; + } + + LockFile() {} + LockFile(const nlohmann::json & json) : FlakeInputs(json) {} + LockFile(FlakeDep && dep) + { + flakeDeps = std::move(dep.flakeDeps); + nonFlakeDeps = std::move(dep.nonFlakeDeps); + } + + nlohmann::json toJson() const; +}; struct SourceInfo { @@ -129,14 +162,21 @@ Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); struct ResolvedFlake { Flake flake; - std::map flakeDeps; // The key in this map, is the originalRef as written in flake.nix - std::vector nonFlakeDeps; - ResolvedFlake(const Flake & flake) : flake(flake) {} + LockFile lockFile; + ResolvedFlake(Flake && flake, LockFile && lockFile) + : flake(flake), lockFile(lockFile) {} }; ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); -void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v); +void callFlake(EvalState & state, + const Flake & flake, + const FlakeInputs & inputs, + Value & v); + +void callFlake(EvalState & state, + const ResolvedFlake & resFlake, + Value & v); void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8d6716391..d229c7512 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -136,6 +136,7 @@ static nlohmann::json nonFlakeToJson(const NonFlake & nonFlake) return j; } +#if 0 // FIXME: merge info CmdFlakeInfo? struct CmdFlakeDeps : FlakeCommand { @@ -173,6 +174,7 @@ struct CmdFlakeDeps : FlakeCommand } } }; +#endif struct CmdFlakeUpdate : FlakeCommand { @@ -232,6 +234,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON if (json) { auto json = flakeToJson(flake); +#if 0 auto state = getEvalState(); auto vFlake = state->allocValue(); @@ -254,6 +257,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON }); json["outputs"] = std::move(outputs); +#endif std::cout << json.dump() << std::endl; } else @@ -518,7 +522,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command , make_ref() , make_ref() , make_ref() - , make_ref() + //, make_ref() , make_ref() , make_ref() , make_ref() diff --git a/src/nix/installables.cc b/src/nix/installables.cc index b6f05b314..86b4a9b93 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -190,6 +190,7 @@ void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const flake::ResolvedFlake & resFlake) { +#if 0 if (std::get_if(&origFlakeRef.data)) return; /* Get the store paths of all non-local flakes. */ @@ -224,6 +225,7 @@ void makeFlakeClosureGCRoot(Store & store, debug("writing GC root '%s' for flake closure of '%s'", symlink, origFlakeRef); replaceSymlink(closurePath, symlink); store.addIndirectRoot(symlink); +#endif } struct InstallableFlake : InstallableValue diff --git a/src/nlohmann/json_fwd.hpp b/src/nlohmann/json_fwd.hpp new file mode 100644 index 000000000..ae6e4c64f --- /dev/null +++ b/src/nlohmann/json_fwd.hpp @@ -0,0 +1,10 @@ +#pragma once + +namespace nlohmann { + +struct json : basic_json<> +{ + using basic_json<>::basic_json; +}; + +} diff --git a/tests/flakes.sh b/tests/flakes.sh index 998abfd09..29845e5ed 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -191,6 +191,8 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth # Check whether it saved the lockfile [[ ! (-z $(git -C $flake3Dir diff master)) ]] +git -C $flake3Dir commit -m 'Add lockfile' + # Unsupported epochs should be an error. sed -i $flake3Dir/flake.nix -e s/201906/201909/ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | grep 'unsupported epoch' @@ -241,3 +243,12 @@ git -C $flake3Dir commit -m 'Add nonFlakeInputs' # Check whether `nix build` works with a lockfile which is missing a nonFlakeInputs nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth + +# Check whether flake input fetching is lazy: flake3:sth does not +# depend on flake2, so this shouldn't fail. +rm -rf $TEST_HOME/.cache +clearStore +mv $flake2Dir $flake2Dir.tmp +nix build -o $TEST_ROOT/result --flake-registry $registry flake3:sth +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy) +mv $flake2Dir.tmp $flake2Dir From 278114d559109199ff8e6f23b6700ab7909f5320 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 19:45:16 +0200 Subject: [PATCH 02/13] Fix GC closure generation --- src/libexpr/primops/flake.cc | 17 +++++++++-------- src/libexpr/primops/flake.hh | 26 +++++++++++++++++--------- src/nix/installables.cc | 21 +++++++++++++-------- tests/flakes.sh | 1 + 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 189663e51..d0405a377 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -50,7 +50,7 @@ void writeRegistry(const FlakeRegistry & registry, const Path & path) writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } -NonFlakeDep::NonFlakeDep(const nlohmann::json & json) +AbstractDep::AbstractDep(const nlohmann::json & json) : ref(json["uri"]) , narHash(Hash((std::string) json["narHash"])) { @@ -58,7 +58,7 @@ NonFlakeDep::NonFlakeDep(const nlohmann::json & json) throw Error("lockfile contains mutable flakeref '%s'", ref); } -nlohmann::json NonFlakeDep::toJson() const +nlohmann::json AbstractDep::toJson() const { nlohmann::json json; json["uri"] = ref.to_string(); @@ -66,22 +66,23 @@ nlohmann::json NonFlakeDep::toJson() const return json; } +Path AbstractDep::computeStorePath(Store & store) const +{ + return store.makeFixedOutputPath(true, narHash, "source"); +} + FlakeDep::FlakeDep(const nlohmann::json & json) : FlakeInputs(json) + , AbstractDep(json) , id(json["id"]) - , ref(json["uri"]) - , narHash(Hash((std::string) json["narHash"])) { - if (!ref.isImmutable()) - throw Error("lockfile contains mutable flakeref '%s'", ref); } nlohmann::json FlakeDep::toJson() const { auto json = FlakeInputs::toJson(); + json.update(AbstractDep::toJson()); json["id"] = id; - json["uri"] = ref.to_string(); - json["narHash"] = narHash.to_string(SRI); return json; } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 692fa744d..933bc2593 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -1,3 +1,5 @@ +#pragma once + #include "types.hh" #include "flakeref.hh" @@ -8,6 +10,7 @@ namespace nix { struct Value; class EvalState; +class Store; namespace flake { @@ -37,22 +40,29 @@ enum HandleLockFile : unsigned int , UseNewLockFile // `RecreateLockFile` without writing to file }; -struct NonFlakeDep +struct AbstractDep { FlakeRef ref; Hash narHash; - NonFlakeDep(const FlakeRef & flakeRef, const Hash & narHash) + AbstractDep(const FlakeRef & flakeRef, const Hash & narHash) : ref(flakeRef), narHash(narHash) {}; - NonFlakeDep(const nlohmann::json & json); + AbstractDep(const nlohmann::json & json); + + nlohmann::json toJson() const; + + Path computeStorePath(Store & store) const; +}; + +struct NonFlakeDep : AbstractDep +{ + using AbstractDep::AbstractDep; bool operator ==(const NonFlakeDep & other) const { return ref == other.ref && narHash == other.narHash; } - - nlohmann::json toJson() const; }; struct FlakeDep; @@ -68,14 +78,12 @@ struct FlakeInputs nlohmann::json toJson() const; }; -struct FlakeDep : FlakeInputs +struct FlakeDep : FlakeInputs, AbstractDep { FlakeId id; - FlakeRef ref; - Hash narHash; FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) - : id(id), ref(flakeRef), narHash(narHash) {}; + : AbstractDep(flakeRef, narHash), id(id) {}; FlakeDep(const nlohmann::json & json); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 86b4a9b93..c44a37f1e 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -190,22 +190,28 @@ void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const flake::ResolvedFlake & resFlake) { -#if 0 if (std::get_if(&origFlakeRef.data)) return; /* Get the store paths of all non-local flakes. */ PathSet closure; - std::queue> queue; - queue.push(resFlake); + assert(store.isValidPath(resFlake.flake.sourceInfo.storePath)); + closure.insert(resFlake.flake.sourceInfo.storePath); + + std::queue> queue; + queue.push(resFlake.lockFile); while (!queue.empty()) { - const flake::ResolvedFlake & flake = queue.front(); + const flake::FlakeInputs & flake = queue.front(); queue.pop(); - if (!std::get_if(&flake.flake.sourceInfo.resolvedRef.data)) - closure.insert(flake.flake.sourceInfo.storePath); - for (const auto & dep : flake.flakeDeps) + /* Note: due to lazy fetching, these paths might not exist + yet. */ + for (auto & dep : flake.flakeDeps) { + closure.insert(dep.second.computeStorePath(store)); queue.push(dep.second); + } + for (auto & dep : flake.nonFlakeDeps) + closure.insert(dep.second.computeStorePath(store)); } if (closure.empty()) return; @@ -225,7 +231,6 @@ void makeFlakeClosureGCRoot(Store & store, debug("writing GC root '%s' for flake closure of '%s'", symlink, origFlakeRef); replaceSymlink(closurePath, symlink); store.addIndirectRoot(symlink); -#endif } struct InstallableFlake : InstallableValue diff --git a/tests/flakes.sh b/tests/flakes.sh index 29845e5ed..f44b9509f 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -204,6 +204,7 @@ nix flake list --flake-registry file://$registry --tarball-ttl 0 | grep -q flake mv $registry.tmp $registry # Test whether flakes are registered as GC roots for offline use. +# FIXME: use tarballs rather than git. rm -rf $TEST_HOME/.cache nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar mv $flake1Dir $flake1Dir.tmp From 9e99b5205c4035753106448241ae44e7447f019c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:01:21 +0200 Subject: [PATCH 03/13] Move LockFile and related types to a separate file --- src/libexpr/primops/flake.cc | 103 +----------------------------- src/libexpr/primops/flake.hh | 85 +----------------------- src/libexpr/primops/lockfile.cc | 104 ++++++++++++++++++++++++++++++ src/libexpr/primops/lockfile.hh | 110 ++++++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+), 184 deletions(-) create mode 100644 src/libexpr/primops/lockfile.cc create mode 100644 src/libexpr/primops/lockfile.hh diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index d0405a377..b38971c36 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -1,4 +1,5 @@ #include "flake.hh" +#include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" #include "fetchGit.hh" @@ -50,104 +51,6 @@ void writeRegistry(const FlakeRegistry & registry, const Path & path) writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } -AbstractDep::AbstractDep(const nlohmann::json & json) - : ref(json["uri"]) - , narHash(Hash((std::string) json["narHash"])) -{ - if (!ref.isImmutable()) - throw Error("lockfile contains mutable flakeref '%s'", ref); -} - -nlohmann::json AbstractDep::toJson() const -{ - nlohmann::json json; - json["uri"] = ref.to_string(); - json["narHash"] = narHash.to_string(SRI); - return json; -} - -Path AbstractDep::computeStorePath(Store & store) const -{ - return store.makeFixedOutputPath(true, narHash, "source"); -} - -FlakeDep::FlakeDep(const nlohmann::json & json) - : FlakeInputs(json) - , AbstractDep(json) - , id(json["id"]) -{ -} - -nlohmann::json FlakeDep::toJson() const -{ - auto json = FlakeInputs::toJson(); - json.update(AbstractDep::toJson()); - json["id"] = id; - return json; -} - -FlakeInputs::FlakeInputs(const nlohmann::json & json) -{ - auto nonFlakeInputs = json["nonFlakeInputs"]; - for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) - nonFlakeDeps.insert_or_assign(i.key(), NonFlakeDep(*i)); - - auto inputs = json["inputs"]; - for (auto i = inputs.begin(); i != inputs.end(); ++i) - flakeDeps.insert_or_assign(i.key(), FlakeDep(*i)); -} - -nlohmann::json FlakeInputs::toJson() const -{ - nlohmann::json json; - { - auto j = nlohmann::json::object(); - for (auto & i : nonFlakeDeps) - j[i.first] = i.second.toJson(); - json["nonFlakeInputs"] = std::move(j); - } - { - auto j = nlohmann::json::object(); - for (auto & i : flakeDeps) - j[i.first.to_string()] = i.second.toJson(); - json["inputs"] = std::move(j); - } - return json; -} - -nlohmann::json LockFile::toJson() const -{ - auto json = FlakeInputs::toJson(); - json["version"] = 2; - return json; -} - -LockFile readLockFile(const Path & path) -{ - if (pathExists(path)) { - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 2) - throw Error("lock file '%s' has unsupported version %d", path, version); - - return LockFile(json); - } else - return LockFile(); -} - -std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) -{ - stream << lockFile.toJson().dump(4); // '4' = indentation in json file - return stream; -} - -void writeLockFile(const LockFile & lockFile, const Path & path) -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", lockFile)); -} - Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; @@ -471,7 +374,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc if (!recreateLockFile(handleLockFile)) { // If recreateLockFile, start with an empty lockfile // FIXME: symlink attack - oldLockFile = readLockFile( + oldLockFile = LockFile::read( state.store->toRealPath(flake.sourceInfo.storePath) + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); } @@ -483,7 +386,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { if (auto refData = std::get_if(&topRef.data)) { - writeLockFile(lockFile, refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); + lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. runProgram("git", true, { "-C", refData->path, "add", diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 933bc2593..bbf35da02 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -2,15 +2,12 @@ #include "types.hh" #include "flakeref.hh" - -#include -#include +#include "lockfile.hh" namespace nix { struct Value; class EvalState; -class Store; namespace flake { @@ -40,86 +37,6 @@ enum HandleLockFile : unsigned int , UseNewLockFile // `RecreateLockFile` without writing to file }; -struct AbstractDep -{ - FlakeRef ref; - Hash narHash; - - AbstractDep(const FlakeRef & flakeRef, const Hash & narHash) - : ref(flakeRef), narHash(narHash) {}; - - AbstractDep(const nlohmann::json & json); - - nlohmann::json toJson() const; - - Path computeStorePath(Store & store) const; -}; - -struct NonFlakeDep : AbstractDep -{ - using AbstractDep::AbstractDep; - - bool operator ==(const NonFlakeDep & other) const - { - return ref == other.ref && narHash == other.narHash; - } -}; - -struct FlakeDep; - -struct FlakeInputs -{ - std::map flakeDeps; - std::map nonFlakeDeps; - - FlakeInputs() {}; - FlakeInputs(const nlohmann::json & json); - - nlohmann::json toJson() const; -}; - -struct FlakeDep : FlakeInputs, AbstractDep -{ - FlakeId id; - - FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) - : AbstractDep(flakeRef, narHash), id(id) {}; - - FlakeDep(const nlohmann::json & json); - - bool operator ==(const FlakeDep & other) const - { - return - id == other.id - && ref == other.ref - && narHash == other.narHash - && flakeDeps == other.flakeDeps - && nonFlakeDeps == other.nonFlakeDeps; - } - - nlohmann::json toJson() const; -}; - -struct LockFile : FlakeInputs -{ - bool operator ==(const LockFile & other) const - { - return - flakeDeps == other.flakeDeps - && nonFlakeDeps == other.nonFlakeDeps; - } - - LockFile() {} - LockFile(const nlohmann::json & json) : FlakeInputs(json) {} - LockFile(FlakeDep && dep) - { - flakeDeps = std::move(dep.flakeDeps); - nonFlakeDeps = std::move(dep.nonFlakeDeps); - } - - nlohmann::json toJson() const; -}; - struct SourceInfo { // Immutable flakeref that this source tree was obtained from. diff --git a/src/libexpr/primops/lockfile.cc b/src/libexpr/primops/lockfile.cc new file mode 100644 index 000000000..ea0a93510 --- /dev/null +++ b/src/libexpr/primops/lockfile.cc @@ -0,0 +1,104 @@ +#include "lockfile.hh" +#include "store-api.hh" + +namespace nix::flake { + +AbstractDep::AbstractDep(const nlohmann::json & json) + : ref(json["uri"]) + , narHash(Hash((std::string) json["narHash"])) +{ + if (!ref.isImmutable()) + throw Error("lockfile contains mutable flakeref '%s'", ref); +} + +nlohmann::json AbstractDep::toJson() const +{ + nlohmann::json json; + json["uri"] = ref.to_string(); + json["narHash"] = narHash.to_string(SRI); + return json; +} + +Path AbstractDep::computeStorePath(Store & store) const +{ + return store.makeFixedOutputPath(true, narHash, "source"); +} + +FlakeDep::FlakeDep(const nlohmann::json & json) + : FlakeInputs(json) + , AbstractDep(json) + , id(json["id"]) +{ +} + +nlohmann::json FlakeDep::toJson() const +{ + auto json = FlakeInputs::toJson(); + json.update(AbstractDep::toJson()); + json["id"] = id; + return json; +} + +FlakeInputs::FlakeInputs(const nlohmann::json & json) +{ + auto nonFlakeInputs = json["nonFlakeInputs"]; + for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) + nonFlakeDeps.insert_or_assign(i.key(), NonFlakeDep(*i)); + + auto inputs = json["inputs"]; + for (auto i = inputs.begin(); i != inputs.end(); ++i) + flakeDeps.insert_or_assign(i.key(), FlakeDep(*i)); +} + +nlohmann::json FlakeInputs::toJson() const +{ + nlohmann::json json; + { + auto j = nlohmann::json::object(); + for (auto & i : nonFlakeDeps) + j[i.first] = i.second.toJson(); + json["nonFlakeInputs"] = std::move(j); + } + { + auto j = nlohmann::json::object(); + for (auto & i : flakeDeps) + j[i.first.to_string()] = i.second.toJson(); + json["inputs"] = std::move(j); + } + return json; +} + +nlohmann::json LockFile::toJson() const +{ + auto json = FlakeInputs::toJson(); + json["version"] = 2; + return json; +} + +LockFile LockFile::read(const Path & path) +{ + if (pathExists(path)) { + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 2) + throw Error("lock file '%s' has unsupported version %d", path, version); + + return LockFile(json); + } else + return LockFile(); +} + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) +{ + stream << lockFile.toJson().dump(4); // '4' = indentation in json file + return stream; +} + +void LockFile::write(const Path & path) const +{ + createDirs(dirOf(path)); + writeFile(path, fmt("%s\n", *this)); +} + +} diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/primops/lockfile.hh new file mode 100644 index 000000000..fee15c1da --- /dev/null +++ b/src/libexpr/primops/lockfile.hh @@ -0,0 +1,110 @@ +#pragma once + +#include "flakeref.hh" + +#include + +namespace nix { +class Store; +} + +namespace nix::flake { + +/* Common lock file information about a flake input, namely the + immutable ref and the NAR hash. */ +struct AbstractDep +{ + FlakeRef ref; + Hash narHash; + + AbstractDep(const FlakeRef & flakeRef, const Hash & narHash) + : ref(flakeRef), narHash(narHash) + { + assert(ref.isImmutable()); + }; + + AbstractDep(const nlohmann::json & json); + + nlohmann::json toJson() const; + + Path computeStorePath(Store & store) const; +}; + +/* Lock file information about a non-flake input. */ +struct NonFlakeDep : AbstractDep +{ + using AbstractDep::AbstractDep; + + bool operator ==(const NonFlakeDep & other) const + { + return ref == other.ref && narHash == other.narHash; + } +}; + +struct FlakeDep; + +/* Lock file information about the dependencies of a flake. */ +struct FlakeInputs +{ + std::map flakeDeps; + std::map nonFlakeDeps; + + FlakeInputs() {}; + FlakeInputs(const nlohmann::json & json); + + nlohmann::json toJson() const; +}; + +/* Lock file information about a flake input. */ +struct FlakeDep : FlakeInputs, AbstractDep +{ + FlakeId id; + + FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) + : AbstractDep(flakeRef, narHash), id(id) {}; + + FlakeDep(const nlohmann::json & json); + + bool operator ==(const FlakeDep & other) const + { + return + id == other.id + && ref == other.ref + && narHash == other.narHash + && flakeDeps == other.flakeDeps + && nonFlakeDeps == other.nonFlakeDeps; + } + + nlohmann::json toJson() const; +}; + +/* An entire lock file. Note that this cannot be a FlakeDep for the + top-level flake, because then the lock file would need to contain + the hash of the top-level flake, but committing the lock file + would invalidate that hash. */ +struct LockFile : FlakeInputs +{ + bool operator ==(const LockFile & other) const + { + return + flakeDeps == other.flakeDeps + && nonFlakeDeps == other.nonFlakeDeps; + } + + LockFile() {} + LockFile(const nlohmann::json & json) : FlakeInputs(json) {} + LockFile(FlakeDep && dep) + { + flakeDeps = std::move(dep.flakeDeps); + nonFlakeDeps = std::move(dep.nonFlakeDeps); + } + + nlohmann::json toJson() const; + + static LockFile read(const Path & path); + + void write(const Path & path) const; +}; + +} + From 5fe7be2409966d673d59d049c3fc6e7710d03b53 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:08:13 +0200 Subject: [PATCH 04/13] Rename dep -> input Also use nlohmann::json range-based for. --- src/libexpr/primops/flake.cc | 34 ++++++++++++------------- src/libexpr/primops/lockfile.cc | 28 ++++++++++----------- src/libexpr/primops/lockfile.hh | 44 ++++++++++++++++----------------- src/nix/installables.cc | 4 +-- 4 files changed, 54 insertions(+), 56 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index b38971c36..6919bc66b 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -316,7 +316,7 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) else assert(false); } -static std::pair updateLocks( +static std::pair updateLocks( EvalState & state, const FlakeRef & flakeRef, HandleLockFile handleLockFile, @@ -325,7 +325,7 @@ static std::pair updateLocks( { auto flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); - FlakeDep newEntry( + FlakeInput newEntry( flake.id, flake.sourceInfo.resolvedRef, flake.sourceInfo.narHash); @@ -333,28 +333,28 @@ static std::pair updateLocks( for (auto & input : flake.nonFlakeInputs) { auto & id = input.first; auto & ref = input.second; - auto i = oldEntry.nonFlakeDeps.find(id); - if (i != oldEntry.nonFlakeDeps.end()) { - newEntry.nonFlakeDeps.insert_or_assign(i->first, i->second); + auto i = oldEntry.nonFlakeInputs.find(id); + if (i != oldEntry.nonFlakeInputs.end()) { + newEntry.nonFlakeInputs.insert_or_assign(i->first, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update non-flake dependency '%s' in pure mode", id); auto nonFlake = getNonFlake(state, ref, id, allowedToUseRegistries(handleLockFile, false)); - newEntry.nonFlakeDeps.insert_or_assign(id, - NonFlakeDep( + newEntry.nonFlakeInputs.insert_or_assign(id, + NonFlakeInput( nonFlake.sourceInfo.resolvedRef, nonFlake.sourceInfo.narHash)); } } for (auto & inputRef : flake.inputs) { - auto i = oldEntry.flakeDeps.find(inputRef); - if (i != oldEntry.flakeDeps.end()) { - newEntry.flakeDeps.insert_or_assign(inputRef, i->second); + auto i = oldEntry.flakeInputs.find(inputRef); + if (i != oldEntry.flakeInputs.end()) { + newEntry.flakeInputs.insert_or_assign(inputRef, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update flake dependency '%s' in pure mode", inputRef); - newEntry.flakeDeps.insert_or_assign(inputRef, + newEntry.flakeInputs.insert_or_assign(inputRef, updateLocks(state, inputRef, handleLockFile, {}, false).second); } } @@ -434,7 +434,7 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo it doesn't appear in 'builtins'. */ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - auto lazyFlake = (FlakeDep *) args[0]->attrs; + auto lazyFlake = (FlakeInput *) args[0]->attrs; auto flake = getFlake(state, lazyFlake->ref, false); callFlake(state, flake, *lazyFlake, v); } @@ -448,10 +448,10 @@ void callFlake(EvalState & state, // ...}'. This attrset is passed lazily as an argument to 'outputs'. state.mkAttrs(v, - inputs.flakeDeps.size() + - inputs.nonFlakeDeps.size() + 8); + inputs.flakeInputs.size() + + inputs.nonFlakeInputs.size() + 8); - for (auto & dep : inputs.flakeDeps) { + for (auto & dep : inputs.flakeInputs) { auto vFlake = state.allocAttr(v, dep.second.id); auto vPrimOp = state.allocValue(); static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); @@ -460,11 +460,11 @@ void callFlake(EvalState & state, auto vArg = state.allocValue(); vArg->type = tNull; // FIXME: leak - vArg->attrs = (Bindings *) new FlakeDep(dep.second); // evil! also inefficient + vArg->attrs = (Bindings *) new FlakeInput(dep.second); // evil! also inefficient mkApp(*vFlake, *vPrimOp, *vArg); } - for (auto & dep : inputs.nonFlakeDeps) { + for (auto & dep : inputs.nonFlakeInputs) { auto vNonFlake = state.allocAttr(v, dep.first); state.mkAttrs(*vNonFlake, 8); diff --git a/src/libexpr/primops/lockfile.cc b/src/libexpr/primops/lockfile.cc index ea0a93510..97c748c66 100644 --- a/src/libexpr/primops/lockfile.cc +++ b/src/libexpr/primops/lockfile.cc @@ -3,7 +3,7 @@ namespace nix::flake { -AbstractDep::AbstractDep(const nlohmann::json & json) +AbstractInput::AbstractInput(const nlohmann::json & json) : ref(json["uri"]) , narHash(Hash((std::string) json["narHash"])) { @@ -11,7 +11,7 @@ AbstractDep::AbstractDep(const nlohmann::json & json) throw Error("lockfile contains mutable flakeref '%s'", ref); } -nlohmann::json AbstractDep::toJson() const +nlohmann::json AbstractInput::toJson() const { nlohmann::json json; json["uri"] = ref.to_string(); @@ -19,35 +19,33 @@ nlohmann::json AbstractDep::toJson() const return json; } -Path AbstractDep::computeStorePath(Store & store) const +Path AbstractInput::computeStorePath(Store & store) const { return store.makeFixedOutputPath(true, narHash, "source"); } -FlakeDep::FlakeDep(const nlohmann::json & json) +FlakeInput::FlakeInput(const nlohmann::json & json) : FlakeInputs(json) - , AbstractDep(json) + , AbstractInput(json) , id(json["id"]) { } -nlohmann::json FlakeDep::toJson() const +nlohmann::json FlakeInput::toJson() const { auto json = FlakeInputs::toJson(); - json.update(AbstractDep::toJson()); + json.update(AbstractInput::toJson()); json["id"] = id; return json; } FlakeInputs::FlakeInputs(const nlohmann::json & json) { - auto nonFlakeInputs = json["nonFlakeInputs"]; - for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) - nonFlakeDeps.insert_or_assign(i.key(), NonFlakeDep(*i)); + for (auto & i : json["nonFlakeInputs"].items()) + nonFlakeInputs.insert_or_assign(i.key(), NonFlakeInput(i.value())); - auto inputs = json["inputs"]; - for (auto i = inputs.begin(); i != inputs.end(); ++i) - flakeDeps.insert_or_assign(i.key(), FlakeDep(*i)); + for (auto & i : json["inputs"].items()) + flakeInputs.insert_or_assign(i.key(), FlakeInput(i.value())); } nlohmann::json FlakeInputs::toJson() const @@ -55,13 +53,13 @@ nlohmann::json FlakeInputs::toJson() const nlohmann::json json; { auto j = nlohmann::json::object(); - for (auto & i : nonFlakeDeps) + for (auto & i : nonFlakeInputs) j[i.first] = i.second.toJson(); json["nonFlakeInputs"] = std::move(j); } { auto j = nlohmann::json::object(); - for (auto & i : flakeDeps) + for (auto & i : flakeInputs) j[i.first.to_string()] = i.second.toJson(); json["inputs"] = std::move(j); } diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/primops/lockfile.hh index fee15c1da..f2e598528 100644 --- a/src/libexpr/primops/lockfile.hh +++ b/src/libexpr/primops/lockfile.hh @@ -12,18 +12,18 @@ namespace nix::flake { /* Common lock file information about a flake input, namely the immutable ref and the NAR hash. */ -struct AbstractDep +struct AbstractInput { FlakeRef ref; Hash narHash; - AbstractDep(const FlakeRef & flakeRef, const Hash & narHash) + AbstractInput(const FlakeRef & flakeRef, const Hash & narHash) : ref(flakeRef), narHash(narHash) { assert(ref.isImmutable()); }; - AbstractDep(const nlohmann::json & json); + AbstractInput(const nlohmann::json & json); nlohmann::json toJson() const; @@ -31,23 +31,23 @@ struct AbstractDep }; /* Lock file information about a non-flake input. */ -struct NonFlakeDep : AbstractDep +struct NonFlakeInput : AbstractInput { - using AbstractDep::AbstractDep; + using AbstractInput::AbstractInput; - bool operator ==(const NonFlakeDep & other) const + bool operator ==(const NonFlakeInput & other) const { return ref == other.ref && narHash == other.narHash; } }; -struct FlakeDep; +struct FlakeInput; /* Lock file information about the dependencies of a flake. */ struct FlakeInputs { - std::map flakeDeps; - std::map nonFlakeDeps; + std::map flakeInputs; + std::map nonFlakeInputs; FlakeInputs() {}; FlakeInputs(const nlohmann::json & json); @@ -56,29 +56,29 @@ struct FlakeInputs }; /* Lock file information about a flake input. */ -struct FlakeDep : FlakeInputs, AbstractDep +struct FlakeInput : FlakeInputs, AbstractInput { FlakeId id; - FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) - : AbstractDep(flakeRef, narHash), id(id) {}; + FlakeInput(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) + : AbstractInput(flakeRef, narHash), id(id) {}; - FlakeDep(const nlohmann::json & json); + FlakeInput(const nlohmann::json & json); - bool operator ==(const FlakeDep & other) const + bool operator ==(const FlakeInput & other) const { return id == other.id && ref == other.ref && narHash == other.narHash - && flakeDeps == other.flakeDeps - && nonFlakeDeps == other.nonFlakeDeps; + && flakeInputs == other.flakeInputs + && nonFlakeInputs == other.nonFlakeInputs; } nlohmann::json toJson() const; }; -/* An entire lock file. Note that this cannot be a FlakeDep for the +/* An entire lock file. Note that this cannot be a FlakeInput for the top-level flake, because then the lock file would need to contain the hash of the top-level flake, but committing the lock file would invalidate that hash. */ @@ -87,16 +87,16 @@ struct LockFile : FlakeInputs bool operator ==(const LockFile & other) const { return - flakeDeps == other.flakeDeps - && nonFlakeDeps == other.nonFlakeDeps; + flakeInputs == other.flakeInputs + && nonFlakeInputs == other.nonFlakeInputs; } LockFile() {} LockFile(const nlohmann::json & json) : FlakeInputs(json) {} - LockFile(FlakeDep && dep) + LockFile(FlakeInput && dep) { - flakeDeps = std::move(dep.flakeDeps); - nonFlakeDeps = std::move(dep.nonFlakeDeps); + flakeInputs = std::move(dep.flakeInputs); + nonFlakeInputs = std::move(dep.nonFlakeInputs); } nlohmann::json toJson() const; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index c44a37f1e..e7549b57c 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -206,11 +206,11 @@ void makeFlakeClosureGCRoot(Store & store, queue.pop(); /* Note: due to lazy fetching, these paths might not exist yet. */ - for (auto & dep : flake.flakeDeps) { + for (auto & dep : flake.flakeInputs) { closure.insert(dep.second.computeStorePath(store)); queue.push(dep.second); } - for (auto & dep : flake.nonFlakeDeps) + for (auto & dep : flake.nonFlakeInputs) closure.insert(dep.second.computeStorePath(store)); } From 4ec1a9ab40705b651fe4b2ef6186c0a6ae279681 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:33:49 +0200 Subject: [PATCH 05/13] Fix test --- tests/flakes.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/flakes.sh b/tests/flakes.sh index f44b9509f..f9f8e5272 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -245,6 +245,8 @@ git -C $flake3Dir commit -m 'Add nonFlakeInputs' # Check whether `nix build` works with a lockfile which is missing a nonFlakeInputs nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth +git -C $flake3Dir commit -m 'Update nonFlakeInputs' + # Check whether flake input fetching is lazy: flake3:sth does not # depend on flake2, so this shouldn't fail. rm -rf $TEST_HOME/.cache From 45b5c606ac44550de14562df4fa99773a81a1015 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:34:08 +0200 Subject: [PATCH 06/13] Don't register invalid paths as GC roots Unfortunately this doesn't work. Maybe we should keep separate roots for each path. --- src/nix/installables.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index e7549b57c..ca88ec0da 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -207,11 +207,16 @@ void makeFlakeClosureGCRoot(Store & store, /* Note: due to lazy fetching, these paths might not exist yet. */ for (auto & dep : flake.flakeInputs) { - closure.insert(dep.second.computeStorePath(store)); + auto path = dep.second.computeStorePath(store); + if (store.isValidPath(path)) + closure.insert(path); queue.push(dep.second); } - for (auto & dep : flake.nonFlakeInputs) - closure.insert(dep.second.computeStorePath(store)); + for (auto & dep : flake.nonFlakeInputs) { + auto path = dep.second.computeStorePath(store); + if (store.isValidPath(path)) + closure.insert(path); + } } if (closure.empty()) return; From 1c5067b9a7e1f561bf9e9e84642c495a50ca44a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:34:44 +0200 Subject: [PATCH 07/13] Check hash --- src/libexpr/primops/flake.cc | 6 ++++-- src/libexpr/primops/lockfile.hh | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 6919bc66b..52d8df69e 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -409,8 +409,6 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo { auto & path = sourceInfo.storePath; assert(state.store->isValidPath(path)); - // FIXME: turn into fetchGit etc. - // FIXME: check narHash. mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); if (sourceInfo.resolvedRef.rev) { @@ -436,6 +434,10 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V { auto lazyFlake = (FlakeInput *) args[0]->attrs; auto flake = getFlake(state, lazyFlake->ref, false); + + if (flake.sourceInfo.narHash != lazyFlake->narHash) + throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); + callFlake(state, flake, *lazyFlake, v); } diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/primops/lockfile.hh index f2e598528..b76124190 100644 --- a/src/libexpr/primops/lockfile.hh +++ b/src/libexpr/primops/lockfile.hh @@ -106,5 +106,7 @@ struct LockFile : FlakeInputs void write(const Path & path) const; }; +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); + } From 1e53a07712fba830eb3967cc16894992d5a33922 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:56:13 +0200 Subject: [PATCH 08/13] Make non-flake inputs lazy Also add a proper test for non-flake inputs. --- src/libexpr/primops/flake.cc | 44 ++++++++++++++++++++++++------------ src/libexpr/primops/flake.hh | 1 - src/nix/flake.cc | 2 +- tests/flakes.sh | 25 +++++++++++++++++--- 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 52d8df69e..f99738db5 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -276,7 +276,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe } // Get the `NonFlake` corresponding to a `FlakeRef`. -NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias, bool impureIsAllowed = false) +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); @@ -290,8 +290,6 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al if (state.allowedPaths) state.allowedPaths->insert(nonFlake.sourceInfo.storePath); - nonFlake.alias = alias; - return nonFlake; } @@ -339,7 +337,7 @@ static std::pair updateLocks( } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update non-flake dependency '%s' in pure mode", id); - auto nonFlake = getNonFlake(state, ref, id, allowedToUseRegistries(handleLockFile, false)); + auto nonFlake = getNonFlake(state, ref, allowedToUseRegistries(handleLockFile, false)); newEntry.nonFlakeInputs.insert_or_assign(id, NonFlakeInput( nonFlake.sourceInfo.resolvedRef, @@ -441,6 +439,25 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V callFlake(state, flake, *lazyFlake, v); } +static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs; + + auto nonFlake = getNonFlake(state, lazyNonFlake->ref); + + if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash) + throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", nonFlake.sourceInfo.resolvedRef); + + state.mkAttrs(v, 8); + + assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); + + mkString(*state.allocAttr(v, state.sOutPath), + nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); + + emitSourceInfoAttrs(state, nonFlake.sourceInfo, v); +} + void callFlake(EvalState & state, const Flake & flake, const FlakeInputs & inputs, @@ -468,16 +485,15 @@ void callFlake(EvalState & state, for (auto & dep : inputs.nonFlakeInputs) { auto vNonFlake = state.allocAttr(v, dep.first); - state.mkAttrs(*vNonFlake, 8); - - auto nonFlake = getNonFlake(state, dep.second.ref, dep.first); - - assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); - - mkString(*state.allocAttr(*vNonFlake, state.sOutPath), - nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); - - emitSourceInfoAttrs(state, nonFlake.sourceInfo, *vNonFlake); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new NonFlakeInput(dep.second); // evil! also inefficient + mkApp(*vNonFlake, *vPrimOp, *vArg); } mkString(*state.allocAttr(v, state.sDescription), flake.description); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index bbf35da02..b8d0da252 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -75,7 +75,6 @@ struct Flake struct NonFlake { - FlakeAlias alias; FlakeRef originalRef; SourceInfo sourceInfo; NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d229c7512..653154aaa 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -122,6 +122,7 @@ static nlohmann::json flakeToJson(const Flake & flake) return j; } +#if 0 static void printNonFlakeInfo(const NonFlake & nonFlake) { std::cout << fmt("ID: %s\n", nonFlake.alias); @@ -136,7 +137,6 @@ static nlohmann::json nonFlakeToJson(const NonFlake & nonFlake) return j; } -#if 0 // FIXME: merge info CmdFlakeInfo? struct CmdFlakeDeps : FlakeCommand { diff --git a/tests/flakes.sh b/tests/flakes.sh index f9f8e5272..1cd8259b9 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -83,7 +83,7 @@ git -C $flake3Dir add flake.nix git -C $flake3Dir commit -m 'Initial' cat > $nonFlakeDir/README.md < $flake3Dir/flake.nix < \$out + ''; + }; }; } EOF -git -C $flake3Dir add flake.nix +cp ./config.nix $flake3Dir + +git -C $flake3Dir add flake.nix config.nix git -C $flake3Dir commit -m 'Add nonFlakeInputs' -# Check whether `nix build` works with a lockfile which is missing a nonFlakeInputs +# Check whether `nix build` works with a lockfile which is missing a +# nonFlakeInputs. nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth git -C $flake3Dir commit -m 'Update nonFlakeInputs' +nix build -o $TEST_ROOT/result --flake-registry $registry flake3:fnord +[[ $(cat $TEST_ROOT/result) = FNORD ]] + # Check whether flake input fetching is lazy: flake3:sth does not # depend on flake2, so this shouldn't fail. rm -rf $TEST_HOME/.cache clearStore mv $flake2Dir $flake2Dir.tmp +mv $nonFlakeDir $nonFlakeDir.tmp nix build -o $TEST_ROOT/result --flake-registry $registry flake3:sth (! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy) +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:fnord) mv $flake2Dir.tmp $flake2Dir +mv $nonFlakeDir.tmp $nonFlakeDir +nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy flake3:fnord From 4d31cf83f288ed2e4a14f72e99b77859de981bb4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:57:18 +0200 Subject: [PATCH 09/13] Update flake.lock --- flake.lock | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b85571b61..727d16d73 100644 --- a/flake.lock +++ b/flake.lock @@ -1,10 +1,13 @@ { "inputs": { "nixpkgs": { - "narHash": "sha256-rMiWaLXkhizEEMEeMDutUl0Y/c+VEjfjvMkvBwvuQJU=", - "uri": "github:edolstra/nixpkgs/eeeffd24cd7e407cfaa99e98cfbb8f93bf4cc033" + "id": "nixpkgs", + "inputs": {}, + "narHash": "sha256-eYtxncIMFVmOHaHBtTdPGcs/AnJqKqA6tHCm0UmPYQU=", + "nonFlakeInputs": {}, + "uri": "github:edolstra/nixpkgs/e9d5882bb861dc48f8d46960e7c820efdbe8f9c1" } }, "nonFlakeInputs": {}, - "version": 1 + "version": 2 } From 087530dec40bd5ab23fe2da83fca517bb91d2282 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 21:07:55 +0200 Subject: [PATCH 10/13] Add comments --- src/libexpr/primops/flake.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index f99738db5..000a43764 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -314,6 +314,15 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) else assert(false); } +/* Given a flakeref and its subtree of the lockfile, return an updated + subtree of the lockfile. That is, if the 'flake.nix' of the + referenced flake has inputs that don't have a corresponding entry + in the lockfile, they're added to the lockfile; conversely, any + lockfile entries that don't have a corresponding entry in flake.nix + are removed. + + Note that this is lazy: we only recursively fetch inputs that are + not in the lockfile yet. */ static std::pair updateLocks( EvalState & state, const FlakeRef & flakeRef, @@ -360,9 +369,8 @@ static std::pair updateLocks( return {flake, newEntry}; } -/* Given a flake reference, recursively fetch it and its dependencies. - FIXME: this should return a graph of flakes. -*/ +/* Compute an in-memory lockfile for the specified top-level flake, + and optionally write it to file, it the flake is writable. */ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); From ce225615c3ee08b7b63a8488dbf74ff2598d8d74 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 21:10:53 +0200 Subject: [PATCH 11/13] Eliminate duplicate fetching of the top-level flake --- src/libexpr/primops/flake.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 000a43764..734f650f2 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -325,13 +325,11 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) not in the lockfile yet. */ static std::pair updateLocks( EvalState & state, - const FlakeRef & flakeRef, + const Flake & flake, HandleLockFile handleLockFile, const FlakeInputs & oldEntry, bool topRef) { - auto flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); - FlakeInput newEntry( flake.id, flake.sourceInfo.resolvedRef, @@ -362,7 +360,9 @@ static std::pair updateLocks( if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update flake dependency '%s' in pure mode", inputRef); newEntry.flakeInputs.insert_or_assign(inputRef, - updateLocks(state, inputRef, handleLockFile, {}, false).second); + updateLocks(state, + getFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false)), + handleLockFile, {}, false).second); } } @@ -385,9 +385,8 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); } - // FIXME: get rid of duplicate getFlake call LockFile lockFile(updateLocks( - state, topRef, handleLockFile, oldLockFile, true).second); + state, flake, handleLockFile, oldLockFile, true).second); if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { From 1b057929885fd3f339d4c85b44ad9f10fef7d8a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 22:35:43 +0200 Subject: [PATCH 12/13] Shorter syntax for referencing flake outputs Fixes #2819. --- src/libexpr/primops/flake.cc | 16 +++++++++++++--- tests/flakes.sh | 12 ++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 734f650f2..793d6da35 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -468,10 +468,12 @@ static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args void callFlake(EvalState & state, const Flake & flake, const FlakeInputs & inputs, - Value & v) + Value & vRes) { - // Construct the resulting attrset '{description, outputs, - // ...}'. This attrset is passed lazily as an argument to 'outputs'. + // Construct the resulting attrset '{outputs, ...}'. This attrset + // is passed lazily as an argument to the 'outputs' function. + + auto & v = *state.allocValue(); state.mkAttrs(v, inputs.flakeInputs.size() + @@ -513,6 +515,14 @@ void callFlake(EvalState & state, v.attrs->push_back(Attr(state.symbols.create("self"), &v)); v.attrs->sort(); + + /* For convenience, put the outputs directly in the result, so you + can refer to an output of an input as 'inputs.foo.bar' rather + than 'inputs.foo.outputs.bar'. */ + auto v2 = *state.allocValue(); + state.eval(state.parseExprFromString("res: res.outputs // res", "/"), v2); + + state.callFunction(v2, v, vRes, noPos); } void callFlake(EvalState & state, diff --git a/tests/flakes.sh b/tests/flakes.sh index 1cd8259b9..c380b405b 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -55,7 +55,7 @@ cat > $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Wed, 5 Jun 2019 16:51:54 +0200 Subject: [PATCH 13/13] Move flake-related stuff to src/libexpr/flake --- src/libexpr/eval.cc | 2 +- src/libexpr/{primops => flake}/flake.cc | 2 +- src/libexpr/{primops => flake}/flake.hh | 0 src/libexpr/{primops => flake}/flakeref.cc | 0 src/libexpr/{primops => flake}/flakeref.hh | 0 src/libexpr/{primops => flake}/lockfile.cc | 0 src/libexpr/{primops => flake}/lockfile.hh | 0 src/libexpr/local.mk | 7 ++++++- src/nix/flake.cc | 2 +- src/nix/installables.cc | 2 +- 10 files changed, 10 insertions(+), 5 deletions(-) rename src/libexpr/{primops => flake}/flake.cc (99%) rename src/libexpr/{primops => flake}/flake.hh (100%) rename src/libexpr/{primops => flake}/flakeref.cc (100%) rename src/libexpr/{primops => flake}/flakeref.hh (100%) rename src/libexpr/{primops => flake}/lockfile.cc (100%) rename src/libexpr/{primops => flake}/lockfile.hh (100%) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0f8a105b1..46c622ee8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -7,7 +7,7 @@ #include "eval-inline.hh" #include "download.hh" #include "json.hh" -#include "primops/flake.hh" +#include "flake/flake.hh" #include #include diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/flake/flake.cc similarity index 99% rename from src/libexpr/primops/flake.cc rename to src/libexpr/flake/flake.cc index 793d6da35..bb0543541 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -2,7 +2,7 @@ #include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" -#include "fetchGit.hh" +#include "primops/fetchGit.hh" #include "download.hh" #include "args.hh" diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/flake/flake.hh similarity index 100% rename from src/libexpr/primops/flake.hh rename to src/libexpr/flake/flake.hh diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/flake/flakeref.cc similarity index 100% rename from src/libexpr/primops/flakeref.cc rename to src/libexpr/flake/flakeref.cc diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/flake/flakeref.hh similarity index 100% rename from src/libexpr/primops/flakeref.hh rename to src/libexpr/flake/flakeref.hh diff --git a/src/libexpr/primops/lockfile.cc b/src/libexpr/flake/lockfile.cc similarity index 100% rename from src/libexpr/primops/lockfile.cc rename to src/libexpr/flake/lockfile.cc diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/flake/lockfile.hh similarity index 100% rename from src/libexpr/primops/lockfile.hh rename to src/libexpr/flake/lockfile.hh diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ccd5293e4..a9cb6b7b6 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/primops/*.cc) \ + $(wildcard $(d)/flake/*.cc) \ + $(d)/lexer-tab.cc \ + $(d)/parser-tab.cc libexpr_LIBS = libutil libstore diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 653154aaa..af1a361b3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -4,7 +4,7 @@ #include "progress-bar.hh" #include "eval.hh" #include "eval-inline.hh" -#include "primops/flake.hh" +#include "flake/flake.hh" #include "get-drvs.hh" #include "store-api.hh" diff --git a/src/nix/installables.cc b/src/nix/installables.cc index ca88ec0da..a85295a09 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -7,7 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" -#include "primops/flake.hh" +#include "flake/flake.hh" #include #include