forked from lix-project/lix
Merge pull request #2920 from NixOS/lazy-flakes
Lazy flake input fetching
This commit is contained in:
Eelco Dolstra
GitHub
commit
671f16aee0
|
|
@ -1,10 +1,13 @@
|
|||
{
|
||||
"inputs": {
|
||||
"nixpkgs": {
|
||||
"narHash": "sha256-rMiWaLXkhizEEMEeMDutUl0Y/c+VEjfjvMkvBwvuQJU=",
|
||||
"uri": "github:edolstra/nixpkgs/eeeffd24cd7e407cfaa99e98cfbb8f93bf4cc033"
|
||||
"id": "nixpkgs",
|
||||
"inputs": {},
|
||||
"narHash": "sha256-eYtxncIMFVmOHaHBtTdPGcs/AnJqKqA6tHCm0UmPYQU=",
|
||||
"nonFlakeInputs": {},
|
||||
"uri": "github:edolstra/nixpkgs/e9d5882bb861dc48f8d46960e7c820efdbe8f9c1"
|
||||
}
|
||||
},
|
||||
"nonFlakeInputs": {},
|
||||
"version": 1
|
||||
"version": 2
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
#include "eval-inline.hh"
|
||||
#include "download.hh"
|
||||
#include "json.hh"
|
||||
#include "primops/flake.hh"
|
||||
#include "flake/flake.hh"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
#include "flake.hh"
|
||||
#include "lockfile.hh"
|
||||
#include "primops.hh"
|
||||
#include "eval-inline.hh"
|
||||
#include "fetchGit.hh"
|
||||
#include "primops/fetchGit.hh"
|
||||
#include "download.hh"
|
||||
#include "args.hh"
|
||||
|
||||
|
|
@ -43,106 +44,13 @@ std::shared_ptr<FlakeRegistry> readRegistry(const Path & path)
|
|||
void writeRegistry(const FlakeRegistry & registry, const Path & path)
|
||||
{
|
||||
nlohmann::json json;
|
||||
json["version"] = 1;
|
||||
json["version"] = 2;
|
||||
for (auto elem : registry.entries)
|
||||
json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} };
|
||||
createDirs(dirOf(path));
|
||||
writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file.
|
||||
}
|
||||
|
||||
LockFile::FlakeEntry readFlakeEntry(nlohmann::json json)
|
||||
{
|
||||
FlakeRef flakeRef(json["uri"]);
|
||||
if (!flakeRef.isImmutable())
|
||||
throw Error("cannot use mutable flake '%s' in pure mode", flakeRef);
|
||||
|
||||
LockFile::FlakeEntry entry(flakeRef, Hash((std::string) json["narHash"]));
|
||||
|
||||
auto nonFlakeInputs = json["nonFlakeInputs"];
|
||||
|
||||
for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) {
|
||||
FlakeRef flakeRef(i->value("uri", ""));
|
||||
if (!flakeRef.isImmutable())
|
||||
throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef);
|
||||
LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", "")));
|
||||
entry.nonFlakeEntries.insert_or_assign(i.key(), nonEntry);
|
||||
}
|
||||
|
||||
auto inputs = json["inputs"];
|
||||
|
||||
for (auto i = inputs.begin(); i != inputs.end(); ++i)
|
||||
entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i));
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
LockFile readLockFile(const Path & path)
|
||||
{
|
||||
LockFile lockFile;
|
||||
|
||||
if (!pathExists(path))
|
||||
return lockFile;
|
||||
|
||||
auto json = nlohmann::json::parse(readFile(path));
|
||||
|
||||
auto version = json.value("version", 0);
|
||||
if (version != 1)
|
||||
throw Error("lock file '%s' has unsupported version %d", path, version);
|
||||
|
||||
auto nonFlakeInputs = json["nonFlakeInputs"];
|
||||
|
||||
for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) {
|
||||
FlakeRef flakeRef(i->value("uri", ""));
|
||||
LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", "")));
|
||||
if (!flakeRef.isImmutable())
|
||||
throw Error("found mutable FlakeRef '%s' in lockfile at path %s", flakeRef, path);
|
||||
lockFile.nonFlakeEntries.insert_or_assign(i.key(), nonEntry);
|
||||
}
|
||||
|
||||
auto inputs = json["inputs"];
|
||||
|
||||
for (auto i = inputs.begin(); i != inputs.end(); ++i)
|
||||
lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i));
|
||||
|
||||
return lockFile;
|
||||
}
|
||||
|
||||
nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry)
|
||||
{
|
||||
nlohmann::json json;
|
||||
json["uri"] = entry.ref.to_string();
|
||||
json["narHash"] = entry.narHash.to_string(SRI);
|
||||
for (auto & x : entry.nonFlakeEntries) {
|
||||
json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string();
|
||||
json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI);
|
||||
}
|
||||
for (auto & x : entry.flakeEntries)
|
||||
json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second);
|
||||
return json;
|
||||
}
|
||||
|
||||
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
|
||||
{
|
||||
nlohmann::json json;
|
||||
json["version"] = 1;
|
||||
json["nonFlakeInputs"] = nlohmann::json::object();
|
||||
for (auto & x : lockFile.nonFlakeEntries) {
|
||||
json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string();
|
||||
json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI);
|
||||
}
|
||||
json["inputs"] = nlohmann::json::object();
|
||||
for (auto & x : lockFile.flakeEntries)
|
||||
json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second);
|
||||
stream << json.dump(4); // '4' = indentation in json file
|
||||
return stream;
|
||||
}
|
||||
|
||||
void writeLockFile(const LockFile & lockFile, const Path & path)
|
||||
{
|
||||
createDirs(dirOf(path));
|
||||
writeFile(path, fmt("%s\n", lockFile));
|
||||
}
|
||||
|
||||
Path getUserRegistryPath()
|
||||
{
|
||||
return getHome() + "/.config/nix/registry.json";
|
||||
|
|
@ -368,7 +276,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe
|
|||
}
|
||||
|
||||
// Get the `NonFlake` corresponding to a `FlakeRef`.
|
||||
NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias, bool impureIsAllowed = false)
|
||||
NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false)
|
||||
{
|
||||
auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed);
|
||||
debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string());
|
||||
|
|
@ -382,38 +290,9 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al
|
|||
if (state.allowedPaths)
|
||||
state.allowedPaths->insert(nonFlake.sourceInfo.storePath);
|
||||
|
||||
nonFlake.alias = alias;
|
||||
|
||||
return nonFlake;
|
||||
}
|
||||
|
||||
LockFile entryToLockFile(const LockFile::FlakeEntry & entry)
|
||||
{
|
||||
LockFile lockFile;
|
||||
lockFile.flakeEntries = entry.flakeEntries;
|
||||
lockFile.nonFlakeEntries = entry.nonFlakeEntries;
|
||||
return lockFile;
|
||||
}
|
||||
|
||||
LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake)
|
||||
{
|
||||
LockFile::FlakeEntry entry(
|
||||
resolvedFlake.flake.sourceInfo.resolvedRef,
|
||||
resolvedFlake.flake.sourceInfo.narHash);
|
||||
|
||||
for (auto & info : resolvedFlake.flakeDeps)
|
||||
entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second));
|
||||
|
||||
for (auto & nonFlake : resolvedFlake.nonFlakeDeps) {
|
||||
LockFile::NonFlakeEntry nonEntry(
|
||||
nonFlake.sourceInfo.resolvedRef,
|
||||
nonFlake.sourceInfo.narHash);
|
||||
entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry);
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
bool allowedToWrite(HandleLockFile handle)
|
||||
{
|
||||
return handle == UpdateLockFile || handle == RecreateLockFile;
|
||||
|
|
@ -435,70 +314,84 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef)
|
|||
else assert(false);
|
||||
}
|
||||
|
||||
ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef,
|
||||
HandleLockFile handleLockFile, LockFile lockFile = {}, bool topRef = false)
|
||||
/* Given a flakeref and its subtree of the lockfile, return an updated
|
||||
subtree of the lockfile. That is, if the 'flake.nix' of the
|
||||
referenced flake has inputs that don't have a corresponding entry
|
||||
in the lockfile, they're added to the lockfile; conversely, any
|
||||
lockfile entries that don't have a corresponding entry in flake.nix
|
||||
are removed.
|
||||
|
||||
Note that this is lazy: we only recursively fetch inputs that are
|
||||
not in the lockfile yet. */
|
||||
static std::pair<Flake, FlakeInput> updateLocks(
|
||||
EvalState & state,
|
||||
const Flake & flake,
|
||||
HandleLockFile handleLockFile,
|
||||
const FlakeInputs & oldEntry,
|
||||
bool topRef)
|
||||
{
|
||||
Flake flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef));
|
||||
FlakeInput newEntry(
|
||||
flake.id,
|
||||
flake.sourceInfo.resolvedRef,
|
||||
flake.sourceInfo.narHash);
|
||||
|
||||
ResolvedFlake deps(flake);
|
||||
|
||||
for (auto & nonFlakeInfo : flake.nonFlakeInputs) {
|
||||
FlakeRef ref = nonFlakeInfo.second;
|
||||
auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first);
|
||||
if (i != lockFile.nonFlakeEntries.end()) {
|
||||
NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first);
|
||||
if (nonFlake.sourceInfo.narHash != i->second.narHash)
|
||||
throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string());
|
||||
deps.nonFlakeDeps.push_back(nonFlake);
|
||||
for (auto & input : flake.nonFlakeInputs) {
|
||||
auto & id = input.first;
|
||||
auto & ref = input.second;
|
||||
auto i = oldEntry.nonFlakeInputs.find(id);
|
||||
if (i != oldEntry.nonFlakeInputs.end()) {
|
||||
newEntry.nonFlakeInputs.insert_or_assign(i->first, i->second);
|
||||
} else {
|
||||
if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries)
|
||||
throw Error("cannot update non-flake dependency '%s' in pure mode", nonFlakeInfo.first);
|
||||
deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first, allowedToUseRegistries(handleLockFile, false)));
|
||||
throw Error("cannot update non-flake dependency '%s' in pure mode", id);
|
||||
auto nonFlake = getNonFlake(state, ref, allowedToUseRegistries(handleLockFile, false));
|
||||
newEntry.nonFlakeInputs.insert_or_assign(id,
|
||||
NonFlakeInput(
|
||||
nonFlake.sourceInfo.resolvedRef,
|
||||
nonFlake.sourceInfo.narHash));
|
||||
}
|
||||
}
|
||||
|
||||
for (auto newFlakeRef : flake.inputs) {
|
||||
auto i = lockFile.flakeEntries.find(newFlakeRef);
|
||||
if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible
|
||||
ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second));
|
||||
if (newResFlake.flake.sourceInfo.narHash != i->second.narHash)
|
||||
throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string());
|
||||
deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake);
|
||||
for (auto & inputRef : flake.inputs) {
|
||||
auto i = oldEntry.flakeInputs.find(inputRef);
|
||||
if (i != oldEntry.flakeInputs.end()) {
|
||||
newEntry.flakeInputs.insert_or_assign(inputRef, i->second);
|
||||
} else {
|
||||
if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries)
|
||||
throw Error("cannot update flake dependency '%s' in pure mode", newFlakeRef.to_string());
|
||||
deps.flakeDeps.insert_or_assign(newFlakeRef, resolveFlakeFromLockFile(state, newFlakeRef, handleLockFile));
|
||||
throw Error("cannot update flake dependency '%s' in pure mode", inputRef);
|
||||
newEntry.flakeInputs.insert_or_assign(inputRef,
|
||||
updateLocks(state,
|
||||
getFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false)),
|
||||
handleLockFile, {}, false).second);
|
||||
}
|
||||
}
|
||||
|
||||
return deps;
|
||||
return {flake, newEntry};
|
||||
}
|
||||
|
||||
/* Given a flake reference, recursively fetch it and its dependencies.
|
||||
FIXME: this should return a graph of flakes.
|
||||
*/
|
||||
/* Compute an in-memory lockfile for the specified top-level flake,
|
||||
and optionally write it to file, it the flake is writable. */
|
||||
ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile)
|
||||
{
|
||||
Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true));
|
||||
auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true));
|
||||
|
||||
LockFile oldLockFile;
|
||||
|
||||
if (!recreateLockFile(handleLockFile)) {
|
||||
// If recreateLockFile, start with an empty lockfile
|
||||
// FIXME: symlink attack
|
||||
oldLockFile = readLockFile(
|
||||
oldLockFile = LockFile::read(
|
||||
state.store->toRealPath(flake.sourceInfo.storePath)
|
||||
+ "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock");
|
||||
}
|
||||
|
||||
LockFile lockFile(oldLockFile);
|
||||
|
||||
ResolvedFlake resFlake = resolveFlakeFromLockFile(state, topRef, handleLockFile, lockFile, true);
|
||||
lockFile = entryToLockFile(dependenciesToFlakeEntry(resFlake));
|
||||
LockFile lockFile(updateLocks(
|
||||
state, flake, handleLockFile, oldLockFile, true).second);
|
||||
|
||||
if (!(lockFile == oldLockFile)) {
|
||||
if (allowedToWrite(handleLockFile)) {
|
||||
if (auto refData = std::get_if<FlakeRef::IsPath>(&topRef.data)) {
|
||||
writeLockFile(lockFile, refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock");
|
||||
lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock");
|
||||
|
||||
// Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store.
|
||||
runProgram("git", true, { "-C", refData->path, "add",
|
||||
|
|
@ -509,7 +402,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc
|
|||
warn("using updated lockfile without writing it to file");
|
||||
}
|
||||
|
||||
return resFlake;
|
||||
return ResolvedFlake(std::move(flake), std::move(lockFile));
|
||||
}
|
||||
|
||||
void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile)
|
||||
|
|
@ -520,7 +413,7 @@ void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateL
|
|||
static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs)
|
||||
{
|
||||
auto & path = sourceInfo.storePath;
|
||||
state.store->isValidPath(path);
|
||||
assert(state.store->isValidPath(path));
|
||||
mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path});
|
||||
|
||||
if (sourceInfo.resolvedRef.rev) {
|
||||
|
|
@ -539,40 +432,104 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo
|
|||
std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S")));
|
||||
}
|
||||
|
||||
void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)
|
||||
/* Helper primop to make callFlake (below) fetch/call its inputs
|
||||
lazily. Note that this primop cannot be called by user code since
|
||||
it doesn't appear in 'builtins'. */
|
||||
static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
{
|
||||
// Construct the resulting attrset '{description, outputs,
|
||||
// ...}'. This attrset is passed lazily as an argument to 'outputs'.
|
||||
auto lazyFlake = (FlakeInput *) args[0]->attrs;
|
||||
auto flake = getFlake(state, lazyFlake->ref, false);
|
||||
|
||||
state.mkAttrs(v, resFlake.flakeDeps.size() + resFlake.nonFlakeDeps.size() + 8);
|
||||
if (flake.sourceInfo.narHash != lazyFlake->narHash)
|
||||
throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef);
|
||||
|
||||
for (auto info : resFlake.flakeDeps) {
|
||||
const ResolvedFlake newResFlake = info.second;
|
||||
auto vFlake = state.allocAttr(v, newResFlake.flake.id);
|
||||
callFlake(state, newResFlake, *vFlake);
|
||||
}
|
||||
callFlake(state, flake, *lazyFlake, v);
|
||||
}
|
||||
|
||||
for (const NonFlake nonFlake : resFlake.nonFlakeDeps) {
|
||||
auto vNonFlake = state.allocAttr(v, nonFlake.alias);
|
||||
state.mkAttrs(*vNonFlake, 8);
|
||||
static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
{
|
||||
auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs;
|
||||
|
||||
state.store->isValidPath(nonFlake.sourceInfo.storePath);
|
||||
mkString(*state.allocAttr(*vNonFlake, state.sOutPath),
|
||||
auto nonFlake = getNonFlake(state, lazyNonFlake->ref);
|
||||
|
||||
if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash)
|
||||
throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", nonFlake.sourceInfo.resolvedRef);
|
||||
|
||||
state.mkAttrs(v, 8);
|
||||
|
||||
assert(state.store->isValidPath(nonFlake.sourceInfo.storePath));
|
||||
|
||||
mkString(*state.allocAttr(v, state.sOutPath),
|
||||
nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath});
|
||||
|
||||
emitSourceInfoAttrs(state, nonFlake.sourceInfo, *vNonFlake);
|
||||
emitSourceInfoAttrs(state, nonFlake.sourceInfo, v);
|
||||
}
|
||||
|
||||
void callFlake(EvalState & state,
|
||||
const Flake & flake,
|
||||
const FlakeInputs & inputs,
|
||||
Value & vRes)
|
||||
{
|
||||
// Construct the resulting attrset '{outputs, ...}'. This attrset
|
||||
// is passed lazily as an argument to the 'outputs' function.
|
||||
|
||||
auto & v = *state.allocValue();
|
||||
|
||||
state.mkAttrs(v,
|
||||
inputs.flakeInputs.size() +
|
||||
inputs.nonFlakeInputs.size() + 8);
|
||||
|
||||
for (auto & dep : inputs.flakeInputs) {
|
||||
auto vFlake = state.allocAttr(v, dep.second.id);
|
||||
auto vPrimOp = state.allocValue();
|
||||
static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake"));
|
||||
vPrimOp->type = tPrimOp;
|
||||
vPrimOp->primOp = primOp;
|
||||
auto vArg = state.allocValue();
|
||||
vArg->type = tNull;
|
||||
// FIXME: leak
|
||||
vArg->attrs = (Bindings *) new FlakeInput(dep.second); // evil! also inefficient
|
||||
mkApp(*vFlake, *vPrimOp, *vArg);
|
||||
}
|
||||
|
||||
mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description);
|
||||
for (auto & dep : inputs.nonFlakeInputs) {
|
||||
auto vNonFlake = state.allocAttr(v, dep.first);
|
||||
auto vPrimOp = state.allocValue();
|
||||
static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake"));
|
||||
vPrimOp->type = tPrimOp;
|
||||
vPrimOp->primOp = primOp;
|
||||
auto vArg = state.allocValue();
|
||||
vArg->type = tNull;
|
||||
// FIXME: leak
|
||||
vArg->attrs = (Bindings *) new NonFlakeInput(dep.second); // evil! also inefficient
|
||||
mkApp(*vNonFlake, *vPrimOp, *vArg);
|
||||
}
|
||||
|
||||
emitSourceInfoAttrs(state, resFlake.flake.sourceInfo, v);
|
||||
mkString(*state.allocAttr(v, state.sDescription), flake.description);
|
||||
|
||||
emitSourceInfoAttrs(state, flake.sourceInfo, v);
|
||||
|
||||
auto vOutputs = state.allocAttr(v, state.symbols.create("outputs"));
|
||||
mkApp(*vOutputs, *resFlake.flake.vOutputs, v);
|
||||
mkApp(*vOutputs, *flake.vOutputs, v);
|
||||
|
||||
v.attrs->push_back(Attr(state.symbols.create("self"), &v));
|
||||
|
||||
v.attrs->sort();
|
||||
|
||||
/* For convenience, put the outputs directly in the result, so you
|
||||
can refer to an output of an input as 'inputs.foo.bar' rather
|
||||
than 'inputs.foo.outputs.bar'. */
|
||||
auto v2 = *state.allocValue();
|
||||
state.eval(state.parseExprFromString("res: res.outputs // res", "/"), v2);
|
||||
|
||||
state.callFunction(v2, v, vRes, noPos);
|
||||
}
|
||||
|
||||
void callFlake(EvalState & state,
|
||||
const ResolvedFlake & resFlake,
|
||||
Value & v)
|
||||
{
|
||||
callFlake(state, resFlake.flake, resFlake.lockFile, v);
|
||||
}
|
||||
|
||||
// This function is exposed to be used in nix files.
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
#include "flakeref.hh"
|
||||
|
||||
#include <variant>
|
||||
#include "lockfile.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -19,51 +20,12 @@ struct FlakeRegistry
|
|||
std::map<FlakeRef, FlakeRef> entries;
|
||||
};
|
||||
|
||||
struct LockFile
|
||||
{
|
||||
struct NonFlakeEntry
|
||||
{
|
||||
FlakeRef ref;
|
||||
Hash narHash;
|
||||
NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {};
|
||||
|
||||
bool operator ==(const NonFlakeEntry & other) const
|
||||
{
|
||||
return ref == other.ref && narHash == other.narHash;
|
||||
}
|
||||
};
|
||||
|
||||
struct FlakeEntry
|
||||
{
|
||||
FlakeRef ref;
|
||||
Hash narHash;
|
||||
std::map<FlakeRef, FlakeEntry> flakeEntries;
|
||||
std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries;
|
||||
FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {};
|
||||
|
||||
bool operator ==(const FlakeEntry & other) const
|
||||
{
|
||||
return
|
||||
ref == other.ref
|
||||
&& narHash == other.narHash
|
||||
&& flakeEntries == other.flakeEntries
|
||||
&& nonFlakeEntries == other.nonFlakeEntries;
|
||||
}
|
||||
};
|
||||
|
||||
std::map<FlakeRef, FlakeEntry> flakeEntries;
|
||||
std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries;
|
||||
|
||||
bool operator ==(const LockFile & other) const
|
||||
{
|
||||
return
|
||||
flakeEntries == other.flakeEntries
|
||||
&& nonFlakeEntries == other.nonFlakeEntries;
|
||||
}
|
||||
};
|
||||
|
||||
typedef std::vector<std::shared_ptr<FlakeRegistry>> Registries;
|
||||
|
||||
std::shared_ptr<FlakeRegistry> readRegistry(const Path &);
|
||||
|
||||
void writeRegistry(const FlakeRegistry &, const Path &);
|
||||
|
||||
Path getUserRegistryPath();
|
||||
|
||||
enum HandleLockFile : unsigned int
|
||||
|
|
@ -75,10 +37,6 @@ enum HandleLockFile : unsigned int
|
|||
, UseNewLockFile // `RecreateLockFile` without writing to file
|
||||
};
|
||||
|
||||
std::shared_ptr<FlakeRegistry> readRegistry(const Path &);
|
||||
|
||||
void writeRegistry(const FlakeRegistry &, const Path &);
|
||||
|
||||
struct SourceInfo
|
||||
{
|
||||
// Immutable flakeref that this source tree was obtained from.
|
||||
|
|
@ -117,7 +75,6 @@ struct Flake
|
|||
|
||||
struct NonFlake
|
||||
{
|
||||
FlakeAlias alias;
|
||||
FlakeRef originalRef;
|
||||
SourceInfo sourceInfo;
|
||||
NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo)
|
||||
|
|
@ -129,14 +86,21 @@ Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed);
|
|||
struct ResolvedFlake
|
||||
{
|
||||
Flake flake;
|
||||
std::map<FlakeRef, ResolvedFlake> flakeDeps; // The key in this map, is the originalRef as written in flake.nix
|
||||
std::vector<NonFlake> nonFlakeDeps;
|
||||
ResolvedFlake(const Flake & flake) : flake(flake) {}
|
||||
LockFile lockFile;
|
||||
ResolvedFlake(Flake && flake, LockFile && lockFile)
|
||||
: flake(flake), lockFile(lockFile) {}
|
||||
};
|
||||
|
||||
ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile);
|
||||
|
||||
void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v);
|
||||
void callFlake(EvalState & state,
|
||||
const Flake & flake,
|
||||
const FlakeInputs & inputs,
|
||||
Value & v);
|
||||
|
||||
void callFlake(EvalState & state,
|
||||
const ResolvedFlake & resFlake,
|
||||
Value & v);
|
||||
|
||||
void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile);
|
||||
|
||||
102
src/libexpr/flake/lockfile.cc
Normal file
102
src/libexpr/flake/lockfile.cc
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
#include "lockfile.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix::flake {
|
||||
|
||||
AbstractInput::AbstractInput(const nlohmann::json & json)
|
||||
: ref(json["uri"])
|
||||
, narHash(Hash((std::string) json["narHash"]))
|
||||
{
|
||||
if (!ref.isImmutable())
|
||||
throw Error("lockfile contains mutable flakeref '%s'", ref);
|
||||
}
|
||||
|
||||
nlohmann::json AbstractInput::toJson() const
|
||||
{
|
||||
nlohmann::json json;
|
||||
json["uri"] = ref.to_string();
|
||||
json["narHash"] = narHash.to_string(SRI);
|
||||
return json;
|
||||
}
|
||||
|
||||
Path AbstractInput::computeStorePath(Store & store) const
|
||||
{
|
||||
return store.makeFixedOutputPath(true, narHash, "source");
|
||||
}
|
||||
|
||||
FlakeInput::FlakeInput(const nlohmann::json & json)
|
||||
: FlakeInputs(json)
|
||||
, AbstractInput(json)
|
||||
, id(json["id"])
|
||||
{
|
||||
}
|
||||
|
||||
nlohmann::json FlakeInput::toJson() const
|
||||
{
|
||||
auto json = FlakeInputs::toJson();
|
||||
json.update(AbstractInput::toJson());
|
||||
json["id"] = id;
|
||||
return json;
|
||||
}
|
||||
|
||||
FlakeInputs::FlakeInputs(const nlohmann::json & json)
|
||||
{
|
||||
for (auto & i : json["nonFlakeInputs"].items())
|
||||
nonFlakeInputs.insert_or_assign(i.key(), NonFlakeInput(i.value()));
|
||||
|
||||
for (auto & i : json["inputs"].items())
|
||||
flakeInputs.insert_or_assign(i.key(), FlakeInput(i.value()));
|
||||
}
|
||||
|
||||
nlohmann::json FlakeInputs::toJson() const
|
||||
{
|
||||
nlohmann::json json;
|
||||
{
|
||||
auto j = nlohmann::json::object();
|
||||
for (auto & i : nonFlakeInputs)
|
||||
j[i.first] = i.second.toJson();
|
||||
json["nonFlakeInputs"] = std::move(j);
|
||||
}
|
||||
{
|
||||
auto j = nlohmann::json::object();
|
||||
for (auto & i : flakeInputs)
|
||||
j[i.first.to_string()] = i.second.toJson();
|
||||
json["inputs"] = std::move(j);
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
nlohmann::json LockFile::toJson() const
|
||||
{
|
||||
auto json = FlakeInputs::toJson();
|
||||
json["version"] = 2;
|
||||
return json;
|
||||
}
|
||||
|
||||
LockFile LockFile::read(const Path & path)
|
||||
{
|
||||
if (pathExists(path)) {
|
||||
auto json = nlohmann::json::parse(readFile(path));
|
||||
|
||||
auto version = json.value("version", 0);
|
||||
if (version != 2)
|
||||
throw Error("lock file '%s' has unsupported version %d", path, version);
|
||||
|
||||
return LockFile(json);
|
||||
} else
|
||||
return LockFile();
|
||||
}
|
||||
|
||||
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
|
||||
{
|
||||
stream << lockFile.toJson().dump(4); // '4' = indentation in json file
|
||||
return stream;
|
||||
}
|
||||
|
||||
void LockFile::write(const Path & path) const
|
||||
{
|
||||
createDirs(dirOf(path));
|
||||
writeFile(path, fmt("%s\n", *this));
|
||||
}
|
||||
|
||||
}
|
||||
112
src/libexpr/flake/lockfile.hh
Normal file
112
src/libexpr/flake/lockfile.hh
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
#pragma once
|
||||
|
||||
#include "flakeref.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix {
|
||||
class Store;
|
||||
}
|
||||
|
||||
namespace nix::flake {
|
||||
|
||||
/* Common lock file information about a flake input, namely the
|
||||
immutable ref and the NAR hash. */
|
||||
struct AbstractInput
|
||||
{
|
||||
FlakeRef ref;
|
||||
Hash narHash;
|
||||
|
||||
AbstractInput(const FlakeRef & flakeRef, const Hash & narHash)
|
||||
: ref(flakeRef), narHash(narHash)
|
||||
{
|
||||
assert(ref.isImmutable());
|
||||
};
|
||||
|
||||
AbstractInput(const nlohmann::json & json);
|
||||
|
||||
nlohmann::json toJson() const;
|
||||
|
||||
Path computeStorePath(Store & store) const;
|
||||
};
|
||||
|
||||
/* Lock file information about a non-flake input. */
|
||||
struct NonFlakeInput : AbstractInput
|
||||
{
|
||||
using AbstractInput::AbstractInput;
|
||||
|
||||
bool operator ==(const NonFlakeInput & other) const
|
||||
{
|
||||
return ref == other.ref && narHash == other.narHash;
|
||||
}
|
||||
};
|
||||
|
||||
struct FlakeInput;
|
||||
|
||||
/* Lock file information about the dependencies of a flake. */
|
||||
struct FlakeInputs
|
||||
{
|
||||
std::map<FlakeRef, FlakeInput> flakeInputs;
|
||||
std::map<FlakeAlias, NonFlakeInput> nonFlakeInputs;
|
||||
|
||||
FlakeInputs() {};
|
||||
FlakeInputs(const nlohmann::json & json);
|
||||
|
||||
nlohmann::json toJson() const;
|
||||
};
|
||||
|
||||
/* Lock file information about a flake input. */
|
||||
struct FlakeInput : FlakeInputs, AbstractInput
|
||||
{
|
||||
FlakeId id;
|
||||
|
||||
FlakeInput(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash)
|
||||
: AbstractInput(flakeRef, narHash), id(id) {};
|
||||
|
||||
FlakeInput(const nlohmann::json & json);
|
||||
|
||||
bool operator ==(const FlakeInput & other) const
|
||||
{
|
||||
return
|
||||
id == other.id
|
||||
&& ref == other.ref
|
||||
&& narHash == other.narHash
|
||||
&& flakeInputs == other.flakeInputs
|
||||
&& nonFlakeInputs == other.nonFlakeInputs;
|
||||
}
|
||||
|
||||
nlohmann::json toJson() const;
|
||||
};
|
||||
|
||||
/* An entire lock file. Note that this cannot be a FlakeInput for the
|
||||
top-level flake, because then the lock file would need to contain
|
||||
the hash of the top-level flake, but committing the lock file
|
||||
would invalidate that hash. */
|
||||
struct LockFile : FlakeInputs
|
||||
{
|
||||
bool operator ==(const LockFile & other) const
|
||||
{
|
||||
return
|
||||
flakeInputs == other.flakeInputs
|
||||
&& nonFlakeInputs == other.nonFlakeInputs;
|
||||
}
|
||||
|
||||
LockFile() {}
|
||||
LockFile(const nlohmann::json & json) : FlakeInputs(json) {}
|
||||
LockFile(FlakeInput && dep)
|
||||
{
|
||||
flakeInputs = std::move(dep.flakeInputs);
|
||||
nonFlakeInputs = std::move(dep.nonFlakeInputs);
|
||||
}
|
||||
|
||||
nlohmann::json toJson() const;
|
||||
|
||||
static LockFile read(const Path & path);
|
||||
|
||||
void write(const Path & path) const;
|
||||
};
|
||||
|
||||
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr
|
|||
|
||||
libexpr_DIR := $(d)
|
||||
|
||||
libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
|
||||
libexpr_SOURCES := \
|
||||
$(wildcard $(d)/*.cc) \
|
||||
$(wildcard $(d)/primops/*.cc) \
|
||||
$(wildcard $(d)/flake/*.cc) \
|
||||
$(d)/lexer-tab.cc \
|
||||
$(d)/parser-tab.cc
|
||||
|
||||
libexpr_LIBS = libutil libstore
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
#include "progress-bar.hh"
|
||||
#include "eval.hh"
|
||||
#include "eval-inline.hh"
|
||||
#include "primops/flake.hh"
|
||||
#include "flake/flake.hh"
|
||||
#include "get-drvs.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
|
|
@ -122,6 +122,7 @@ static nlohmann::json flakeToJson(const Flake & flake)
|
|||
return j;
|
||||
}
|
||||
|
||||
#if 0
|
||||
static void printNonFlakeInfo(const NonFlake & nonFlake)
|
||||
{
|
||||
std::cout << fmt("ID: %s\n", nonFlake.alias);
|
||||
|
|
@ -173,6 +174,7 @@ struct CmdFlakeDeps : FlakeCommand
|
|||
}
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
struct CmdFlakeUpdate : FlakeCommand
|
||||
{
|
||||
|
|
@ -232,6 +234,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON
|
|||
if (json) {
|
||||
auto json = flakeToJson(flake);
|
||||
|
||||
#if 0
|
||||
auto state = getEvalState();
|
||||
|
||||
auto vFlake = state->allocValue();
|
||||
|
|
@ -254,6 +257,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON
|
|||
});
|
||||
|
||||
json["outputs"] = std::move(outputs);
|
||||
#endif
|
||||
|
||||
std::cout << json.dump() << std::endl;
|
||||
} else
|
||||
|
|
@ -518,7 +522,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command
|
|||
, make_ref<CmdFlakeUpdate>()
|
||||
, make_ref<CmdFlakeInfo>()
|
||||
, make_ref<CmdFlakeCheck>()
|
||||
, make_ref<CmdFlakeDeps>()
|
||||
//, make_ref<CmdFlakeDeps>()
|
||||
, make_ref<CmdFlakeAdd>()
|
||||
, make_ref<CmdFlakeRemove>()
|
||||
, make_ref<CmdFlakePin>()
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
#include "get-drvs.hh"
|
||||
#include "store-api.hh"
|
||||
#include "shared.hh"
|
||||
#include "primops/flake.hh"
|
||||
#include "flake/flake.hh"
|
||||
|
||||
#include <regex>
|
||||
#include <queue>
|
||||
|
|
@ -195,17 +195,29 @@ void makeFlakeClosureGCRoot(Store & store,
|
|||
/* Get the store paths of all non-local flakes. */
|
||||
PathSet closure;
|
||||
|
||||
std::queue<std::reference_wrapper<const flake::ResolvedFlake>> queue;
|
||||
queue.push(resFlake);
|
||||
assert(store.isValidPath(resFlake.flake.sourceInfo.storePath));
|
||||
closure.insert(resFlake.flake.sourceInfo.storePath);
|
||||
|
||||
std::queue<std::reference_wrapper<const flake::FlakeInputs>> queue;
|
||||
queue.push(resFlake.lockFile);
|
||||
|
||||
while (!queue.empty()) {
|
||||
const flake::ResolvedFlake & flake = queue.front();
|
||||
const flake::FlakeInputs & flake = queue.front();
|
||||
queue.pop();
|
||||
if (!std::get_if<FlakeRef::IsPath>(&flake.flake.sourceInfo.resolvedRef.data))
|
||||
closure.insert(flake.flake.sourceInfo.storePath);
|
||||
for (const auto & dep : flake.flakeDeps)
|
||||
/* Note: due to lazy fetching, these paths might not exist
|
||||
yet. */
|
||||
for (auto & dep : flake.flakeInputs) {
|
||||
auto path = dep.second.computeStorePath(store);
|
||||
if (store.isValidPath(path))
|
||||
closure.insert(path);
|
||||
queue.push(dep.second);
|
||||
}
|
||||
for (auto & dep : flake.nonFlakeInputs) {
|
||||
auto path = dep.second.computeStorePath(store);
|
||||
if (store.isValidPath(path))
|
||||
closure.insert(path);
|
||||
}
|
||||
}
|
||||
|
||||
if (closure.empty()) return;
|
||||
|
||||
|
|
|
|||
10
src/nlohmann/json_fwd.hpp
Normal file
10
src/nlohmann/json_fwd.hpp
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
#pragma once
|
||||
|
||||
namespace nlohmann {
|
||||
|
||||
struct json : basic_json<>
|
||||
{
|
||||
using basic_json<>::basic_json;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -55,7 +55,7 @@ cat > $flake2Dir/flake.nix <<EOF
|
|||
description = "Fnord";
|
||||
|
||||
outputs = inputs: rec {
|
||||
packages.bar = inputs.flake1.outputs.packages.foo;
|
||||
packages.bar = inputs.flake1.packages.foo;
|
||||
};
|
||||
}
|
||||
EOF
|
||||
|
|
@ -74,7 +74,7 @@ cat > $flake3Dir/flake.nix <<EOF
|
|||
description = "Fnord";
|
||||
|
||||
outputs = inputs: rec {
|
||||
packages.xyzzy = inputs.flake2.outputs.packages.bar;
|
||||
packages.xyzzy = inputs.flake2.packages.bar;
|
||||
};
|
||||
}
|
||||
EOF
|
||||
|
|
@ -83,7 +83,7 @@ git -C $flake3Dir add flake.nix
|
|||
git -C $flake3Dir commit -m 'Initial'
|
||||
|
||||
cat > $nonFlakeDir/README.md <<EOF
|
||||
Not much
|
||||
FNORD
|
||||
EOF
|
||||
|
||||
git -C $nonFlakeDir add README.md
|
||||
|
|
@ -176,8 +176,8 @@ cat > $flake3Dir/flake.nix <<EOF
|
|||
description = "Fnord";
|
||||
|
||||
outputs = inputs: rec {
|
||||
packages.xyzzy = inputs.flake2.outputs.packages.bar;
|
||||
packages.sth = inputs.flake1.outputs.packages.foo;
|
||||
packages.xyzzy = inputs.flake2.packages.bar;
|
||||
packages.sth = inputs.flake1.packages.foo;
|
||||
};
|
||||
}
|
||||
EOF
|
||||
|
|
@ -191,6 +191,8 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth
|
|||
# Check whether it saved the lockfile
|
||||
[[ ! (-z $(git -C $flake3Dir diff master)) ]]
|
||||
|
||||
git -C $flake3Dir commit -m 'Add lockfile'
|
||||
|
||||
# Unsupported epochs should be an error.
|
||||
sed -i $flake3Dir/flake.nix -e s/201906/201909/
|
||||
nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | grep 'unsupported epoch'
|
||||
|
|
@ -202,6 +204,7 @@ nix flake list --flake-registry file://$registry --tarball-ttl 0 | grep -q flake
|
|||
mv $registry.tmp $registry
|
||||
|
||||
# Test whether flakes are registered as GC roots for offline use.
|
||||
# FIXME: use tarballs rather than git.
|
||||
rm -rf $TEST_HOME/.cache
|
||||
nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar
|
||||
mv $flake1Dir $flake1Dir.tmp
|
||||
|
|
@ -230,14 +233,44 @@ cat > $flake3Dir/flake.nix <<EOF
|
|||
description = "Fnord";
|
||||
|
||||
outputs = inputs: rec {
|
||||
packages.xyzzy = inputs.flake2.outputs.packages.bar;
|
||||
packages.sth = inputs.flake1.outputs.packages.foo;
|
||||
packages.xyzzy = inputs.flake2.packages.bar;
|
||||
packages.sth = inputs.flake1.packages.foo;
|
||||
packages.fnord =
|
||||
with import ./config.nix;
|
||||
mkDerivation {
|
||||
inherit system;
|
||||
name = "fnord";
|
||||
buildCommand = ''
|
||||
cat \${inputs.nonFlake}/README.md > \$out
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
EOF
|
||||
|
||||
git -C $flake3Dir add flake.nix
|
||||
cp ./config.nix $flake3Dir
|
||||
|
||||
git -C $flake3Dir add flake.nix config.nix
|
||||
git -C $flake3Dir commit -m 'Add nonFlakeInputs'
|
||||
|
||||
# Check whether `nix build` works with a lockfile which is missing a nonFlakeInputs
|
||||
# Check whether `nix build` works with a lockfile which is missing a
|
||||
# nonFlakeInputs.
|
||||
nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth
|
||||
|
||||
git -C $flake3Dir commit -m 'Update nonFlakeInputs'
|
||||
|
||||
nix build -o $TEST_ROOT/result --flake-registry $registry flake3:fnord
|
||||
[[ $(cat $TEST_ROOT/result) = FNORD ]]
|
||||
|
||||
# Check whether flake input fetching is lazy: flake3:sth does not
|
||||
# depend on flake2, so this shouldn't fail.
|
||||
rm -rf $TEST_HOME/.cache
|
||||
clearStore
|
||||
mv $flake2Dir $flake2Dir.tmp
|
||||
mv $nonFlakeDir $nonFlakeDir.tmp
|
||||
nix build -o $TEST_ROOT/result --flake-registry $registry flake3:sth
|
||||
(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy)
|
||||
(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:fnord)
|
||||
mv $flake2Dir.tmp $flake2Dir
|
||||
mv $nonFlakeDir.tmp $nonFlakeDir
|
||||
nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy flake3:fnord
|
||||
|
|
|
|||
Loading…
Reference in a new issue