Merge branch 'fixLockFile' of https://github.com/CSVdB/nix into flakes

This commit is contained in:
Eelco Dolstra 2019-05-21 13:46:19 +02:00
commit 818c8da5b8
6 changed files with 231 additions and 123 deletions

View file

@ -50,8 +50,7 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json)
if (!flakeRef.isImmutable())
throw Error("cannot use mutable flake '%s' in pure mode", flakeRef);
Hash hash = Hash((std::string) json["contentHash"]);
LockFile::FlakeEntry entry(flakeRef, hash);
LockFile::FlakeEntry entry(flakeRef, Hash((std::string) json["contentHash"]));
auto nonFlakeRequires = json["nonFlakeRequires"];
@ -59,9 +58,8 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json)
FlakeRef flakeRef(i->value("uri", ""));
if (!flakeRef.isImmutable())
throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef);
Hash hash = Hash((std::string) i->value("contentHash", ""));
LockFile::NonFlakeEntry newEntry(flakeRef, hash);
entry.nonFlakeEntries.insert_or_assign(i.key(), newEntry);
LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("contentHash", "")));
entry.nonFlakeEntries.insert_or_assign(i.key(), nonEntry);
}
auto requires = json["requires"];
@ -89,10 +87,10 @@ LockFile readLockFile(const Path & path)
for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) {
FlakeRef flakeRef(i->value("uri", ""));
LockFile::NonFlakeEntry entry(flakeRef, Hash((std::string) json["contentHash"]));
LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("contentHash", "")));
if (!flakeRef.isImmutable())
throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef);
lockFile.nonFlakeEntries.insert_or_assign(i.key(), entry);
throw Error("found mutable FlakeRef '%s' in lockfile at path %s", flakeRef, path);
lockFile.nonFlakeEntries.insert_or_assign(i.key(), nonEntry);
}
auto requires = json["requires"];
@ -289,8 +287,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe
FlakeRef resolvedRef = sourceInfo.resolvedRef;
resolvedRef = sourceInfo.resolvedRef; // `resolvedRef` is now immutable
state.store->assertStorePath(sourceInfo.storePath);
if (state.allowedPaths)
@ -368,73 +364,135 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al
return nonFlake;
}
/* Given a flake reference, recursively fetch it and its
dependencies.
FIXME: this should return a graph of flakes.
*/
ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef,
RegistryAccess registryAccess, bool isTopFlake)
LockFile entryToLockFile(const LockFile::FlakeEntry & entry)
{
bool allowRegistries = registryAccess == AllowRegistry || (registryAccess == AllowRegistryAtTop && isTopFlake);
Flake flake = getFlake(state, topRef, allowRegistries);
LockFile lockFile;
if (isTopFlake)
lockFile = readLockFile(flake.storePath + flake.resolvedRef.subdir + "/flake.lock"); // FIXME: symlink attack
ResolvedFlake deps(flake);
for (auto & nonFlakeInfo : flake.nonFlakeRequires)
deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first));
for (auto newFlakeRef : flake.requires) {
auto i = lockFile.flakeEntries.find(newFlakeRef);
if (i != lockFile.flakeEntries.end()) newFlakeRef = i->second.ref;
// FIXME: propagate lockFile downwards
deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, registryAccess, false));
}
return deps;
}
LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake)
{
LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.hash);
for (auto & newResFlake : resolvedFlake.flakeDeps)
entry.flakeEntries.insert_or_assign(newResFlake.flake.originalRef, dependenciesToFlakeEntry(newResFlake));
for (auto & nonFlake : resolvedFlake.nonFlakeDeps)
entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, LockFile::NonFlakeEntry(nonFlake.resolvedRef, nonFlake.hash));
return entry;
}
static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef)
{
ResolvedFlake resFlake = resolveFlake(evalState, flakeRef, AllowRegistry);
LockFile::FlakeEntry entry = dependenciesToFlakeEntry(resFlake);
LockFile lockFile;
lockFile.flakeEntries = entry.flakeEntries;
lockFile.nonFlakeEntries = entry.nonFlakeEntries;
return lockFile;
}
void updateLockFile(EvalState & state, const FlakeUri & flakeUri)
LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake)
{
// FIXME: We are writing the lockfile to the store here! Very bad practice!
FlakeRef flakeRef = FlakeRef(flakeUri);
if (auto refData = std::get_if<FlakeRef::IsPath>(&flakeRef.data)) {
auto lockFile = makeLockFile(state, flakeRef);
writeLockFile(lockFile, refData->path + "/" + flakeRef.subdir + "/flake.lock");
LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.hash);
// Hack: Make sure that flake.lock is visible to Git. Otherwise,
// exportGit will fail to copy it to the Nix store.
runProgram("git", true,
{ "-C", refData->path, "add", "--intent-to-add",
(flakeRef.subdir == "" ? "" : flakeRef.subdir + "/") + "flake.lock" });
} else
throw Error("flakeUri %s can't be updated because it is not a path", flakeUri);
for (auto & info : resolvedFlake.flakeDeps)
entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second));
for (auto & nonFlake : resolvedFlake.nonFlakeDeps) {
LockFile::NonFlakeEntry nonEntry(nonFlake.resolvedRef, nonFlake.hash);
entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry);
}
return entry;
}
bool allowedToWrite (HandleLockFile handle)
{
if (handle == AllPure) return false;
else if (handle == TopRefUsesRegistries) return false;
else if (handle == UpdateLockFile) return true;
else if (handle == UseUpdatedLockFile) return false;
else if (handle == RecreateLockFile) return true;
else if (handle == UseNewLockFile) return false;
else assert(false);
}
bool recreateLockFile (HandleLockFile handle)
{
if (handle == AllPure) return false;
else if (handle == TopRefUsesRegistries) return false;
else if (handle == UpdateLockFile) return false;
else if (handle == UseUpdatedLockFile) return false;
else if (handle == RecreateLockFile) return true;
else if (handle == UseNewLockFile) return true;
else assert(false);
}
bool allowedToUseRegistries (HandleLockFile handle, bool isTopRef)
{
if (handle == AllPure) return false;
else if (handle == TopRefUsesRegistries) return isTopRef;
else if (handle == UpdateLockFile) return true;
else if (handle == UseUpdatedLockFile) return true;
else if (handle == RecreateLockFile) return true;
else if (handle == UseNewLockFile) return true;
else assert(false);
}
ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef,
HandleLockFile handleLockFile, LockFile lockFile = {}, bool topRef = false)
{
Flake flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef));
ResolvedFlake deps(flake);
for (auto & nonFlakeInfo : flake.nonFlakeRequires) {
FlakeRef ref = nonFlakeInfo.second;
auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first);
if (i != lockFile.nonFlakeEntries.end()) {
NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first);
if (nonFlake.hash != i->second.contentHash)
throw Error("the content hash of flakeref %s doesn't match", i->second.ref.to_string());
deps.nonFlakeDeps.push_back(nonFlake);
} else {
if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries)
throw Error("the lockfile requires updating nonflake dependency %s in AllPure mode", nonFlakeInfo.first);
deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first));
}
}
for (auto newFlakeRef : flake.requires) {
auto i = lockFile.flakeEntries.find(newFlakeRef);
if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible
ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second));
if (newResFlake.flake.hash != i->second.contentHash)
throw Error("the content hash of flakeref %s doesn't match", i->second.ref.to_string());
deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake);
} else {
if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries)
throw Error("the lockfile requires updating flake dependency %s in AllPure mode", newFlakeRef.to_string());
deps.flakeDeps.insert_or_assign(newFlakeRef, resolveFlakeFromLockFile(state, newFlakeRef, handleLockFile));
}
}
return deps;
}
/* Given a flake reference, recursively fetch it and its dependencies.
FIXME: this should return a graph of flakes.
*/
ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile)
{
Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true));
LockFile lockFile;
if (!recreateLockFile (handleLockFile)) {
// If recreateLockFile, start with an empty lockfile
lockFile = readLockFile(flake.storePath + "/flake.lock"); // FIXME: symlink attack
}
ResolvedFlake resFlake = resolveFlakeFromLockFile(state, topRef, handleLockFile, lockFile, true);
lockFile = entryToLockFile(dependenciesToFlakeEntry(resFlake));
if (allowedToWrite(handleLockFile)) {
if (auto refData = std::get_if<FlakeRef::IsPath>(&topRef.data)) {
writeLockFile(lockFile, refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock");
// Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store.
runProgram("git", true, { "-C", refData->path, "add",
(topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" });
} else std::cout << "Cannot write lockfile because the FlakeRef isn't of the form IsPath." << std::endl;
} else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries)
std::cout << "Using updating lockfile without writing it to file" << std::endl;
return resFlake;
}
void updateLockFile (EvalState & state, const FlakeUri & flakeUri, bool recreateLockFile)
{
FlakeRef flakeRef(flakeUri);
resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile);
}
void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)
@ -444,7 +502,8 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)
state.mkAttrs(v, resFlake.flakeDeps.size() + resFlake.nonFlakeDeps.size() + 8);
for (const ResolvedFlake newResFlake : resFlake.flakeDeps) {
for (auto info : resFlake.flakeDeps) {
const ResolvedFlake newResFlake = info.second;
auto vFlake = state.allocAttr(v, newResFlake.flake.id);
callFlake(state, newResFlake, *vFlake);
}
@ -485,16 +544,16 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)
// Return the `provides` of the top flake, while assigning to `v` the provides
// of the dependencies as well.
void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v)
void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, HandleLockFile handle, Value & v)
{
callFlake(state, resolveFlake(state, flakeRef, registryAccess), v);
callFlake(state, resolveFlake(state, flakeRef, handle), v);
}
// This function is exposed to be used in nix files.
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos),
evalSettings.pureEval ? DisallowRegistry : AllowRegistryAtTop, v);
evalSettings.pureEval ? AllPure : UseUpdatedLockFile, v);
}
static RegisterPrimOp r2("getFlake", 1, prim_getFlake);

View file

@ -36,16 +36,23 @@ struct LockFile
};
std::map<FlakeRef, FlakeEntry> flakeEntries;
std::map<FlakeId, NonFlakeEntry> nonFlakeEntries;
std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries;
};
typedef std::vector<std::shared_ptr<FlakeRegistry>> Registries;
Path getUserRegistryPath();
enum RegistryAccess { DisallowRegistry, AllowRegistry, AllowRegistryAtTop };
enum HandleLockFile
{ AllPure // Everything is handled 100% purely
, TopRefUsesRegistries // The top FlakeRef uses the registries, apart from that, everything happens 100% purely
, UpdateLockFile // Update the existing lockfile and write it to file
, UseUpdatedLockFile // `UpdateLockFile` without writing to file
, RecreateLockFile // Recreate the lockfile from scratch and write it to file
, UseNewLockFile // `RecreateLockFile` without writing to file
};
void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v);
void makeFlakeValue(EvalState &, const FlakeRef &, HandleLockFile, Value &);
std::shared_ptr<FlakeRegistry> readRegistry(const Path &);
@ -84,8 +91,8 @@ struct NonFlake
FlakeRef originalRef;
FlakeRef resolvedRef;
std::optional<uint64_t> revCount;
Hash hash;
Path storePath;
Hash hash; // content hash
// date
NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef),
resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {};
@ -98,14 +105,14 @@ Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed);
struct ResolvedFlake
{
Flake flake;
std::vector<ResolvedFlake> flakeDeps; // The flake dependencies
std::map<FlakeRef, ResolvedFlake> flakeDeps; // The key in this map, is the originalRef as written in flake.nix
std::vector<NonFlake> nonFlakeDeps;
ResolvedFlake(const Flake & flake) : flake(flake) {}
};
ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registryAccess, bool isTopFlake = true);
ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile);
void updateLockFile(EvalState &, const FlakeUri &);
void updateLockFile(EvalState &, const FlakeUri &, bool recreateLockFile);
void gitCloneFlake (std::string flakeUri, EvalState &, Registries, Path);
}

View file

@ -76,10 +76,14 @@ struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs
{
std::optional<Path> file;
bool updateLockFile = true;
SourceExprCommand();
bool recreateLockFile = false;
bool saveLockFile = true;
bool noRegistries = false;
ref<EvalState> getEvalState();
std::vector<std::shared_ptr<Installable>> parseInstallables(

View file

@ -114,7 +114,7 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs
FlakeRef flakeRef(flakeUri);
ResolvedFlake resFlake = resolveFlake(*evalState, flakeRef, AllowRegistryAtTop);
ResolvedFlake resFlake = resolveFlake(*evalState, flakeRef, UpdateLockFile);
std::queue<ResolvedFlake> todo;
todo.push(resFlake);
@ -126,13 +126,13 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs
for (NonFlake & nonFlake : resFlake.nonFlakeDeps)
printNonFlakeInfo(nonFlake, json);
for (ResolvedFlake & newResFlake : resFlake.flakeDeps)
todo.push(newResFlake);
for (auto info : resFlake.flakeDeps)
todo.push(info.second);
}
}
};
struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs
struct CmdFlakeUpdate : StoreCommand, FlakeCommand, MixEvalArgs
{
std::string name() override
{
@ -148,8 +148,8 @@ struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs
{
auto evalState = std::make_shared<EvalState>(searchPath, store);
if (gitPath == "") gitPath = absPath(".");
updateLockFile(*evalState, gitPath);
bool recreateLockFile = true;
updateLockFile(*evalState, flakeUri, recreateLockFile);
}
};

View file

@ -23,9 +23,19 @@ SourceExprCommand::SourceExprCommand()
.dest(&file);
mkFlag()
.longName("no-update")
.description("don't create/update flake lock files")
.set(&updateLockFile, false);
.longName("recreate-lock-file")
.description("recreate lock file from scratch")
.set(&recreateLockFile, true);
mkFlag()
.longName("dont-save-lock-file")
.description("save the newly generated lock file")
.set(&saveLockFile, false);
mkFlag()
.longName("no-registries")
.description("don't use flake registries")
.set(&noRegistries, true);
}
ref<EvalState> SourceExprCommand::getEvalState()
@ -157,13 +167,13 @@ struct InstallableFlake : InstallableValue
Value * toValue(EvalState & state) override
{
auto path = std::get_if<FlakeRef::IsPath>(&flakeRef.data);
if (cmd.updateLockFile && path) {
updateLockFile(state, path->path);
}
auto vFlake = state.allocValue();
makeFlakeValue(state, flakeRef, AllowRegistryAtTop, *vFlake);
HandleLockFile handle = cmd.noRegistries ? AllPure :
cmd.recreateLockFile ?
(cmd.saveLockFile ? RecreateLockFile : UseNewLockFile)
: (cmd.saveLockFile ? UpdateLockFile : UseUpdatedLockFile);
makeFlakeValue(state, flakeRef, handle, *vFlake);
auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value;

View file

@ -9,11 +9,11 @@ clearStore
registry=$TEST_ROOT/registry.json
flake1=$TEST_ROOT/flake1
flake2=$TEST_ROOT/flake2
flake3=$TEST_ROOT/flake3
flake1Dir=$TEST_ROOT/flake1
flake2Dir=$TEST_ROOT/flake2
flake3Dir=$TEST_ROOT/flake3
for repo in $flake1 $flake2 $flake3; do
for repo in $flake1Dir $flake2Dir $flake3Dir; do
rm -rf $repo
mkdir $repo
git -C $repo init
@ -21,7 +21,7 @@ for repo in $flake1 $flake2 $flake3; do
git -C $repo config user.name "Foobar"
done
cat > $flake1/flake.nix <<EOF
cat > $flake1Dir/flake.nix <<EOF
{
name = "flake1";
@ -36,11 +36,11 @@ cat > $flake1/flake.nix <<EOF
}
EOF
cp ./simple.nix ./simple.builder.sh ./config.nix $flake1/
git -C $flake1 add flake.nix simple.nix simple.builder.sh config.nix
git -C $flake1 commit -m 'Initial'
cp ./simple.nix ./simple.builder.sh ./config.nix $flake1Dir/
git -C $flake1Dir add flake.nix simple.nix simple.builder.sh config.nix
git -C $flake1Dir commit -m 'Initial'
cat > $flake2/flake.nix <<EOF
cat > $flake2Dir/flake.nix <<EOF
{
name = "flake2";
@ -56,10 +56,10 @@ cat > $flake2/flake.nix <<EOF
}
EOF
git -C $flake2 add flake.nix
git -C $flake2 commit -m 'Initial'
git -C $flake2Dir add flake.nix
git -C $flake2Dir commit -m 'Initial'
cat > $flake3/flake.nix <<EOF
cat > $flake3Dir/flake.nix <<EOF
{
name = "flake3";
@ -75,20 +75,20 @@ cat > $flake3/flake.nix <<EOF
}
EOF
git -C $flake3 add flake.nix
git -C $flake3 commit -m 'Initial'
git -C $flake3Dir add flake.nix
git -C $flake3Dir commit -m 'Initial'
cat > $registry <<EOF
{
"flakes": {
"flake1": {
"uri": "file://$flake1"
"uri": "file://$flake1Dir"
},
"flake2": {
"uri": "file://$flake2"
"uri": "file://$flake2Dir"
},
"flake3": {
"uri": "file://$flake3"
"uri": "file://$flake3Dir"
},
"nixpkgs": {
"uri": "flake1"
@ -118,26 +118,54 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake1:
[[ -e $TEST_ROOT/result/hello ]]
# Building a flake with an unlocked dependency should fail in pure mode.
(! nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar)
(! nix eval "(builtins.getFlake "$flake2Dir")")
# But should succeed in impure mode.
# FIXME: this currently fails.
#nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar --impure
nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar --impure
# Test automatic lock file generation.
nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar
[[ -e $flake2/flake.lock ]]
git -C $flake2 commit flake.lock -m 'Add flake.lock'
nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir:bar
[[ -e $flake2Dir/flake.lock ]]
git -C $flake2Dir commit flake.lock -m 'Add flake.lock'
# Rerunning the build should not change the lockfile.
#nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar
#[[ -z $(git -C $flake2 diff) ]]
nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir:bar
[[ -z $(git -C $flake2Dir diff master) ]]
# Now we should be able to build the flake in pure mode.
nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar
# Or without a registry.
nix build -o $TEST_ROOT/result file://$flake2:bar
nix build -o $TEST_ROOT/result file://$flake2Dir:bar
# Test whether indirect dependencies work.
#nix build -o $TEST_ROOT/result --flake-registry $registry $flake3:xyzzy
nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:xyzzy
# Add dependency to flake3
rm $flake3Dir/flake.nix
cat > $flake3Dir/flake.nix <<EOF
{
name = "flake3";
epoch = 2019;
requires = [ "flake1" "flake2" ];
description = "Fnord";
provides = deps: rec {
packages.xyzzy = deps.flake2.provides.packages.bar;
packages.sth = deps.flake1.provides.packages.foo;
};
}
EOF
git -C $flake3Dir add flake.nix
git -C $flake3Dir commit -m 'Update flake.nix'
# Check whether `nix build` works with an incomplete lockfile
nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth
# Check whether it saved the lockfile
[[ ! (-z $(git -C $flake3Dir diff master)) ]]