Added nonFlakeRequires and the command nix flake deps

This commit is contained in:
Nick Van den Broeck 2019-03-21 09:30:16 +01:00
parent 87033f2c4e
commit 18c019b616
4 changed files with 200 additions and 93 deletions

View file

@ -18,20 +18,19 @@ std::shared_ptr<FlakeRegistry> readRegistry(const Path & path)
{ {
auto registry = std::make_shared<FlakeRegistry>(); auto registry = std::make_shared<FlakeRegistry>();
try { if (!pathExists(path))
auto json = nlohmann::json::parse(readFile(path)); return std::make_shared<FlakeRegistry>();
auto version = json.value("version", 0); auto json = nlohmann::json::parse(readFile(path));
if (version != 1)
throw Error("flake registry '%s' has unsupported version %d", path, version);
auto flakes = json["flakes"]; auto version = json.value("version", 0);
for (auto i = flakes.begin(); i != flakes.end(); ++i) { if (version != 1)
FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; throw Error("flake registry '%s' has unsupported version %d", path, version);
registry->entries.emplace(i.key(), entry);
} auto flakes = json["flakes"];
} catch (SysError & e) { for (auto i = flakes.begin(); i != flakes.end(); ++i) {
if (e.errNo != ENOENT) throw; FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))};
registry->entries.emplace(i.key(), entry);
} }
return registry; return registry;
@ -54,7 +53,6 @@ Path getUserRegistryPath()
{ {
return getHome() + "/.config/nix/registry.json"; return getHome() + "/.config/nix/registry.json";
} }
std::shared_ptr<FlakeRegistry> getGlobalRegistry() std::shared_ptr<FlakeRegistry> getGlobalRegistry()
{ {
// FIXME: get from nixos.org. // FIXME: get from nixos.org.
@ -76,12 +74,20 @@ std::shared_ptr<FlakeRegistry> getFlagRegistry()
const std::vector<std::shared_ptr<FlakeRegistry>> EvalState::getFlakeRegistries() const std::vector<std::shared_ptr<FlakeRegistry>> EvalState::getFlakeRegistries()
{ {
std::vector<std::shared_ptr<FlakeRegistry>> registries; std::vector<std::shared_ptr<FlakeRegistry>> registries;
registries.push_back(getGlobalRegistry()); if (evalSettings.pureEval) {
registries.push_back(getUserRegistry()); registries.push_back(std::make_shared<FlakeRegistry>()); // global
registries.push_back(std::make_shared<FlakeRegistry>()); // user
registries.push_back(std::make_shared<FlakeRegistry>()); // local
} else {
registries.push_back(getGlobalRegistry());
registries.push_back(getUserRegistry());
registries.push_back(getLocalRegistry());
}
registries.push_back(getFlagRegistry()); registries.push_back(getFlagRegistry());
return registries; return registries;
} }
// Creates a Nix attribute set value listing all dependencies, so they can be used in `provides`.
Value * makeFlakeRegistryValue(EvalState & state) Value * makeFlakeRegistryValue(EvalState & state)
{ {
auto v = state.allocValue(); auto v = state.allocValue();
@ -199,7 +205,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef)
Flake getFlake(EvalState & state, const FlakeRef & flakeRef) Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
{ {
auto sourceInfo = fetchFlake(state, flakeRef); FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef);
debug("got flake source '%s' with revision %s", debug("got flake source '%s' with revision %s",
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
@ -209,18 +215,16 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
if (state.allowedPaths) if (state.allowedPaths)
state.allowedPaths->insert(flakePath); state.allowedPaths->insert(flakePath);
FlakeRef newFlakeRef(flakeRef); Flake flake(flakeRef);
if (std::get_if<FlakeRef::IsGitHub>(&newFlakeRef.data)) { if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
FlakeSourceInfo srcInfo = fetchFlake(state, newFlakeRef); if (sourceInfo.rev)
if (srcInfo.rev) { flake.ref = FlakeRef(flakeRef.baseRef().to_string()
std::string uri = flakeRef.baseRef().to_string(); + "/" + sourceInfo.rev->to_string(Base16, false));
newFlakeRef = FlakeRef(uri + "/" + srcInfo.rev->to_string(Base16, false));
}
} }
Flake flake(newFlakeRef);
flake.path = flakePath; flake.path = flakePath;
flake.revCount = sourceInfo.revCount; flake.revCount = sourceInfo.revCount;
flake.path = flakePath;
Value vInfo; Value vInfo;
state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack
@ -242,6 +246,15 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
*(**requires).value->listElems()[n], *(**requires).pos))); *(**requires).value->listElems()[n], *(**requires).pos)));
} }
if (std::optional<Attr *> nonFlakeRequires = vInfo.attrs->get(state.symbols.create("nonFlakeRequires"))) {
state.forceAttrs(*(**nonFlakeRequires).value, *(**nonFlakeRequires).pos);
for (Attr attr : *(*(**nonFlakeRequires).value).attrs) {
std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos);
FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri);
flake.nonFlakeRequires.insert_or_assign(attr.name, nonFlakeRef);
}
}
if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) { if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) {
state.forceFunction(*(**provides).value, *(**provides).pos); state.forceFunction(*(**provides).value, *(**provides).pos);
flake.vProvides = (**provides).value; flake.vProvides = (**provides).value;
@ -250,86 +263,107 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack
if (pathExists(lockFile)) { flake.lockFile = readRegistry(lockFile);
flake.lockFile = readRegistry(lockFile); for (auto & entry : flake.lockFile->entries)
for (auto & entry : flake.lockFile->entries) if (!entry.second.ref.isImmutable())
if (!entry.second.ref.isImmutable()) throw Error("flake lock file '%s' contains mutable entry '%s'",
throw Error("flake lock file '%s' contains mutable entry '%s'", lockFile, entry.second.ref.to_string());
lockFile, entry.second.ref.to_string());
}
return flake; return flake;
} }
// Get the `NonFlake` corresponding to a `FlakeRef`.
NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flakeId)
{
FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef);
debug("got non-flake source '%s' with revision %s",
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
auto flakePath = sourceInfo.storePath;
state.store->assertStorePath(flakePath);
if (state.allowedPaths)
state.allowedPaths->insert(flakePath);
NonFlake nonFlake(flakeRef);
if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
if (sourceInfo.rev)
nonFlake.ref = FlakeRef(flakeRef.baseRef().to_string()
+ "/" + sourceInfo.rev->to_string(Base16, false));
}
nonFlake.path = flakePath;
nonFlake.id = flakeId;
return nonFlake;
}
/* Given a flake reference, recursively fetch it and its /* Given a flake reference, recursively fetch it and its
dependencies. dependencies.
FIXME: this should return a graph of flakes. FIXME: this should return a graph of flakes.
*/ */
static std::tuple<FlakeId, std::map<FlakeId, Flake>> resolveFlake(EvalState & state, Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef)
const FlakeRef & topRef, bool impureTopRef)
{ {
std::map<FlakeId, Flake> done; Dependencies deps;
std::queue<std::tuple<FlakeRef, bool>> todo; std::queue<FlakeRef> todo;
std::optional<FlakeId> topFlakeId; /// FIXME: ambiguous bool isTopLevel = true;
todo.push({topRef, true}); todo.push(topRef);
auto registries = state.getFlakeRegistries(); auto registries = state.getFlakeRegistries();
//std::shared_ptr<FlakeRegistry> localRegistry = registries.at(2);
while (!todo.empty()) { while (!todo.empty()) {
auto [flakeRef, toplevel] = todo.front(); auto flakeRef = todo.front();
todo.pop(); todo.pop();
if (auto refData = std::get_if<FlakeRef::IsFlakeId>(&flakeRef.data)) { if (auto refData = std::get_if<FlakeRef::IsFlakeId>(&flakeRef.data)) {
if (done.count(refData->id)) continue; // optimization if (done.count(refData->id)) continue; // optimization
flakeRef = lookupFlake(state, flakeRef, flakeRef = lookupFlake(state, flakeRef, registries);
!evalSettings.pureEval || (toplevel && impureTopRef) ? registries : std::vector<std::shared_ptr<FlakeRegistry>>()); if (evalSettings.pureEval && !flakeRef.isImmutable() && (!isTopLevel || !impureTopRef))
// This is why we need the `registries`.
}
#if 0
if (evalSettings.pureEval && !flakeRef.isImmutable() && (!toplevel || !impureTopRef))
throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string()); throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string());
#endif
auto flake = getFlake(state, flakeRef); auto flake = getFlake(state, flakeRef);
if (done.count(flake.id)) continue; if (isTopLevel) {
deps.topFlakeId = flake.id;
isTopLevel = false;
}
if (toplevel) topFlakeId = flake.id; for (auto & flakeRef : flake.requires)
todo.push(flakeRef);
for (auto & require : flake.requires) for (auto & x : flake.nonFlakeRequires)
todo.push({require, false}); deps.nonFlakes.push_back(getNonFlake(state, x.second, x.first));
// TODO (Nick): If there are 2 non-flake dependencies with the same
// FlakeId, this will lead to trouble! One of the dependencies won't
// be used!
#if 0 deps.flakes.push_back(flake);
// The following piece of code basically adds the FlakeRefs from
// the lockfiles of dependencies to the localRegistry. This is used
// to resolve future `FlakeId`s, in `lookupFlake` a bit above this.
if (flake.lockFile)
for (auto & entry : flake.lockFile->entries) {
if (localRegistry->entries.count(entry.first)) continue;
localRegistry->entries.emplace(entry.first, entry.second);
}
#endif
done.emplace(flake.id, std::move(flake));
} }
assert(topFlakeId); return deps;
return {*topFlakeId, std::move(done)};
} }
FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef)
{ {
FlakeRegistry newLockFile; FlakeRegistry newLockFile;
std::map<FlakeId, Flake> myDependencyMap = get<1>(resolveFlake(evalState, flakeRef, false)); Dependencies deps = resolveFlake(evalState, flakeRef, false);
// Nick assumed that "topRefPure" means that the Flake for flakeRef can be // Nick assumed that "topRefPure" means that the Flake for flakeRef can be
// fetched purely. // fetched purely.
for (auto const& require : myDependencyMap) { for (auto const& require : deps.flakes) {
FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.second.ref); FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.ref);
// The FlakeRefs are immutable because they come out of the Flake objects, // The FlakeRefs are immutable because they come out of the Flake objects.
// not from the requires. if (require.id != deps.topFlakeId)
newLockFile.entries.insert(std::pair<FlakeId, FlakeRegistry::Entry>(require.first, entry)); newLockFile.entries.insert_or_assign(require.id, entry);
// TODO (Nick): If there are 2 flake dependencies with the same FlakeId,
// one of them gets ignored!
}
for (auto const& nonFlake : deps.nonFlakes) {
FlakeRegistry::Entry entry = FlakeRegistry::Entry(nonFlake.ref);
newLockFile.entries.insert_or_assign(nonFlake.id, entry);
// We are assuming the sets of FlakeIds for flakes and non-flakes
// are disjoint.
} }
return newLockFile; return newLockFile;
} }
@ -348,21 +382,26 @@ void updateLockFile(EvalState & state, std::string path)
} }
} }
Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v) // Return the `provides` of the top flake, while assigning to `v` the provides
// of the dependencies as well.
Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, bool impureTopRef, Value & v)
{ {
auto [topFlakeId, flakes] = resolveFlake(state, flakeRef, impureTopRef); FlakeRef flakeRef = FlakeRef(flakeUri);
Dependencies deps = resolveFlake(state, flakeRef, impure);
// FIXME: we should call each flake with only its dependencies // FIXME: we should call each flake with only its dependencies
// (rather than the closure of the top-level flake). // (rather than the closure of the top-level flake).
auto vResult = state.allocValue(); auto vResult = state.allocValue();
// This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`.
state.mkAttrs(*vResult, flakes.size()); state.mkAttrs(*vResult, dependencies.flakes.size());
Value * vTop = 0; Value * vTop = 0;
for (auto & flake : flakes) { for (auto & flake : deps.flakes) {
auto vFlake = state.allocAttr(*vResult, flake.second.id); auto vFlake = state.allocAttr(*vResult, flake.id);
if (topFlakeId == flake.second.id) vTop = vFlake; if (topFlakeId == flake.second.id) vTop = vFlake;
state.mkAttrs(*vFlake, 4); state.mkAttrs(*vFlake, 4);

View file

@ -38,15 +38,34 @@ struct Flake
std::optional<uint64_t> revCount; std::optional<uint64_t> revCount;
std::vector<FlakeRef> requires; std::vector<FlakeRef> requires;
std::shared_ptr<FlakeRegistry> lockFile; std::shared_ptr<FlakeRegistry> lockFile;
std::map<FlakeId, FlakeRef> nonFlakeRequires;
Value * vProvides; // FIXME: gc Value * vProvides; // FIXME: gc
// commit hash
// date // date
// content hash // content hash
Flake(FlakeRef & flakeRef) : ref(flakeRef) {}; Flake(const FlakeRef flakeRef) : ref(flakeRef) {};
};
struct NonFlake
{
FlakeId id;
FlakeRef ref;
Path path;
// date
// content hash
NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {};
}; };
Flake getFlake(EvalState &, const FlakeRef &); Flake getFlake(EvalState &, const FlakeRef &);
struct Dependencies
{
FlakeId topFlakeId;
std::vector<Flake> flakes;
std::vector<NonFlake> nonFlakes;
};
Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef);
FlakeRegistry updateLockFile(EvalState &, Flake &); FlakeRegistry updateLockFile(EvalState &, Flake &);
void updateLockFile(EvalState &, std::string); void updateLockFile(EvalState &, std::string);

View file

@ -11,7 +11,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand
{ {
Path outLink = "result"; Path outLink = "result";
std::optional<std::string> gitRepo = std::nullopt; bool updateLock = true;
CmdBuild() CmdBuild()
{ {
@ -28,9 +28,9 @@ struct CmdBuild : MixDryRun, InstallablesCommand
.set(&outLink, Path("")); .set(&outLink, Path(""));
mkFlag() mkFlag()
.longName("update-lock-file") .longName("no-update")
.description("update the lock file") .description("don't update the lock file")
.dest(&gitRepo); .set(&updateLock, false);
} }
std::string name() override std::string name() override
@ -78,8 +78,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand
} }
} }
if (gitRepo) if(updateLock)
updateLockFile(*evalState, *gitRepo); for (int i = 0; i < installables.size(); i++)
if (auto flakeUri = installableToFlakeUri)
updateLockFile(*evalState, FlakeRef(*flakeUri));
} }
}; };

View file

@ -36,6 +36,60 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs
} }
}; };
void printFlakeInfo(Flake & flake, bool json) {
if (json) {
nlohmann::json j;
j["name"] = flake.id;
j["location"] = flake.path;
j["description"] = flake.description;
std::cout << j.dump(4) << std::endl;
} else {
std::cout << "Name: " << flake.id << "\n";
std::cout << "Description: " << flake.description << "\n";
std::cout << "Location: " << flake.path << "\n";
}
}
void printNonFlakeInfo(NonFlake & nonFlake, bool json) {
if (json) {
nlohmann::json j;
j["name"] = nonFlake.id;
j["location"] = nonFlake.path;
std::cout << j.dump(4) << std::endl;
} else {
std::cout << "name: " << nonFlake.id << "\n";
std::cout << "Location: " << nonFlake.path << "\n";
}
}
struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs
{
std::string name() override
{
return "deps";
}
std::string description() override
{
return "list informaton about dependencies";
}
void run(nix::ref<nix::Store> store) override
{
auto evalState = std::make_shared<EvalState>(searchPath, store);
FlakeRef flakeRef(flakeUri);
Dependencies deps = resolveFlake(*evalState, flakeRef, true);
for (auto & flake : deps.flakes)
printFlakeInfo(flake, json);
for (auto & nonFlake : deps.nonFlakes)
printNonFlakeInfo(nonFlake, json);
}
};
struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs
{ {
std::string name() override std::string name() override
@ -73,15 +127,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand
{ {
auto evalState = std::make_shared<EvalState>(searchPath, store); auto evalState = std::make_shared<EvalState>(searchPath, store);
nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri));
if (json) { printFlakeInfo(flake, json);
nlohmann::json j;
j["location"] = flake.path;
j["description"] = flake.description;
std::cout << j.dump(4) << std::endl;
} else {
std::cout << "Description: " << flake.description << "\n";
std::cout << "Location: " << flake.path << "\n";
}
} }
}; };
@ -218,6 +264,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command
: MultiCommand({make_ref<CmdFlakeList>() : MultiCommand({make_ref<CmdFlakeList>()
, make_ref<CmdFlakeUpdate>() , make_ref<CmdFlakeUpdate>()
, make_ref<CmdFlakeInfo>() , make_ref<CmdFlakeInfo>()
, make_ref<CmdFlakeDeps>()
, make_ref<CmdFlakeAdd>() , make_ref<CmdFlakeAdd>()
, make_ref<CmdFlakeRemove>() , make_ref<CmdFlakeRemove>()
, make_ref<CmdFlakePin>() , make_ref<CmdFlakePin>()