lix/src/libexpr/primops/flake.cc

442 lines
15 KiB
C++
Raw Normal View History

2019-02-12 17:23:11 +00:00
#include "flake.hh"
2018-11-29 18:18:36 +00:00
#include "primops.hh"
#include "eval-inline.hh"
#include "fetchGit.hh"
#include "download.hh"
#include "args.hh"
2018-11-29 18:18:36 +00:00
#include <iostream>
2018-11-29 18:18:36 +00:00
#include <queue>
2018-11-30 15:11:15 +00:00
#include <regex>
2018-11-29 18:18:36 +00:00
#include <nlohmann/json.hpp>
namespace nix {
2019-02-12 21:43:22 +00:00
/* Read the registry or a lock file. (Currently they have an identical
format. */
2019-03-21 08:30:16 +00:00
std::shared_ptr<FlakeRegistry> readRegistry(const Path & path)
2019-02-12 21:43:22 +00:00
{
2019-03-21 08:30:16 +00:00
auto registry = std::make_shared<FlakeRegistry>();
2019-02-12 21:43:22 +00:00
if (!pathExists(path))
return std::make_shared<FlakeRegistry>();
auto json = nlohmann::json::parse(readFile(path));
auto version = json.value("version", 0);
if (version != 1)
throw Error("flake registry '%s' has unsupported version %d", path, version);
auto flakes = json["flakes"];
for (auto i = flakes.begin(); i != flakes.end(); ++i) {
FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))};
registry->entries.emplace(i.key(), entry);
2019-02-12 21:43:22 +00:00
}
return registry;
}
2019-02-25 12:46:37 +00:00
/* Write the registry or lock file to a file. */
void writeRegistry(FlakeRegistry registry, Path path)
2019-02-25 12:46:37 +00:00
{
nlohmann::json json = {};
json["version"] = 1;
2019-02-25 12:46:37 +00:00
json["flakes"] = {};
for (auto elem : registry.entries) {
2019-03-10 06:05:05 +00:00
json["flakes"][elem.first] = { {"uri", elem.second.ref.to_string()} };
2019-02-25 12:46:37 +00:00
}
createDirs(dirOf(path));
2019-02-25 12:46:37 +00:00
writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file.
}
2019-03-21 08:30:16 +00:00
Path getUserRegistryPath()
2018-11-29 18:18:36 +00:00
{
2019-03-21 08:30:16 +00:00
return getHome() + "/.config/nix/registry.json";
}
std::shared_ptr<FlakeRegistry> getGlobalRegistry()
{
// FIXME: get from nixos.org.
Path registryFile = settings.nixDataDir + "/nix/flake-registry.json";
return readRegistry(registryFile);
2019-03-21 08:30:16 +00:00
}
2018-11-29 18:18:36 +00:00
2019-03-21 08:30:16 +00:00
std::shared_ptr<FlakeRegistry> getUserRegistry()
{
return readRegistry(getUserRegistryPath());
}
std::shared_ptr<FlakeRegistry> getFlagRegistry()
{
return std::make_shared<FlakeRegistry>();
// TODO: Implement this once the right flags are implemented.
}
2018-11-29 18:18:36 +00:00
2019-03-21 08:30:16 +00:00
const std::vector<std::shared_ptr<FlakeRegistry>> EvalState::getFlakeRegistries()
{
std::vector<std::shared_ptr<FlakeRegistry>> registries;
if (evalSettings.pureEval) {
registries.push_back(std::make_shared<FlakeRegistry>()); // global
registries.push_back(std::make_shared<FlakeRegistry>()); // user
registries.push_back(std::make_shared<FlakeRegistry>()); // local
} else {
registries.push_back(getGlobalRegistry());
registries.push_back(getUserRegistry());
registries.push_back(getLocalRegistry());
}
2019-03-21 08:30:16 +00:00
registries.push_back(getFlagRegistry());
return registries;
2018-11-29 18:18:36 +00:00
}
// Creates a Nix attribute set value listing all dependencies, so they can be used in `provides`.
Value * makeFlakeRegistryValue(EvalState & state)
2018-11-29 18:18:36 +00:00
{
auto v = state.allocValue();
2019-03-21 08:30:16 +00:00
auto registries = state.getFlakeRegistries();
2018-11-29 18:18:36 +00:00
2019-03-21 08:30:16 +00:00
int size = 0;
for (auto registry : registries)
size += registry->entries.size();
state.mkAttrs(*v, size);
2018-11-29 18:18:36 +00:00
2019-03-21 08:30:16 +00:00
for (auto & registry : registries) {
for (auto & entry : registry->entries) {
auto vEntry = state.allocAttr(*v, entry.first);
state.mkAttrs(*vEntry, 2);
mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.ref.to_string());
vEntry->attrs->sort();
}
2018-11-29 18:18:36 +00:00
}
v->attrs->sort();
2018-11-29 18:18:36 +00:00
return v;
}
2018-11-29 18:18:36 +00:00
2019-02-12 21:43:22 +00:00
static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef,
2019-03-21 08:30:16 +00:00
std::vector<std::shared_ptr<FlakeRegistry>> registries)
2019-02-12 17:23:11 +00:00
{
if (auto refData = std::get_if<FlakeRef::IsFlakeId>(&flakeRef.data)) {
2019-02-12 21:43:22 +00:00
for (auto registry : registries) {
auto i = registry->entries.find(refData->id);
if (i != registry->entries.end()) {
auto newRef = FlakeRef(i->second.ref);
if (!newRef.isDirect())
throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string());
if (refData->ref)
newRef.setRef(*refData->ref);
if (refData->rev)
newRef.setRev(*refData->rev);
2019-02-12 21:43:22 +00:00
return newRef;
}
}
throw Error("cannot find flake '%s' in the flake registry or in the flake lock file", refData->id);
2019-02-12 17:23:11 +00:00
} else
return flakeRef;
}
struct FlakeSourceInfo
{
Path storePath;
std::optional<Hash> rev;
std::optional<uint64_t> revCount;
};
static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef)
2018-11-29 18:18:36 +00:00
{
FlakeRef directFlakeRef = FlakeRef(flakeRef);
if (!flakeRef.isDirect()) {
2019-03-21 08:30:16 +00:00
directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries());
}
assert(directFlakeRef.isDirect());
// NOTE FROM NICK: I don't see why one wouldn't fetch FlakeId flakes..
if (auto refData = std::get_if<FlakeRef::IsGitHub>(&directFlakeRef.data)) {
// FIXME: require hash in pure mode.
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
2019-02-12 17:23:11 +00:00
// FIXME: support passing auth tokens for private repos.
auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
refData->owner, refData->repo,
refData->rev
? refData->rev->to_string(Base16, false)
: refData->ref
? *refData->ref
: "master");
auto result = getDownloader()->downloadCached(state.store, url, true, "source",
Hash(), nullptr, refData->rev ? 1000000000 : settings.tarballTtl);
if (!result.etag)
throw Error("did not receive an ETag header from '%s'", url);
if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"')
throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url);
FlakeSourceInfo info;
info.storePath = result.path;
info.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1);
return info;
}
else if (auto refData = std::get_if<FlakeRef::IsGit>(&directFlakeRef.data)) {
2019-02-12 17:23:11 +00:00
auto gitInfo = exportGit(state.store, refData->uri, refData->ref,
refData->rev ? refData->rev->to_string(Base16, false) : "", "source");
FlakeSourceInfo info;
info.storePath = gitInfo.storePath;
info.rev = Hash(gitInfo.rev, htSHA1);
info.revCount = gitInfo.revCount;
return info;
}
else if (auto refData = std::get_if<FlakeRef::IsPath>(&directFlakeRef.data)) {
if (!pathExists(refData->path + "/.git"))
throw Error("flake '%s' does not reference a Git repository", refData->path);
auto gitInfo = exportGit(state.store, refData->path, {}, "", "source");
FlakeSourceInfo info;
info.storePath = gitInfo.storePath;
info.rev = Hash(gitInfo.rev, htSHA1);
info.revCount = gitInfo.revCount;
return info;
2018-11-30 15:11:15 +00:00
}
2018-11-29 18:18:36 +00:00
2019-02-12 17:23:11 +00:00
else abort();
2018-11-30 15:11:15 +00:00
}
Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
2018-11-30 15:11:15 +00:00
{
FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef);
debug("got flake source '%s' with revision %s",
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
auto flakePath = sourceInfo.storePath;
2018-11-30 15:11:15 +00:00
state.store->assertStorePath(flakePath);
if (state.allowedPaths)
state.allowedPaths->insert(flakePath);
Flake flake(flakeRef);
if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
if (sourceInfo.rev)
flake.ref = FlakeRef(flakeRef.baseRef().to_string()
+ "/" + sourceInfo.rev->to_string(Base16, false));
}
flake.path = flakePath;
flake.revCount = sourceInfo.revCount;
2018-11-29 18:18:36 +00:00
Value vInfo;
2019-02-12 21:43:22 +00:00
state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack
2018-11-29 18:18:36 +00:00
state.forceAttrs(vInfo);
if (auto name = vInfo.attrs->get(state.sName))
2019-02-12 17:23:11 +00:00
flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos);
2018-11-29 18:18:36 +00:00
else
throw Error("flake lacks attribute 'name'");
if (auto description = vInfo.attrs->get(state.sDescription))
flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos);
if (auto requires = vInfo.attrs->get(state.symbols.create("requires"))) {
state.forceList(*(**requires).value, *(**requires).pos);
for (unsigned int n = 0; n < (**requires).value->listSize(); ++n)
2019-02-12 21:43:22 +00:00
flake.requires.push_back(FlakeRef(state.forceStringNoCtx(
*(**requires).value->listElems()[n], *(**requires).pos)));
2018-11-29 18:18:36 +00:00
}
if (std::optional<Attr *> nonFlakeRequires = vInfo.attrs->get(state.symbols.create("nonFlakeRequires"))) {
state.forceAttrs(*(**nonFlakeRequires).value, *(**nonFlakeRequires).pos);
for (Attr attr : *(*(**nonFlakeRequires).value).attrs) {
std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos);
FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri);
flake.nonFlakeRequires.insert_or_assign(attr.name, nonFlakeRef);
}
}
2018-11-29 18:18:36 +00:00
if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) {
state.forceFunction(*(**provides).value, *(**provides).pos);
flake.vProvides = (**provides).value;
} else
throw Error("flake lacks attribute 'provides'");
2019-02-12 21:43:22 +00:00
auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack
flake.lockFile = readRegistry(lockFile);
for (auto & entry : flake.lockFile->entries)
if (!entry.second.ref.isImmutable())
throw Error("flake lock file '%s' contains mutable entry '%s'",
lockFile, entry.second.ref.to_string());
2019-02-12 21:43:22 +00:00
2018-11-29 18:18:36 +00:00
return flake;
}
// Get the `NonFlake` corresponding to a `FlakeRef`.
NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flakeId)
{
FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef);
debug("got non-flake source '%s' with revision %s",
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
auto flakePath = sourceInfo.storePath;
state.store->assertStorePath(flakePath);
if (state.allowedPaths)
state.allowedPaths->insert(flakePath);
NonFlake nonFlake(flakeRef);
if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
if (sourceInfo.rev)
nonFlake.ref = FlakeRef(flakeRef.baseRef().to_string()
+ "/" + sourceInfo.rev->to_string(Base16, false));
}
nonFlake.path = flakePath;
nonFlake.id = flakeId;
return nonFlake;
}
/* Given a flake reference, recursively fetch it and its
dependencies.
FIXME: this should return a graph of flakes.
*/
Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef)
2018-11-29 18:18:36 +00:00
{
Dependencies deps;
std::queue<FlakeRef> todo;
bool isTopLevel = true;
todo.push(topRef);
2018-11-29 18:18:36 +00:00
auto registries = state.getFlakeRegistries();
2019-02-12 21:43:22 +00:00
2018-11-29 18:18:36 +00:00
while (!todo.empty()) {
auto flakeRef = todo.front();
2018-11-29 18:18:36 +00:00
todo.pop();
if (std::get_if<FlakeRef::IsFlakeId>(&flakeRef.data))
flakeRef = lookupFlake(state, flakeRef, registries);
if (evalSettings.pureEval && !flakeRef.isImmutable() && (!isTopLevel || !impureTopRef))
throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string());
2019-02-12 17:23:11 +00:00
auto flake = getFlake(state, flakeRef);
if (isTopLevel) {
deps.topFlakeId = flake.id;
isTopLevel = false;
}
for (auto & flakeRef : flake.requires)
todo.push(flakeRef);
2018-11-29 18:18:36 +00:00
for (auto & x : flake.nonFlakeRequires)
deps.nonFlakes.push_back(getNonFlake(state, x.second, x.first));
// TODO (Nick): If there are 2 non-flake dependencies with the same
// FlakeId, this will lead to trouble! One of the dependencies won't
// be used!
2019-02-12 21:43:22 +00:00
deps.flakes.push_back(flake);
2018-11-29 18:18:36 +00:00
}
return deps;
2018-11-29 18:18:36 +00:00
}
FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef)
{
FlakeRegistry newLockFile;
Dependencies deps = resolveFlake(evalState, flakeRef, false);
// Nick assumed that "topRefPure" means that the Flake for flakeRef can be
// fetched purely.
for (auto const& require : deps.flakes) {
FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.ref);
// The FlakeRefs are immutable because they come out of the Flake objects.
if (require.id != deps.topFlakeId)
newLockFile.entries.insert_or_assign(require.id, entry);
// TODO (Nick): If there are 2 flake dependencies with the same FlakeId,
// one of them gets ignored!
}
for (auto const& nonFlake : deps.nonFlakes) {
FlakeRegistry::Entry entry = FlakeRegistry::Entry(nonFlake.ref);
newLockFile.entries.insert_or_assign(nonFlake.id, entry);
// We are assuming the sets of FlakeIds for flakes and non-flakes
// are disjoint.
}
return newLockFile;
}
void updateLockFile(EvalState & state, Path path)
{
// 'path' is the path to the local flake repo.
2019-02-21 05:53:01 +00:00
FlakeRef flakeRef = FlakeRef("file://" + path);
if (std::get_if<FlakeRef::IsGit>(&flakeRef.data)) {
FlakeRegistry newLockFile = updateLockFile(state, flakeRef);
writeRegistry(newLockFile, path + "/flake.lock");
} else if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
2019-02-21 05:53:01 +00:00
throw UsageError("you can only update local flakes, not flakes on GitHub");
} else {
2019-02-21 05:53:01 +00:00
throw UsageError("you can only update local flakes, not flakes through their FlakeId");
}
}
// Return the `provides` of the top flake, while assigning to `v` the provides
// of the dependencies as well.
Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v)
2018-11-29 18:18:36 +00:00
{
FlakeRef flakeRef = FlakeRef(flakeUri);
Dependencies deps = resolveFlake(state, flakeRef, impure);
// FIXME: we should call each flake with only its dependencies
// (rather than the closure of the top-level flake).
2018-11-29 18:18:36 +00:00
auto vResult = state.allocValue();
// This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`.
2018-11-29 18:18:36 +00:00
state.mkAttrs(*vResult, dependencies.flakes.size());
2018-11-29 18:18:36 +00:00
Value * vTop = 0;
for (auto & flake : deps.flakes) {
auto vFlake = state.allocAttr(*vResult, flake.id);
if (topFlakeId == flake.second.id) vTop = vFlake;
state.mkAttrs(*vFlake, 4);
mkString(*state.allocAttr(*vFlake, state.sDescription), flake.description);
state.store->assertStorePath(flake.path);
mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.path, {flake.path});
if (flake.second.revCount)
mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.revCount);
2018-11-29 18:18:36 +00:00
auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides"));
mkApp(*vProvides, *flake.vProvides, *vResult); // Should this be vResult or vFlake??? Or both!
2018-11-29 18:18:36 +00:00
vFlake->attrs->sort();
}
vResult->attrs->sort();
v = *vResult;
assert(vTop);
return vTop;
}
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), false, v);
2018-11-29 18:18:36 +00:00
}
static RegisterPrimOp r2("getFlake", 1, prim_getFlake);
}