lix/src/libexpr/primops/flake.cc

467 lines
16 KiB
C++
Raw Normal View History

2019-02-12 17:23:11 +00:00
#include "flake.hh"
2018-11-29 18:18:36 +00:00
#include "primops.hh"
#include "eval-inline.hh"
#include "fetchGit.hh"
#include "download.hh"
#include "args.hh"
2018-11-29 18:18:36 +00:00
#include <iostream>
2018-11-29 18:18:36 +00:00
#include <queue>
2018-11-30 15:11:15 +00:00
#include <regex>
2018-11-29 18:18:36 +00:00
#include <nlohmann/json.hpp>
namespace nix {
2019-04-15 12:08:18 +00:00
/* Read a registry. */
2019-03-21 08:30:16 +00:00
std::shared_ptr<FlakeRegistry> readRegistry(const Path & path)
2019-02-12 21:43:22 +00:00
{
2019-03-21 08:30:16 +00:00
auto registry = std::make_shared<FlakeRegistry>();
2019-02-12 21:43:22 +00:00
if (!pathExists(path))
return std::make_shared<FlakeRegistry>();
auto json = nlohmann::json::parse(readFile(path));
auto version = json.value("version", 0);
if (version != 1)
throw Error("flake registry '%s' has unsupported version %d", path, version);
auto flakes = json["flakes"];
2019-04-08 17:03:00 +00:00
for (auto i = flakes.begin(); i != flakes.end(); ++i)
registry->entries.emplace(i.key(), FlakeRef(i->value("uri", "")));
2019-02-12 21:43:22 +00:00
return registry;
}
2019-04-15 12:08:18 +00:00
/* Write a registry to a file. */
2019-04-16 12:27:54 +00:00
void writeRegistry(const FlakeRegistry & registry, const Path & path)
2019-02-25 12:46:37 +00:00
{
2019-03-29 15:18:25 +00:00
nlohmann::json json;
json["version"] = 1;
2019-04-08 17:03:00 +00:00
for (auto elem : registry.entries)
json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} };
createDirs(dirOf(path));
2019-02-25 12:46:37 +00:00
writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file.
}
2019-03-29 15:18:25 +00:00
LockFile::FlakeEntry readFlakeEntry(nlohmann::json json)
{
FlakeRef flakeRef(json["uri"]);
if (!flakeRef.isImmutable())
throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string());
LockFile::FlakeEntry entry(flakeRef);
auto nonFlakeRequires = json["nonFlakeRequires"];
for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) {
FlakeRef flakeRef(i->value("uri", ""));
if (!flakeRef.isImmutable())
throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string());
entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef);
}
auto requires = json["requires"];
for (auto i = requires.begin(); i != requires.end(); ++i)
entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i));
return entry;
}
LockFile readLockFile(const Path & path)
{
LockFile lockFile;
if (!pathExists(path))
return lockFile;
auto json = nlohmann::json::parse(readFile(path));
auto version = json.value("version", 0);
if (version != 1)
throw Error("lock file '%s' has unsupported version %d", path, version);
auto nonFlakeRequires = json["nonFlakeRequires"];
for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) {
FlakeRef flakeRef(i->value("uri", ""));
if (!flakeRef.isImmutable())
throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string());
lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef);
}
auto requires = json["requires"];
for (auto i = requires.begin(); i != requires.end(); ++i)
lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i));
return lockFile;
}
2019-04-16 12:27:54 +00:00
nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry)
2019-03-29 15:18:25 +00:00
{
nlohmann::json json;
json["uri"] = entry.ref.to_string();
for (auto & x : entry.nonFlakeEntries)
json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string();
for (auto & x : entry.flakeEntries)
2019-04-16 14:18:47 +00:00
json["requires"][x.first.to_string()] = flakeEntryToJson(x.second);
2019-03-29 15:18:25 +00:00
return json;
}
2019-04-16 12:27:54 +00:00
void writeLockFile(const LockFile & lockFile, const Path & path)
2019-03-29 15:18:25 +00:00
{
nlohmann::json json;
json["version"] = 1;
json["nonFlakeRequires"] = nlohmann::json::object();
2019-03-29 15:18:25 +00:00
for (auto & x : lockFile.nonFlakeEntries)
json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string();
json["requires"] = nlohmann::json::object();
2019-03-29 15:18:25 +00:00
for (auto & x : lockFile.flakeEntries)
2019-04-16 14:18:47 +00:00
json["requires"][x.first.to_string()] = flakeEntryToJson(x.second);
2019-03-29 15:18:25 +00:00
createDirs(dirOf(path));
writeFile(path, json.dump(4)); // '4' = indentation in json file
}
2019-04-08 17:03:00 +00:00
std::shared_ptr<FlakeRegistry> getGlobalRegistry()
2019-03-21 08:30:16 +00:00
{
Path registryFile = settings.nixDataDir + "/nix/flake-registry.json";
return readRegistry(registryFile);
2019-03-21 08:30:16 +00:00
}
2019-04-08 17:03:00 +00:00
Path getUserRegistryPath()
2019-03-21 08:30:16 +00:00
{
2019-04-08 17:03:00 +00:00
return getHome() + "/.config/nix/registry.json";
2019-03-21 08:30:16 +00:00
}
2019-04-08 17:03:00 +00:00
std::shared_ptr<FlakeRegistry> getUserRegistry()
2019-03-21 08:30:16 +00:00
{
2019-04-08 17:03:00 +00:00
return readRegistry(getUserRegistryPath());
2019-03-21 08:30:16 +00:00
}
2019-03-21 08:30:16 +00:00
std::shared_ptr<FlakeRegistry> getFlagRegistry()
{
2019-04-08 17:03:00 +00:00
// TODO (Nick): Implement this.
2019-03-21 08:30:16 +00:00
return std::make_shared<FlakeRegistry>();
}
2018-11-29 18:18:36 +00:00
2019-03-21 08:30:16 +00:00
const std::vector<std::shared_ptr<FlakeRegistry>> EvalState::getFlakeRegistries()
{
std::vector<std::shared_ptr<FlakeRegistry>> registries;
registries.push_back(getGlobalRegistry());
registries.push_back(getUserRegistry());
2019-03-21 08:30:16 +00:00
registries.push_back(getFlagRegistry());
return registries;
2018-11-29 18:18:36 +00:00
}
2019-02-12 21:43:22 +00:00
static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef,
2019-04-16 12:27:54 +00:00
const std::vector<std::shared_ptr<FlakeRegistry>> & registries,
std::vector<FlakeRef> pastSearches = {})
2019-02-12 17:23:11 +00:00
{
if (registries.empty() && !flakeRef.isDirect())
throw Error("indirect flake reference '%s' is not allowed", flakeRef.to_string());
2019-04-08 17:03:00 +00:00
for (std::shared_ptr<FlakeRegistry> registry : registries) {
auto i = registry->entries.find(flakeRef);
if (i != registry->entries.end()) {
auto newRef = i->second;
if (std::get_if<FlakeRef::IsAlias>(&flakeRef.data)) {
if (flakeRef.ref) newRef.ref = flakeRef.ref;
if (flakeRef.rev) newRef.rev = flakeRef.rev;
}
std::string errorMsg = "found cycle in flake registries: ";
for (FlakeRef oldRef : pastSearches) {
errorMsg += oldRef.to_string();
if (oldRef == newRef)
throw Error(errorMsg);
errorMsg += " - ";
2019-02-12 21:43:22 +00:00
}
2019-04-08 17:03:00 +00:00
pastSearches.push_back(newRef);
return lookupFlake(state, newRef, registries, pastSearches);
2019-02-12 21:43:22 +00:00
}
2019-04-08 17:03:00 +00:00
}
2019-04-08 17:03:00 +00:00
if (!flakeRef.isDirect())
throw Error("could not resolve flake reference '%s'", flakeRef.to_string());
2019-04-08 17:03:00 +00:00
return flakeRef;
2019-02-12 17:23:11 +00:00
}
2019-04-06 18:45:35 +00:00
static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bool impureIsAllowed = false)
2018-11-29 18:18:36 +00:00
{
FlakeRef fRef = lookupFlake(state, flakeRef,
impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>());
2019-04-17 11:54:06 +00:00
if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable())
throw Error("requested to fetch mutable flake '%s' in pure mode", fRef.to_string());
2019-04-08 17:03:00 +00:00
// This only downloads only one revision of the repo, not the entire history.
2019-04-06 18:45:35 +00:00
if (auto refData = std::get_if<FlakeRef::IsGitHub>(&fRef.data)) {
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
2019-02-12 17:23:11 +00:00
auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
refData->owner, refData->repo,
2019-04-06 18:45:35 +00:00
fRef.rev ? fRef.rev->to_string(Base16, false)
: fRef.ref ? *fRef.ref : "master");
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "")
url += "?access_token=" + accessToken;
auto result = getDownloader()->downloadCached(state.store, url, true, "source",
2019-04-06 18:45:35 +00:00
Hash(), nullptr, fRef.rev ? 1000000000 : settings.tarballTtl);
if (!result.etag)
throw Error("did not receive an ETag header from '%s'", url);
if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"')
throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url);
FlakeSourceInfo info(fRef);
info.storePath = result.path;
info.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1);
info.flakeRef.rev = info.rev;
info.flakeRef.ref = {};
return info;
}
2019-04-08 17:03:00 +00:00
// This downloads the entire git history
2019-04-06 18:45:35 +00:00
else if (auto refData = std::get_if<FlakeRef::IsGit>(&fRef.data)) {
auto gitInfo = exportGit(state.store, refData->uri, fRef.ref,
fRef.rev ? fRef.rev->to_string(Base16, false) : "", "source");
FlakeSourceInfo info(fRef);
info.storePath = gitInfo.storePath;
info.rev = Hash(gitInfo.rev, htSHA1);
info.revCount = gitInfo.revCount;
info.flakeRef.rev = info.rev;
// FIXME: ensure info.flakeRef.ref is set.
return info;
}
2019-04-08 17:03:00 +00:00
else if (auto refData = std::get_if<FlakeRef::IsPath>(&fRef.data)) {
if (!pathExists(refData->path + "/.git"))
throw Error("flake '%s' does not reference a Git repository", refData->path);
auto gitInfo = exportGit(state.store, refData->path, {}, "", "source");
FlakeSourceInfo info(fRef);
info.storePath = gitInfo.storePath;
info.rev = Hash(gitInfo.rev, htSHA1);
info.revCount = gitInfo.revCount;
info.flakeRef.rev = info.rev;
return info;
2018-11-30 15:11:15 +00:00
}
2018-11-29 18:18:36 +00:00
2019-02-12 17:23:11 +00:00
else abort();
2018-11-30 15:11:15 +00:00
}
2019-03-29 15:18:25 +00:00
// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function.
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false)
2018-11-30 15:11:15 +00:00
{
FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed);
debug("got flake source '%s' with revision %s",
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
state.store->assertStorePath(sourceInfo.storePath);
2018-11-30 15:11:15 +00:00
if (state.allowedPaths)
state.allowedPaths->insert(sourceInfo.storePath);
Flake flake(flakeRef, std::move(sourceInfo));
if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
// FIXME: ehm?
if (flake.sourceInfo.rev)
flake.ref = FlakeRef(flakeRef.baseRef().to_string()
+ "/" + flake.sourceInfo.rev->to_string(Base16, false));
}
2018-11-29 18:18:36 +00:00
Value vInfo;
state.evalFile(sourceInfo.storePath + "/flake.nix", vInfo); // FIXME: symlink attack
2018-11-29 18:18:36 +00:00
state.forceAttrs(vInfo);
2019-04-16 11:56:08 +00:00
// FIXME: change to "id"?
2018-11-29 18:18:36 +00:00
if (auto name = vInfo.attrs->get(state.sName))
2019-02-12 17:23:11 +00:00
flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos);
2018-11-29 18:18:36 +00:00
else
throw Error("flake lacks attribute 'name'");
if (auto description = vInfo.attrs->get(state.sDescription))
flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos);
if (auto requires = vInfo.attrs->get(state.symbols.create("requires"))) {
state.forceList(*(**requires).value, *(**requires).pos);
for (unsigned int n = 0; n < (**requires).value->listSize(); ++n)
2019-02-12 21:43:22 +00:00
flake.requires.push_back(FlakeRef(state.forceStringNoCtx(
*(**requires).value->listElems()[n], *(**requires).pos)));
2018-11-29 18:18:36 +00:00
}
if (std::optional<Attr *> nonFlakeRequires = vInfo.attrs->get(state.symbols.create("nonFlakeRequires"))) {
state.forceAttrs(*(**nonFlakeRequires).value, *(**nonFlakeRequires).pos);
for (Attr attr : *(*(**nonFlakeRequires).value).attrs) {
std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos);
FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri);
flake.nonFlakeRequires.insert_or_assign(attr.name, nonFlakeRef);
}
}
2018-11-29 18:18:36 +00:00
if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) {
state.forceFunction(*(**provides).value, *(**provides).pos);
flake.vProvides = (**provides).value;
} else
throw Error("flake lacks attribute 'provides'");
return flake;
}
// Get the `NonFlake` corresponding to a `FlakeRef`.
2019-03-21 08:30:16 +00:00
NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias)
{
FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef);
debug("got non-flake source '%s' with revision %s",
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
auto flakePath = sourceInfo.storePath;
state.store->assertStorePath(flakePath);
if (state.allowedPaths)
state.allowedPaths->insert(flakePath);
NonFlake nonFlake(flakeRef);
if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
if (sourceInfo.rev)
nonFlake.ref = FlakeRef(flakeRef.baseRef().to_string()
+ "/" + sourceInfo.rev->to_string(Base16, false));
}
nonFlake.path = flakePath;
2019-03-21 08:30:16 +00:00
nonFlake.alias = alias;
return nonFlake;
}
/* Given a flake reference, recursively fetch it and its
dependencies.
FIXME: this should return a graph of flakes.
*/
Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef,
RegistryAccess registryAccess, bool isTopFlake)
2018-11-29 18:18:36 +00:00
{
Flake flake = getFlake(state, topRef,
registryAccess == AllowRegistry || (registryAccess == AllowRegistryAtTop && isTopFlake));
2019-04-16 14:18:47 +00:00
LockFile lockFile;
if (isTopFlake)
lockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack
2019-03-29 15:18:25 +00:00
Dependencies deps(flake);
2018-11-29 18:18:36 +00:00
2019-03-29 15:18:25 +00:00
for (auto & nonFlakeInfo : flake.nonFlakeRequires)
deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first));
2019-04-16 14:18:47 +00:00
for (auto newFlakeRef : flake.requires) {
auto i = lockFile.flakeEntries.find(newFlakeRef);
if (i != lockFile.flakeEntries.end()) newFlakeRef = i->second.ref;
// FIXME: propagate lockFile downwards
deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, registryAccess, false));
2019-04-16 14:18:47 +00:00
}
2019-03-29 15:18:25 +00:00
return deps;
}
2019-04-16 12:27:54 +00:00
LockFile::FlakeEntry dependenciesToFlakeEntry(const Dependencies & deps)
2019-03-29 15:18:25 +00:00
{
LockFile::FlakeEntry entry(deps.flake.sourceInfo.flakeRef);
2018-11-29 18:18:36 +00:00
2019-04-16 12:27:54 +00:00
for (auto & deps : deps.flakeDeps)
2019-03-29 15:18:25 +00:00
entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps));
2019-02-12 21:43:22 +00:00
2019-04-16 12:27:54 +00:00
for (auto & nonFlake : deps.nonFlakeDeps)
2019-03-21 08:30:16 +00:00
entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.ref);
2018-11-29 18:18:36 +00:00
2019-03-29 15:18:25 +00:00
return entry;
2018-11-29 18:18:36 +00:00
}
static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef)
{
Dependencies deps = resolveFlake(evalState, flakeRef, AllowRegistry);
2019-03-29 15:18:25 +00:00
LockFile::FlakeEntry entry = dependenciesToFlakeEntry(deps);
LockFile lockFile;
lockFile.flakeEntries = entry.flakeEntries;
lockFile.nonFlakeEntries = entry.nonFlakeEntries;
return lockFile;
}
2019-04-16 12:27:54 +00:00
void updateLockFile(EvalState & state, const Path & path)
{
// FIXME: don't copy 'path' to the store (especially since we
// dirty it immediately afterwards).
FlakeRef flakeRef = FlakeRef(path); // FIXME: ugly
auto lockFile = makeLockFile(state, flakeRef);
writeLockFile(lockFile, path + "/flake.lock");
// Hack: Make sure that flake.lock is visible to Git. Otherwise,
// exportGit will fail to copy it to the Nix store.
runProgram("git", true, { "-C", path, "add", "flake.lock" });
}
2019-04-16 11:56:08 +00:00
void callFlake(EvalState & state, const Dependencies & flake, Value & v)
2018-11-29 18:18:36 +00:00
{
2019-04-16 11:56:08 +00:00
// Construct the resulting attrset '{description, provides,
// ...}'. This attrset is passed lazily as an argument to 'provides'.
2018-11-29 18:18:36 +00:00
2019-04-19 08:58:08 +00:00
state.mkAttrs(v, flake.flakeDeps.size() + flake.nonFlakeDeps.size() + 8);
2018-11-29 18:18:36 +00:00
2019-04-16 11:56:08 +00:00
for (auto & dep : flake.flakeDeps) {
auto vFlake = state.allocAttr(v, dep.flake.id);
callFlake(state, dep, *vFlake);
}
2019-04-16 11:56:08 +00:00
for (auto & dep : flake.nonFlakeDeps) {
auto vNonFlake = state.allocAttr(v, dep.alias);
state.mkAttrs(*vNonFlake, 4);
2019-04-16 11:56:08 +00:00
state.store->isValidPath(dep.path);
mkString(*state.allocAttr(*vNonFlake, state.sOutPath), dep.path, {dep.path});
}
2019-04-16 11:56:08 +00:00
mkString(*state.allocAttr(v, state.sDescription), flake.flake.description);
auto & path = flake.flake.sourceInfo.storePath;
state.store->isValidPath(path);
mkString(*state.allocAttr(v, state.sOutPath), path, {path});
if (flake.flake.sourceInfo.revCount)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *flake.flake.sourceInfo.revCount);
2018-11-29 18:18:36 +00:00
2019-04-16 11:56:08 +00:00
auto vProvides = state.allocAttr(v, state.symbols.create("provides"));
mkApp(*vProvides, *flake.flake.vProvides, v);
2018-11-29 18:18:36 +00:00
2019-04-16 14:29:44 +00:00
v.attrs->push_back(Attr(state.symbols.create("self"), &v));
2019-04-16 11:56:08 +00:00
v.attrs->sort();
}
2019-04-16 11:56:08 +00:00
// Return the `provides` of the top flake, while assigning to `v` the provides
// of the dependencies as well.
void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v)
2019-04-16 11:56:08 +00:00
{
callFlake(state, resolveFlake(state, flakeRef, registryAccess), v);
}
2019-03-29 15:18:25 +00:00
// This function is exposed to be used in nix files.
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos),
evalSettings.pureEval ? DisallowRegistry : AllowRegistryAtTop, v);
2018-11-29 18:18:36 +00:00
}
static RegisterPrimOp r2("getFlake", 1, prim_getFlake);
}