lix/src/libexpr/flake/flake.cc

504 lines
18 KiB
C++
Raw Normal View History

2019-02-12 17:23:11 +00:00
#include "flake.hh"
#include "lockfile.hh"
2018-11-29 18:18:36 +00:00
#include "primops.hh"
#include "eval-inline.hh"
#include "store-api.hh"
#include "fetchers/fetchers.hh"
2018-11-29 18:18:36 +00:00
#include <iostream>
#include <ctime>
#include <iomanip>
2018-11-29 18:18:36 +00:00
namespace nix {
using namespace flake;
namespace flake {
/* If 'allowLookup' is true, then resolve 'flakeRef' using the
registries. */
static FlakeRef maybeLookupFlake(
EvalState & state,
const FlakeRef & flakeRef,
bool allowLookup)
2018-11-29 18:18:36 +00:00
{
if (!flakeRef.isDirect()) {
if (allowLookup)
return flakeRef.resolve(state.store);
else
throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", flakeRef);
} else
return flakeRef;
}
typedef std::vector<std::pair<FlakeRef, FlakeRef>> RefMap;
static FlakeRef lookupInRefMap(
const RefMap & refMap,
const FlakeRef & flakeRef)
{
#if 0
// FIXME: inefficient.
for (auto & i : refMap) {
if (flakeRef.contains(i.first)) {
debug("mapping '%s' to previously seen input '%s' -> '%s",
flakeRef, i.first, i.second);
return i.second;
}
}
#endif
return flakeRef;
}
static void expectType(EvalState & state, ValueType type,
Value & value, const Pos & pos)
{
if (value.type == tThunk && value.isTrivial())
state.forceValue(value, pos);
if (value.type != type)
throw Error("expected %s but got %s at %s",
showType(type), showType(value.type), pos);
}
static Flake getFlake(EvalState & state, const FlakeRef & originalRef,
bool allowLookup, RefMap & refMap)
2018-11-30 15:11:15 +00:00
{
auto flakeRef = lookupInRefMap(refMap,
maybeLookupFlake(state,
lookupInRefMap(refMap, originalRef), allowLookup));
auto [sourceInfo, resolvedInput] = flakeRef.input->fetchTree(state.store);
FlakeRef resolvedRef(resolvedInput, flakeRef.subdir);
debug("got flake source '%s' from flake URL '%s'",
state.store->printStorePath(sourceInfo.storePath), resolvedRef);
refMap.push_back({originalRef, resolvedRef});
refMap.push_back({flakeRef, resolvedRef});
if (state.allowedPaths)
state.allowedPaths->insert(sourceInfo.actualPath);
2019-05-01 18:38:41 +00:00
// Guard against symlink attacks.
auto flakeFile = canonPath(sourceInfo.actualPath + "/" + resolvedRef.subdir + "/flake.nix");
if (!isInDir(flakeFile, sourceInfo.actualPath))
throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
resolvedRef, state.store->printStorePath(sourceInfo.storePath));
Flake flake {
.originalRef = originalRef,
.resolvedRef = resolvedRef,
.sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
};
if (!pathExists(flakeFile))
2019-05-01 16:07:36 +00:00
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir);
2018-11-29 18:18:36 +00:00
Value vInfo;
state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
2018-11-29 18:18:36 +00:00
expectType(state, tAttrs, vInfo, Pos(state.symbols.create(flakeFile), 0, 0));
2018-11-29 18:18:36 +00:00
2019-07-11 11:54:53 +00:00
auto sEdition = state.symbols.create("edition");
auto sEpoch = state.symbols.create("epoch"); // FIXME: remove soon
auto edition = vInfo.attrs->get(sEdition);
if (!edition)
edition = vInfo.attrs->get(sEpoch);
if (edition) {
expectType(state, tInt, *(**edition).value, *(**edition).pos);
flake.edition = (**edition).value->integer;
if (flake.edition > 201909)
2019-07-11 11:54:53 +00:00
throw Error("flake '%s' requires unsupported edition %d; please upgrade Nix", flakeRef, flake.edition);
if (flake.edition < 201909)
throw Error("flake '%s' has illegal edition %d", flakeRef, flake.edition);
2019-05-22 12:31:40 +00:00
} else
2019-07-11 11:54:53 +00:00
throw Error("flake '%s' lacks attribute 'edition'", flakeRef);
2019-05-22 12:31:40 +00:00
if (auto description = vInfo.attrs->get(state.sDescription)) {
expectType(state, tString, *(**description).value, *(**description).pos);
flake.description = (**description).value->string.s;
}
2018-11-29 18:18:36 +00:00
auto sInputs = state.symbols.create("inputs");
2019-10-08 14:30:04 +00:00
auto sUrl = state.symbols.create("url");
auto sUri = state.symbols.create("uri"); // FIXME: remove soon
auto sFlake = state.symbols.create("flake");
if (std::optional<Attr *> inputs = vInfo.attrs->get(sInputs)) {
expectType(state, tAttrs, *(**inputs).value, *(**inputs).pos);
for (Attr inputAttr : *(*(**inputs).value).attrs) {
expectType(state, tAttrs, *inputAttr.value, *inputAttr.pos);
2018-11-29 18:18:36 +00:00
FlakeInput input(parseFlakeRef(inputAttr.name));
for (Attr attr : *(inputAttr.value->attrs)) {
2019-10-08 14:30:04 +00:00
if (attr.name == sUrl || attr.name == sUri) {
expectType(state, tString, *attr.value, *attr.pos);
input.ref = parseFlakeRef(attr.value->string.s);
} else if (attr.name == sFlake) {
expectType(state, tBool, *attr.value, *attr.pos);
input.isFlake = attr.value->boolean;
} else
throw Error("flake input '%s' has an unsupported attribute '%s', at %s",
inputAttr.name, attr.name, *attr.pos);
}
flake.inputs.emplace(inputAttr.name, input);
}
}
auto sOutputs = state.symbols.create("outputs");
if (auto outputs = vInfo.attrs->get(sOutputs)) {
expectType(state, tLambda, *(**outputs).value, *(**outputs).pos);
flake.vOutputs = (**outputs).value;
if (flake.vOutputs->lambda.fun->matchAttrs) {
for (auto & formal : flake.vOutputs->lambda.fun->formals->formals) {
if (formal.name != state.sSelf)
flake.inputs.emplace(formal.name, FlakeInput(parseFlakeRef(formal.name)));
}
}
2018-11-29 18:18:36 +00:00
} else
throw Error("flake '%s' lacks attribute 'outputs'", flakeRef);
2018-11-29 18:18:36 +00:00
for (auto & attr : *vInfo.attrs) {
2019-07-11 11:54:53 +00:00
if (attr.name != sEdition &&
attr.name != sEpoch &&
attr.name != state.sDescription &&
attr.name != sInputs &&
attr.name != sOutputs)
throw Error("flake '%s' has an unsupported attribute '%s', at %s",
flakeRef, attr.name, *attr.pos);
}
2018-11-29 18:18:36 +00:00
return flake;
}
Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
{
RefMap refMap;
return getFlake(state, originalRef, allowLookup, refMap);
}
static std::pair<fetchers::Tree, FlakeRef> getNonFlake(
EvalState & state,
const FlakeRef & originalRef,
bool allowLookup,
RefMap & refMap)
{
auto flakeRef = lookupInRefMap(refMap,
maybeLookupFlake(state,
lookupInRefMap(refMap, originalRef), allowLookup));
auto [sourceInfo, resolvedInput] = flakeRef.input->fetchTree(state.store);
FlakeRef resolvedRef(resolvedInput, flakeRef.subdir);
debug("got non-flake source '%s' with flakeref %s",
state.store->printStorePath(sourceInfo.storePath), resolvedRef);
refMap.push_back({originalRef, resolvedRef});
refMap.push_back({flakeRef, resolvedRef});
if (state.allowedPaths)
state.allowedPaths->insert(sourceInfo.actualPath);
return std::make_pair(std::move(sourceInfo), resolvedRef);
}
2020-01-24 12:07:03 +00:00
bool allowedToUseRegistries(LockFileMode handle, bool isTopRef)
{
if (handle == AllPure) return false;
else if (handle == TopRefUsesRegistries) return isTopRef;
else if (handle == UpdateLockFile) return true;
else if (handle == UseUpdatedLockFile) return true;
else if (handle == RecreateLockFile) return true;
else if (handle == UseNewLockFile) return true;
else assert(false);
}
2019-06-04 19:07:55 +00:00
/* Given a flakeref and its subtree of the lockfile, return an updated
subtree of the lockfile. That is, if the 'flake.nix' of the
referenced flake has inputs that don't have a corresponding entry
in the lockfile, they're added to the lockfile; conversely, any
lockfile entries that don't have a corresponding entry in flake.nix
are removed.
Note that this is lazy: we only recursively fetch inputs that are
not in the lockfile yet. */
static std::pair<Flake, LockedInput> updateLocks(
RefMap & refMap,
const std::string & inputPath,
EvalState & state,
const Flake & flake,
2020-01-24 12:07:03 +00:00
LockFileMode lockFileMode,
const LockedInputs & oldEntry,
bool topRef)
2019-05-01 09:38:48 +00:00
{
LockedInput newEntry(
flake.resolvedRef,
flake.originalRef,
flake.sourceInfo->narHash);
std::vector<std::function<void()>> postponed;
for (auto & [id, input] : flake.inputs) {
auto inputPath2 = (inputPath.empty() ? "" : inputPath + "/") + id;
auto i = oldEntry.inputs.find(id);
if (i != oldEntry.inputs.end() && i->second.originalRef == input.ref) {
newEntry.inputs.insert_or_assign(id, i->second);
} else {
2020-01-24 12:07:03 +00:00
if (lockFileMode == AllPure || lockFileMode == TopRefUsesRegistries)
throw Error("cannot update flake input '%s' in pure mode", id);
auto warn = [&](const FlakeRef & resolvedRef, const fetchers::Tree & sourceInfo) {
if (i == oldEntry.inputs.end())
printInfo("mapped flake input '%s' to '%s'",
inputPath2, resolvedRef);
else
printMsg(lvlWarn, "updated flake input '%s' from '%s' to '%s'",
inputPath2, i->second.originalRef, resolvedRef);
};
if (input.isFlake) {
auto actualInput = getFlake(state, input.ref,
2020-01-24 12:07:03 +00:00
allowedToUseRegistries(lockFileMode, false), refMap);
warn(actualInput.resolvedRef, *actualInput.sourceInfo);
postponed.push_back([&, id{id}, inputPath2, actualInput]() {
newEntry.inputs.insert_or_assign(id,
2020-01-24 12:07:03 +00:00
updateLocks(refMap, inputPath2, state, actualInput, lockFileMode, {}, false).second);
});
} else {
auto [sourceInfo, resolvedRef] = getNonFlake(state, input.ref,
2020-01-24 12:07:03 +00:00
allowedToUseRegistries(lockFileMode, false), refMap);
warn(resolvedRef, sourceInfo);
newEntry.inputs.insert_or_assign(id,
LockedInput(resolvedRef, input.ref, sourceInfo.narHash));
}
2019-05-01 09:38:48 +00:00
}
2019-04-16 14:18:47 +00:00
}
for (auto & f : postponed) f();
return {flake, newEntry};
2019-03-29 15:18:25 +00:00
}
2019-06-04 19:07:55 +00:00
/* Compute an in-memory lockfile for the specified top-level flake,
and optionally write it to file, it the flake is writable. */
LockedFlake lockFlake(
EvalState & state,
const FlakeRef & topRef,
2020-01-24 12:07:03 +00:00
LockFileMode lockFileMode)
2019-05-01 09:38:48 +00:00
{
settings.requireExperimentalFeature("flakes");
auto flake = getFlake(state, topRef,
2020-01-24 12:07:03 +00:00
allowedToUseRegistries(lockFileMode, true));
LockFile oldLockFile;
2019-05-01 09:38:48 +00:00
2020-01-24 12:07:03 +00:00
if (lockFileMode != RecreateLockFile && lockFileMode != UseNewLockFile) {
// If recreateLockFile, start with an empty lockfile
// FIXME: symlink attack
oldLockFile = LockFile::read(
flake.sourceInfo->actualPath + "/" + flake.resolvedRef.subdir + "/flake.lock");
}
2019-05-01 09:38:48 +00:00
debug("old lock file: %s", oldLockFile);
RefMap refMap;
2020-01-24 12:07:03 +00:00
LockFile newLockFile(updateLocks(
refMap, "", state, flake, lockFileMode, oldLockFile, true).second);
2019-02-12 21:43:22 +00:00
2020-01-24 12:07:03 +00:00
debug("new lock file: %s", newLockFile);
2020-01-24 12:07:03 +00:00
if (!(newLockFile == oldLockFile)) {
if (lockFileMode == UpdateLockFile || lockFileMode == RecreateLockFile) {
if (auto sourcePath = topRef.input->getSourcePath()) {
2020-01-24 12:07:03 +00:00
if (!newLockFile.isImmutable()) {
if (settings.warnDirty)
warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
} else {
warn("updated lock file of flake '%s'", topRef);
2020-01-24 12:07:03 +00:00
newLockFile.write(*sourcePath + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock");
// FIXME: rewriting the lockfile changed the
// top-level repo, so we should re-read it.
#if 0
// Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store.
runProgram("git", true,
{ "-C", *sourcePath, "add",
"--force",
"--intent-to-add",
(topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" });
#endif
}
2019-05-21 13:03:54 +00:00
} else
warn("cannot write lock file of remote flake '%s'", topRef);
2020-01-24 12:07:03 +00:00
} else if (lockFileMode != AllPure && lockFileMode != TopRefUsesRegistries)
warn("using updated lock file without writing it to file");
}
2018-11-29 18:18:36 +00:00
2020-01-24 12:07:03 +00:00
return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
}
static void emitSourceInfoAttrs(EvalState & state, const fetchers::Tree & sourceInfo, Value & vAttrs)
{
assert(state.store->isValidPath(sourceInfo.storePath));
auto pathS = state.store->printStorePath(sourceInfo.storePath);
mkString(*state.allocAttr(vAttrs, state.sOutPath), pathS, {pathS});
if (sourceInfo.rev) {
mkString(*state.allocAttr(vAttrs, state.symbols.create("rev")),
sourceInfo.rev->gitRev());
mkString(*state.allocAttr(vAttrs, state.symbols.create("shortRev")),
sourceInfo.rev->gitShortRev());
}
if (sourceInfo.revCount)
mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount);
if (sourceInfo.lastModified)
mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")),
fmt("%s", std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S")));
}
struct LazyInput
{
bool isFlake;
LockedInput lockedInput;
};
/* Helper primop to make callFlake (below) fetch/call its inputs
lazily. Note that this primop cannot be called by user code since
it doesn't appear in 'builtins'. */
static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
auto lazyInput = (LazyInput *) args[0]->attrs;
if (lazyInput->isFlake) {
auto flake = getFlake(state, lazyInput->lockedInput.ref, false);
2019-06-04 18:34:44 +00:00
if (flake.sourceInfo->narHash != lazyInput->lockedInput.narHash)
throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile",
lazyInput->lockedInput.ref);
2019-06-04 18:34:44 +00:00
callFlake(state, flake, lazyInput->lockedInput, v);
} else {
RefMap refMap;
auto [sourceInfo, resolvedRef] = getNonFlake(state, lazyInput->lockedInput.ref, false, refMap);
if (sourceInfo.narHash != lazyInput->lockedInput.narHash)
throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile",
lazyInput->lockedInput.ref);
state.mkAttrs(v, 8);
assert(state.store->isValidPath(sourceInfo.storePath));
auto pathS = state.store->printStorePath(sourceInfo.storePath);
mkString(*state.allocAttr(v, state.sOutPath), pathS, {pathS});
emitSourceInfoAttrs(state, sourceInfo, v);
v.attrs->sort();
}
}
void callFlake(EvalState & state,
const Flake & flake,
const LockedInputs & lockedInputs,
Value & vResFinal)
2018-11-29 18:18:36 +00:00
{
auto & vRes = *state.allocValue();
2019-08-30 11:06:23 +00:00
auto & vInputs = *state.allocValue();
state.mkAttrs(vInputs, flake.inputs.size() + 1);
for (auto & [inputId, input] : flake.inputs) {
auto vFlake = state.allocAttr(vInputs, inputId);
auto vPrimOp = state.allocValue();
static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake"));
vPrimOp->type = tPrimOp;
vPrimOp->primOp = primOp;
auto vArg = state.allocValue();
vArg->type = tNull;
auto lockedInput = lockedInputs.inputs.find(inputId);
assert(lockedInput != lockedInputs.inputs.end());
// FIXME: leak
vArg->attrs = (Bindings *) new LazyInput{input.isFlake, lockedInput->second};
mkApp(*vFlake, *vPrimOp, *vArg);
2019-04-16 11:56:08 +00:00
}
2019-08-30 11:06:23 +00:00
auto & vSourceInfo = *state.allocValue();
state.mkAttrs(vSourceInfo, 8);
emitSourceInfoAttrs(state, *flake.sourceInfo, vSourceInfo);
2019-08-30 15:27:51 +00:00
vSourceInfo.attrs->sort();
2018-11-29 18:18:36 +00:00
2019-08-30 11:06:23 +00:00
vInputs.attrs->push_back(Attr(state.sSelf, &vRes));
2018-11-29 18:18:36 +00:00
2019-08-30 11:06:23 +00:00
vInputs.attrs->sort();
/* For convenience, put the outputs directly in the result, so you
can refer to an output of an input as 'inputs.foo.bar' rather
than 'inputs.foo.outputs.bar'. */
2019-08-30 11:06:23 +00:00
auto vCall = *state.allocValue();
state.eval(state.parseExprFromString(
"outputsFun: inputs: sourceInfo: let outputs = outputsFun inputs; in "
"outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; }", "/"), vCall);
auto vCall2 = *state.allocValue();
auto vCall3 = *state.allocValue();
state.callFunction(vCall, *flake.vOutputs, vCall2, noPos);
state.callFunction(vCall2, vInputs, vCall3, noPos);
state.callFunction(vCall3, vSourceInfo, vRes, noPos);
vResFinal = vRes;
2019-04-16 11:56:08 +00:00
}
void callFlake(EvalState & state,
const LockedFlake & lockedFlake,
Value & v)
{
callFlake(state, lockedFlake.flake, lockedFlake.lockFile, v);
}
2019-03-29 15:18:25 +00:00
// This function is exposed to be used in nix files.
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
callFlake(state, lockFlake(state, parseFlakeRef(state.forceStringNoCtx(*args[0], pos)),
2019-05-29 13:44:48 +00:00
evalSettings.pureEval ? AllPure : UseUpdatedLockFile), v);
2018-11-29 18:18:36 +00:00
}
static RegisterPrimOp r2("getFlake", 1, prim_getFlake);
}
Fingerprint LockedFlake::getFingerprint() const
{
// FIXME: as an optimization, if the flake contains a lock file
// and we haven't changed it, then it's sufficient to use
// flake.sourceInfo.storePath for the fingerprint.
return hashString(htSHA256,
fmt("%s;%d;%d;%s",
flake.sourceInfo->storePath.to_string(),
flake.sourceInfo->revCount.value_or(0),
flake.sourceInfo->lastModified.value_or(0),
lockFile));
}
Flake::~Flake() { }
}