Fixed Flake data type and flake fetching

This commit is contained in:
Nick Van den Broeck 2019-05-01 11:38:48 +02:00
parent f8c4742c2f
commit 201f92e02c
3 changed files with 102 additions and 93 deletions

View file

@ -204,28 +204,32 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const
return flakeRef; return flakeRef;
} }
static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef fRef, bool impureIsAllowed = false) // Lookups happen here too
static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false)
{ {
if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) FlakeRef resolvedRef = lookupFlake(state, flakeRef,
throw Error("requested to fetch mutable flake '%s' in pure mode", fRef); impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>());
if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable())
throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef);
// This only downloads only one revision of the repo, not the entire history. // This only downloads only one revision of the repo, not the entire history.
if (auto refData = std::get_if<FlakeRef::IsGitHub>(&fRef.data)) { if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) {
// FIXME: use regular /archive URLs instead? api.github.com // FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits. // might have stricter rate limits.
auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
refData->owner, refData->repo, refData->owner, refData->repo,
fRef.rev ? fRef.rev->to_string(Base16, false) resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false)
: fRef.ref ? *fRef.ref : "master"); : resolvedRef.ref ? *resolvedRef.ref : "master");
std::string accessToken = settings.githubAccessToken.get(); std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "") if (accessToken != "")
url += "?access_token=" + accessToken; url += "?access_token=" + accessToken;
auto result = getDownloader()->downloadCached(state.store, url, true, "source", auto result = getDownloader()->downloadCached(state.store, url, true, "source",
Hash(), nullptr, fRef.rev ? 1000000000 : settings.tarballTtl); Hash(), nullptr, resolvedRef.rev ? 1000000000 : settings.tarballTtl);
if (!result.etag) if (!result.etag)
throw Error("did not receive an ETag header from '%s'", url); throw Error("did not receive an ETag header from '%s'", url);
@ -233,72 +237,60 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef fRef, bool i
if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"')
throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url);
FlakeSourceInfo info(fRef); std::string rev = std::string(*result.etag, 1, result.etag->size() - 2);
const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + rev);
SourceInfo info(ref);
info.storePath = result.path; info.storePath = result.path;
info.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1);
info.flakeRef.rev = info.rev;
info.flakeRef.ref = {};
return info; return info;
} }
// This downloads the entire git history // This downloads the entire git history
else if (auto refData = std::get_if<FlakeRef::IsGit>(&fRef.data)) { else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) {
auto gitInfo = exportGit(state.store, refData->uri, fRef.ref, fRef.rev, "source"); auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source");
FlakeSourceInfo info(fRef); const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false));
SourceInfo info(ref);
info.storePath = gitInfo.storePath; info.storePath = gitInfo.storePath;
info.rev = gitInfo.rev;
info.revCount = gitInfo.revCount; info.revCount = gitInfo.revCount;
info.flakeRef.ref = gitInfo.ref;
info.flakeRef.rev = info.rev;
return info; return info;
} }
else if (auto refData = std::get_if<FlakeRef::IsPath>(&fRef.data)) { else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) {
if (!pathExists(refData->path + "/.git")) if (!pathExists(refData->path + "/.git"))
throw Error("flake '%s' does not reference a Git repository", refData->path); throw Error("flake '%s' does not reference a Git repository", refData->path);
auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source"); auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source");
FlakeSourceInfo info(fRef); const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false));
SourceInfo info(ref);
info.storePath = gitInfo.storePath; info.storePath = gitInfo.storePath;
info.rev = gitInfo.rev;
info.revCount = gitInfo.revCount; info.revCount = gitInfo.revCount;
info.flakeRef.ref = gitInfo.ref;
info.flakeRef.rev = info.rev;
return info; return info;
} }
else abort(); else abort();
} }
// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function. // This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here.
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false)
{ {
FlakeRef resolvedRef = lookupFlake(state, flakeRef, SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed);
impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>()); debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string());
FlakeSourceInfo sourceInfo = fetchFlake(state, resolvedRef, impureIsAllowed); FlakeRef resolvedRef = sourceInfo.resolvedRef;
debug("got flake source '%s' with revision %s",
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
resolvedRef = sourceInfo.flakeRef; // `resolvedRef` is now immutable resolvedRef = sourceInfo.resolvedRef; // `resolvedRef` is now immutable
state.store->assertStorePath(sourceInfo.storePath); state.store->assertStorePath(sourceInfo.storePath);
if (state.allowedPaths) if (state.allowedPaths)
state.allowedPaths->insert(sourceInfo.storePath); state.allowedPaths->insert(sourceInfo.storePath);
Flake flake(resolvedRef, std::move(sourceInfo));
if (std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) {
// FIXME: ehm?
if (flake.sourceInfo.rev)
flake.ref = FlakeRef(resolvedRef.baseRef().to_string()
+ "/" + flake.sourceInfo.rev->to_string(Base16, false));
}
// Guard against symlink attacks. // Guard against symlink attacks.
auto flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix");
if (!isInDir(flakeFile, sourceInfo.storePath)) if (!isInDir(flakeFile, sourceInfo.storePath))
throw Error("flake file '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); throw Error("flake file '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath);
Flake flake(flakeRef, sourceInfo);
if (!pathExists(flakeFile)) if (!pathExists(flakeFile))
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir);
@ -344,24 +336,18 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe
// Get the `NonFlake` corresponding to a `FlakeRef`. // Get the `NonFlake` corresponding to a `FlakeRef`.
NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias) NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias)
{ {
FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); SourceInfo sourceInfo = fetchFlake(state, flakeRef);
debug("got non-flake source '%s' with revision %s", debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string());
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
auto flakePath = sourceInfo.storePath; FlakeRef resolvedRef = sourceInfo.resolvedRef;
state.store->assertStorePath(flakePath);
NonFlake nonFlake(flakeRef, sourceInfo);
nonFlake.storePath = sourceInfo.storePath;
state.store->assertStorePath(nonFlake.storePath);
if (state.allowedPaths) if (state.allowedPaths)
state.allowedPaths->insert(flakePath); state.allowedPaths->insert(nonFlake.storePath);
NonFlake nonFlake(flakeRef);
if (std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) {
if (sourceInfo.rev)
nonFlake.ref = FlakeRef(flakeRef.baseRef().to_string()
+ "/" + sourceInfo.rev->to_string(Base16, false));
}
nonFlake.path = flakePath;
nonFlake.alias = alias; nonFlake.alias = alias;
@ -380,7 +366,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef,
LockFile lockFile; LockFile lockFile;
if (isTopFlake) if (isTopFlake)
lockFile = readLockFile(flake.sourceInfo.storePath + "/" + flake.ref.subdir + "/flake.lock"); // FIXME: symlink attack lockFile = readLockFile(flake.storePath + flake.resolvedRef.subdir + "/flake.lock"); // FIXME: symlink attack
ResolvedFlake deps(flake); ResolvedFlake deps(flake);
@ -399,13 +385,13 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef,
LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake)
{ {
LockFile::FlakeEntry entry(resolvedFlake.flake.sourceInfo.flakeRef); LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef);
for (auto & newResFlake : resolvedFlake.flakeDeps) for (auto & newResFlake : resolvedFlake.flakeDeps)
entry.flakeEntries.insert_or_assign(newResFlake.flake.id, dependenciesToFlakeEntry(newResFlake)); entry.flakeEntries.insert_or_assign(newResFlake.flake.originalRef, dependenciesToFlakeEntry(newResFlake));
for (auto & nonFlake : resolvedFlake.nonFlakeDeps) for (auto & nonFlake : resolvedFlake.nonFlakeDeps)
entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.ref); entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.resolvedRef);
return entry; return entry;
} }
@ -453,18 +439,18 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)
auto vNonFlake = state.allocAttr(v, nonFlake.alias); auto vNonFlake = state.allocAttr(v, nonFlake.alias);
state.mkAttrs(*vNonFlake, 4); state.mkAttrs(*vNonFlake, 4);
state.store->isValidPath(nonFlake.path); state.store->isValidPath(nonFlake.storePath);
mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.path, {nonFlake.path}); mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.storePath, {nonFlake.storePath});
} }
mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description);
auto & path = resFlake.flake.sourceInfo.storePath; auto & path = resFlake.flake.storePath;
state.store->isValidPath(path); state.store->isValidPath(path);
mkString(*state.allocAttr(v, state.sOutPath), path, {path}); mkString(*state.allocAttr(v, state.sOutPath), path, {path});
if (resFlake.flake.sourceInfo.revCount) if (resFlake.flake.revCount)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.sourceInfo.revCount); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.revCount);
auto vProvides = state.allocAttr(v, state.symbols.create("provides")); auto vProvides = state.allocAttr(v, state.symbols.create("provides"));
mkApp(*vProvides, *resFlake.flake.vProvides, v); mkApp(*vProvides, *resFlake.flake.vProvides, v);

View file

@ -23,7 +23,7 @@ struct LockFile
{ {
FlakeRef ref; FlakeRef ref;
std::map<FlakeRef, FlakeEntry> flakeEntries; std::map<FlakeRef, FlakeEntry> flakeEntries;
std::map<FlakeId, FlakeRef> nonFlakeEntries; std::map<FlakeAlias, FlakeRef> nonFlakeEntries;
FlakeEntry(const FlakeRef & flakeRef) : ref(flakeRef) {}; FlakeEntry(const FlakeRef & flakeRef) : ref(flakeRef) {};
}; };
@ -43,37 +43,43 @@ std::shared_ptr<FlakeRegistry> readRegistry(const Path &);
void writeRegistry(const FlakeRegistry &, const Path &); void writeRegistry(const FlakeRegistry &, const Path &);
struct FlakeSourceInfo struct SourceInfo
{ {
FlakeRef flakeRef; FlakeRef resolvedRef;
Path storePath; Path storePath;
std::optional<Hash> rev;
std::optional<uint64_t> revCount; std::optional<uint64_t> revCount;
// date // date
FlakeSourceInfo(const FlakeRef & flakeRef) : flakeRef(flakeRef) { } SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {};
}; };
struct Flake struct Flake
{ {
FlakeId id; FlakeId id;
FlakeRef ref; FlakeRef originalRef;
FlakeRef resolvedRef;
std::string description; std::string description;
FlakeSourceInfo sourceInfo; std::optional<uint64_t> revCount;
Path storePath;
std::vector<FlakeRef> requires; std::vector<FlakeRef> requires;
std::map<FlakeAlias, FlakeRef> nonFlakeRequires; std::map<FlakeAlias, FlakeRef> nonFlakeRequires;
Value * vProvides; // FIXME: gc Value * vProvides; // FIXME: gc
Flake(const FlakeRef & flakeRef, FlakeSourceInfo && sourceInfo) // date
: ref(flakeRef), sourceInfo(sourceInfo) {}; // content hash
Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef),
resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {};
}; };
struct NonFlake struct NonFlake
{ {
FlakeAlias alias; FlakeAlias alias;
FlakeRef ref; FlakeRef originalRef;
Path path; FlakeRef resolvedRef;
std::optional<uint64_t> revCount;
Path storePath;
// date // date
// content hash // content hash
NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {}; NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef),
resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {};
}; };
std::shared_ptr<FlakeRegistry> getGlobalRegistry(); std::shared_ptr<FlakeRegistry> getGlobalRegistry();

View file

@ -45,35 +45,53 @@ void printFlakeInfo(Flake & flake, bool json) {
if (json) { if (json) {
nlohmann::json j; nlohmann::json j;
j["id"] = flake.id; j["id"] = flake.id;
j["uri"] = flake.sourceInfo.flakeRef.to_string(); j["uri"] = flake.resolvedRef.to_string();
j["description"] = flake.description; j["description"] = flake.description;
if (flake.sourceInfo.rev) if (flake.resolvedRef.ref)
j["revision"] = flake.sourceInfo.rev->to_string(Base16, false); j["branch"] = *flake.resolvedRef.ref;
if (flake.sourceInfo.revCount) if (flake.resolvedRef.rev)
j["revCount"] = *flake.sourceInfo.revCount; j["revision"] = flake.resolvedRef.rev->to_string(Base16, false);
j["path"] = flake.sourceInfo.storePath; if (flake.revCount)
j["revCount"] = *flake.revCount;
j["path"] = flake.storePath;
std::cout << j.dump(4) << std::endl; std::cout << j.dump(4) << std::endl;
} else { } else {
std::cout << "ID: " << flake.id << "\n"; std::cout << "ID: " << flake.id << "\n";
std::cout << "URI: " << flake.sourceInfo.flakeRef << "\n"; std::cout << "URI: " << flake.resolvedRef.to_string() << "\n";
std::cout << "Description: " << flake.description << "\n"; std::cout << "Description: " << flake.description << "\n";
if (flake.sourceInfo.rev) if (flake.resolvedRef.ref)
std::cout << "Revision: " << flake.sourceInfo.rev->to_string(Base16, false) << "\n"; std::cout << "Branch: " << *flake.resolvedRef.ref;
if (flake.sourceInfo.revCount) if (flake.resolvedRef.rev)
std::cout << "Revcount: " << *flake.sourceInfo.revCount << "\n"; std::cout << "Revision: " << flake.resolvedRef.rev->to_string(Base16, false) << "\n";
std::cout << "Path: " << flake.sourceInfo.storePath << "\n"; if (flake.revCount)
std::cout << "Revcount: " << *flake.revCount << "\n";
std::cout << "Path: " << flake.storePath << "\n";
} }
} }
void printNonFlakeInfo(NonFlake & nonFlake, bool json) { void printNonFlakeInfo(NonFlake & nonFlake, bool json) {
if (json) { if (json) {
nlohmann::json j; nlohmann::json j;
j["name"] = nonFlake.alias; j["id"] = nonFlake.alias;
j["location"] = nonFlake.path; j["uri"] = nonFlake.resolvedRef.to_string();
if (nonFlake.resolvedRef.ref)
j["branch"] = *nonFlake.resolvedRef.ref;
if (nonFlake.resolvedRef.rev)
j["revision"] = nonFlake.resolvedRef.rev->to_string(Base16, false);
if (nonFlake.revCount)
j["revCount"] = *nonFlake.revCount;
j["path"] = nonFlake.storePath;
std::cout << j.dump(4) << std::endl; std::cout << j.dump(4) << std::endl;
} else { } else {
std::cout << "name: " << nonFlake.alias << "\n"; std::cout << "ID: " << nonFlake.alias << "\n";
std::cout << "Location: " << nonFlake.path << "\n"; std::cout << "URI: " << nonFlake.resolvedRef.to_string() << "\n";
if (nonFlake.resolvedRef.ref)
std::cout << "Branch: " << *nonFlake.resolvedRef.ref;
if (nonFlake.resolvedRef.rev)
std::cout << "Revision: " << nonFlake.resolvedRef.rev->to_string(Base16, false) << "\n";
if (nonFlake.revCount)
std::cout << "Revcount: " << *nonFlake.revCount << "\n";
std::cout << "Path: " << nonFlake.storePath << "\n";
} }
} }
@ -244,14 +262,13 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs
FlakeRegistry userRegistry = *readRegistry(userRegistryPath); FlakeRegistry userRegistry = *readRegistry(userRegistryPath);
auto it = userRegistry.entries.find(FlakeRef(alias)); auto it = userRegistry.entries.find(FlakeRef(alias));
if (it != userRegistry.entries.end()) { if (it != userRegistry.entries.end()) {
it->second = getFlake(*evalState, it->second, true).ref; it->second = getFlake(*evalState, it->second, true).resolvedRef;
// The 'ref' in 'flake' is immutable.
writeRegistry(userRegistry, userRegistryPath); writeRegistry(userRegistry, userRegistryPath);
} else { } else {
std::shared_ptr<FlakeRegistry> globalReg = getGlobalRegistry(); std::shared_ptr<FlakeRegistry> globalReg = getGlobalRegistry();
it = globalReg->entries.find(FlakeRef(alias)); it = globalReg->entries.find(FlakeRef(alias));
if (it != globalReg->entries.end()) { if (it != globalReg->entries.end()) {
FlakeRef newRef = getFlake(*evalState, it->second, true).ref; FlakeRef newRef = getFlake(*evalState, it->second, true).resolvedRef;
userRegistry.entries.insert_or_assign(alias, newRef); userRegistry.entries.insert_or_assign(alias, newRef);
writeRegistry(userRegistry, userRegistryPath); writeRegistry(userRegistry, userRegistryPath);
} else } else