Merge pull request #2897 from NixOS/source-info

Store SourceInfo in Flake/NonFlake
This commit is contained in:
Eelco Dolstra 2019-05-29 10:08:45 +02:00 committed by GitHub
commit 479757dc15
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 120 additions and 112 deletions

View file

@ -105,10 +105,10 @@ nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry)
{
nlohmann::json json;
json["uri"] = entry.ref.to_string();
json["contentHash"] = entry.contentHash.to_string(SRI);
json["contentHash"] = entry.narHash.to_string(SRI);
for (auto & x : entry.nonFlakeEntries) {
json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string();
json["nonFlakeRequires"][x.first]["contentHash"] = x.second.contentHash.to_string(SRI);
json["nonFlakeRequires"][x.first]["contentHash"] = x.second.narHash.to_string(SRI);
}
for (auto & x : entry.flakeEntries)
json["requires"][x.first.to_string()] = flakeEntryToJson(x.second);
@ -122,7 +122,7 @@ void writeLockFile(const LockFile & lockFile, const Path & path)
json["nonFlakeRequires"] = nlohmann::json::object();
for (auto & x : lockFile.nonFlakeEntries) {
json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string();
json["nonFlakeRequires"][x.first]["contentHash"] = x.second.contentHash.to_string(SRI);
json["nonFlakeRequires"][x.first]["contentHash"] = x.second.narHash.to_string(SRI);
}
json["requires"] = nlohmann::json::object();
for (auto & x : lockFile.flakeEntries)
@ -263,6 +263,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1);
SourceInfo info(ref);
info.storePath = result.storePath;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
return info;
}
@ -276,6 +277,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
SourceInfo info(ref);
info.storePath = gitInfo.storePath;
info.revCount = gitInfo.revCount;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
return info;
}
@ -289,6 +291,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool
SourceInfo info(ref);
info.storePath = gitInfo.storePath;
info.revCount = gitInfo.revCount;
info.narHash = state.store->queryPathInfo(info.storePath)->narHash;
return info;
}
@ -315,7 +318,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe
throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath);
Flake flake(flakeRef, sourceInfo);
flake.hash = state.store->queryPathInfo(sourceInfo.storePath)->narHash;
if (!pathExists(realFlakeFile))
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir);
@ -368,19 +370,17 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe
// Get the `NonFlake` corresponding to a `FlakeRef`.
NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias, bool impureIsAllowed = false)
{
SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed);
auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed);
debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string());
FlakeRef resolvedRef = sourceInfo.resolvedRef;
NonFlake nonFlake(flakeRef, sourceInfo);
state.store->assertStorePath(nonFlake.storePath);
state.store->assertStorePath(nonFlake.sourceInfo.storePath);
if (state.allowedPaths)
state.allowedPaths->insert(nonFlake.storePath);
nonFlake.hash = state.store->queryPathInfo(sourceInfo.storePath)->narHash;
state.allowedPaths->insert(nonFlake.sourceInfo.storePath);
nonFlake.alias = alias;
@ -397,13 +397,17 @@ LockFile entryToLockFile(const LockFile::FlakeEntry & entry)
LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake)
{
LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.hash);
LockFile::FlakeEntry entry(
resolvedFlake.flake.sourceInfo.resolvedRef,
resolvedFlake.flake.sourceInfo.narHash);
for (auto & info : resolvedFlake.flakeDeps)
entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second));
for (auto & nonFlake : resolvedFlake.nonFlakeDeps) {
LockFile::NonFlakeEntry nonEntry(nonFlake.resolvedRef, nonFlake.hash);
LockFile::NonFlakeEntry nonEntry(
nonFlake.sourceInfo.resolvedRef,
nonFlake.sourceInfo.narHash);
entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry);
}
@ -443,7 +447,7 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake
auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first);
if (i != lockFile.nonFlakeEntries.end()) {
NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first);
if (nonFlake.hash != i->second.contentHash)
if (nonFlake.sourceInfo.narHash != i->second.narHash)
throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string());
deps.nonFlakeDeps.push_back(nonFlake);
} else {
@ -457,7 +461,7 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake
auto i = lockFile.flakeEntries.find(newFlakeRef);
if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible
ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second));
if (newResFlake.flake.hash != i->second.contentHash)
if (newResFlake.flake.sourceInfo.narHash != i->second.narHash)
throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string());
deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake);
} else {
@ -480,7 +484,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc
if (!recreateLockFile (handleLockFile)) {
// If recreateLockFile, start with an empty lockfile
oldLockFile = readLockFile(flake.storePath + "/flake.lock"); // FIXME: symlink attack
oldLockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack
}
LockFile lockFile(oldLockFile);
@ -510,6 +514,23 @@ void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateL
resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile);
}
static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs)
{
auto & path = sourceInfo.storePath;
state.store->isValidPath(path);
mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path});
if (sourceInfo.resolvedRef.rev) {
mkString(*state.allocAttr(vAttrs, state.symbols.create("rev")),
sourceInfo.resolvedRef.rev->gitRev());
mkString(*state.allocAttr(vAttrs, state.symbols.create("shortRev")),
sourceInfo.resolvedRef.rev->gitShortRev());
}
if (sourceInfo.revCount)
mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount);
}
void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)
{
// Construct the resulting attrset '{description, provides,
@ -525,29 +546,18 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v)
for (const NonFlake nonFlake : resFlake.nonFlakeDeps) {
auto vNonFlake = state.allocAttr(v, nonFlake.alias);
state.mkAttrs(*vNonFlake, 4);
state.mkAttrs(*vNonFlake, 8);
state.store->isValidPath(nonFlake.storePath);
mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.storePath, {nonFlake.storePath});
state.store->isValidPath(nonFlake.sourceInfo.storePath);
mkString(*state.allocAttr(*vNonFlake, state.sOutPath),
nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath});
// FIXME: add rev, shortRev, revCount, ...
emitSourceInfoAttrs(state, nonFlake.sourceInfo, *vNonFlake);
}
mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description);
auto & path = resFlake.flake.storePath;
state.store->isValidPath(path);
mkString(*state.allocAttr(v, state.sOutPath), path, {path});
if (resFlake.flake.resolvedRef.rev) {
mkString(*state.allocAttr(v, state.symbols.create("rev")),
resFlake.flake.resolvedRef.rev->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")),
resFlake.flake.resolvedRef.rev->gitShortRev());
}
if (resFlake.flake.revCount)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.revCount);
emitSourceInfoAttrs(state, resFlake.flake.sourceInfo, v);
auto vProvides = state.allocAttr(v, state.symbols.create("provides"));
mkApp(*vProvides, *resFlake.flake.vProvides, v);

View file

@ -22,28 +22,28 @@ struct LockFile
struct NonFlakeEntry
{
FlakeRef ref;
Hash contentHash;
NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), contentHash(hash) {};
Hash narHash;
NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {};
bool operator ==(const NonFlakeEntry & other) const
{
return ref == other.ref && contentHash == other.contentHash;
return ref == other.ref && narHash == other.narHash;
}
};
struct FlakeEntry
{
FlakeRef ref;
Hash contentHash;
Hash narHash;
std::map<FlakeRef, FlakeEntry> flakeEntries;
std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries;
FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), contentHash(hash) {};
FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {};
bool operator ==(const FlakeEntry & other) const
{
return
ref == other.ref
&& contentHash == other.contentHash
&& narHash == other.narHash
&& flakeEntries == other.flakeEntries
&& nonFlakeEntries == other.nonFlakeEntries;
}
@ -84,7 +84,7 @@ struct SourceInfo
FlakeRef resolvedRef;
Path storePath;
std::optional<uint64_t> revCount;
// date
Hash narHash; // store path hash
SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {};
};
@ -92,32 +92,24 @@ struct Flake
{
FlakeId id;
FlakeRef originalRef;
FlakeRef resolvedRef;
std::string description;
std::optional<uint64_t> revCount;
Path storePath;
Hash hash; // content hash
SourceInfo sourceInfo;
std::vector<FlakeRef> requires;
std::map<FlakeAlias, FlakeRef> nonFlakeRequires;
Value * vProvides; // FIXME: gc
// date
unsigned int epoch;
Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef),
resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {};
Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo)
: originalRef(origRef), sourceInfo(sourceInfo) {};
};
struct NonFlake
{
FlakeAlias alias;
FlakeRef originalRef;
FlakeRef resolvedRef;
std::optional<uint64_t> revCount;
Hash hash;
Path storePath;
// date
NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef),
resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {};
SourceInfo sourceInfo;
NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo)
: originalRef(origRef), sourceInfo(sourceInfo) {};
};
Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed);

View file

@ -70,64 +70,64 @@ struct CmdFlakeList : EvalCommand
}
};
void printFlakeInfo(const Flake & flake, bool json) {
if (json) {
nlohmann::json j;
j["id"] = flake.id;
j["uri"] = flake.resolvedRef.to_string();
j["description"] = flake.description;
if (flake.resolvedRef.ref)
j["branch"] = *flake.resolvedRef.ref;
if (flake.resolvedRef.rev)
j["revision"] = flake.resolvedRef.rev->to_string(Base16, false);
if (flake.revCount)
j["revCount"] = *flake.revCount;
j["path"] = flake.storePath;
j["epoch"] = flake.epoch;
std::cout << j.dump(4) << std::endl;
} else {
std::cout << "ID: " << flake.id << "\n";
std::cout << "URI: " << flake.resolvedRef.to_string() << "\n";
std::cout << "Description: " << flake.description << "\n";
if (flake.resolvedRef.ref)
std::cout << "Branch: " << *flake.resolvedRef.ref << "\n";
if (flake.resolvedRef.rev)
std::cout << "Revision: " << flake.resolvedRef.rev->to_string(Base16, false) << "\n";
if (flake.revCount)
std::cout << "Revcount: " << *flake.revCount << "\n";
std::cout << "Path: " << flake.storePath << "\n";
std::cout << "Epoch: " << flake.epoch << "\n";
}
static void printSourceInfo(const SourceInfo & sourceInfo)
{
std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string());
if (sourceInfo.resolvedRef.ref)
std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref);
if (sourceInfo.resolvedRef.rev)
std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false));
if (sourceInfo.revCount)
std::cout << fmt("Revcount: %s\n", *sourceInfo.revCount);
std::cout << fmt("Path: %s\n", sourceInfo.storePath);
}
void printNonFlakeInfo(const NonFlake & nonFlake, bool json) {
if (json) {
nlohmann::json j;
j["id"] = nonFlake.alias;
j["uri"] = nonFlake.resolvedRef.to_string();
if (nonFlake.resolvedRef.ref)
j["branch"] = *nonFlake.resolvedRef.ref;
if (nonFlake.resolvedRef.rev)
j["revision"] = nonFlake.resolvedRef.rev->to_string(Base16, false);
if (nonFlake.revCount)
j["revCount"] = *nonFlake.revCount;
j["path"] = nonFlake.storePath;
std::cout << j.dump(4) << std::endl;
} else {
std::cout << "ID: " << nonFlake.alias << "\n";
std::cout << "URI: " << nonFlake.resolvedRef.to_string() << "\n";
if (nonFlake.resolvedRef.ref)
std::cout << "Branch: " << *nonFlake.resolvedRef.ref;
if (nonFlake.resolvedRef.rev)
std::cout << "Revision: " << nonFlake.resolvedRef.rev->to_string(Base16, false) << "\n";
if (nonFlake.revCount)
std::cout << "Revcount: " << *nonFlake.revCount << "\n";
std::cout << "Path: " << nonFlake.storePath << "\n";
}
static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j)
{
j["uri"] = sourceInfo.resolvedRef.to_string();
if (sourceInfo.resolvedRef.ref)
j["branch"] = *sourceInfo.resolvedRef.ref;
if (sourceInfo.resolvedRef.rev)
j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false);
if (sourceInfo.revCount)
j["revCount"] = *sourceInfo.revCount;
j["path"] = sourceInfo.storePath;
}
static void printFlakeInfo(const Flake & flake)
{
std::cout << fmt("ID: %s\n", flake.id);
std::cout << fmt("Description: %s\n", flake.description);
std::cout << fmt("Epoch: %s\n", flake.epoch);
printSourceInfo(flake.sourceInfo);
}
static nlohmann::json flakeToJson(const Flake & flake)
{
nlohmann::json j;
j["id"] = flake.id;
j["description"] = flake.description;
j["epoch"] = flake.epoch;
sourceInfoToJson(flake.sourceInfo, j);
return j;
}
static void printNonFlakeInfo(const NonFlake & nonFlake)
{
std::cout << fmt("ID: %s\n", nonFlake.alias);
printSourceInfo(nonFlake.sourceInfo);
}
static nlohmann::json nonFlakeToJson(const NonFlake & nonFlake)
{
nlohmann::json j;
j["id"] = nonFlake.alias;
sourceInfoToJson(nonFlake.sourceInfo, j);
return j;
}
// FIXME: merge info CmdFlakeInfo?
struct CmdFlakeDeps : FlakeCommand, MixJSON
struct CmdFlakeDeps : FlakeCommand
{
std::string name() override
{
@ -147,15 +147,17 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON
std::queue<ResolvedFlake> todo;
todo.push(resolveFlake());
stopProgressBar();
while (!todo.empty()) {
auto resFlake = std::move(todo.front());
todo.pop();
for (auto & nonFlake : resFlake.nonFlakeDeps)
printNonFlakeInfo(nonFlake, json);
printNonFlakeInfo(nonFlake);
for (auto & info : resFlake.flakeDeps) {
printFlakeInfo(info.second.flake, json);
printFlakeInfo(info.second.flake);
todo.push(info.second);
}
}
@ -204,7 +206,11 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON
void run(nix::ref<nix::Store> store) override
{
auto flake = getFlake();
printFlakeInfo(flake, json);
stopProgressBar();
if (json)
std::cout << flakeToJson(flake).dump() << std::endl;
else
printFlakeInfo(flake);
}
};
@ -295,13 +301,13 @@ struct CmdFlakePin : virtual Args, EvalCommand
FlakeRegistry userRegistry = *readRegistry(userRegistryPath);
auto it = userRegistry.entries.find(FlakeRef(alias));
if (it != userRegistry.entries.end()) {
it->second = getFlake(*evalState, it->second, true).resolvedRef;
it->second = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef;
writeRegistry(userRegistry, userRegistryPath);
} else {
std::shared_ptr<FlakeRegistry> globalReg = evalState->getGlobalFlakeRegistry();
it = globalReg->entries.find(FlakeRef(alias));
if (it != globalReg->entries.end()) {
FlakeRef newRef = getFlake(*evalState, it->second, true).resolvedRef;
auto newRef = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef;
userRegistry.entries.insert_or_assign(alias, newRef);
writeRegistry(userRegistry, userRegistryPath);
} else

View file

@ -176,8 +176,8 @@ void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const
while (!queue.empty()) {
const ResolvedFlake & flake = queue.front();
queue.pop();
if (!std::get_if<FlakeRef::IsPath>(&flake.flake.resolvedRef.data))
closure.insert(flake.flake.storePath);
if (!std::get_if<FlakeRef::IsPath>(&flake.flake.sourceInfo.resolvedRef.data))
closure.insert(flake.flake.sourceInfo.storePath);
for (const auto & dep : flake.flakeDeps)
queue.push(dep.second);
}