From 1722ae6ecee54e14164d215ba3d767ea6c352fc3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jun 2020 05:41:18 +0000 Subject: [PATCH 01/32] Pull out PathReferences super class --- src/libstore/path-info.hh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index f5dee00a6..a980e1243 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -20,12 +20,17 @@ namespace nix { class Store; -struct ValidPathInfo +template +struct PathReferences +{ + std::set references; +}; + +struct ValidPathInfo : PathReferences { StorePath path; std::optional deriver; Hash narHash; - StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only From 71e4c9c505f2418084643c1a68da5c89b82038dd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jun 2020 22:46:27 +0000 Subject: [PATCH 02/32] WIP: store separate `hasValidPath` bool --- src/libstore/binary-cache-store.cc | 3 +- src/libstore/build.cc | 14 ++++---- src/libstore/daemon.cc | 10 +++--- src/libstore/export-import.cc | 4 +-- src/libstore/legacy-ssh-store.cc | 7 ++-- src/libstore/local-store.cc | 23 +++++++------ src/libstore/misc.cc | 5 ++- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/path-info.hh | 52 +++++++++++++++++++++++++++++ src/libstore/remote-store.cc | 11 +++--- src/libstore/store-api.cc | 29 ++++++++++------ src/libstore/store-api.hh | 11 ------ src/nix-store/dotgraph.cc | 2 +- src/nix-store/graphml.cc | 2 +- src/nix-store/nix-store.cc | 8 ++--- src/nix/make-content-addressable.cc | 2 +- src/nix/sigs.cc | 3 +- 17 files changed, 119 insertions(+), 69 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..7167ec900 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -125,8 +125,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource reads, but typically they'll already be cached. */ for (auto & ref : info.references) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0c25897f8..c01b2ddaf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3834,7 +3834,7 @@ void DerivationGoal::registerOutputs() ValidPathInfo info(worker.store.parseStorePath(path)); info.narHash = hash.first; info.narSize = hash.second; - info.references = std::move(references); + info.setReferencesPossiblyToSelf(std::move(references)); info.deriver = drvPath; info.ultimate = true; info.ca = ca; @@ -3963,12 +3963,12 @@ void DerivationGoal::checkOutputs(const std::map & outputs) auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -3997,7 +3997,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) auto used = recursive ? getClosure(info.path).first - : info.references; + : info.referencesPossiblyToSelf(); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); @@ -4466,8 +4466,7 @@ void SubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); + addWaitee(worker.makeSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -4487,8 +4486,7 @@ void SubstitutionGoal::referencesValid() } for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + assert(worker.store.isValidPath(i)); state = &SubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 842aef20c..c6b70f6f3 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -326,7 +326,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -601,7 +601,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - writeStorePaths(*store, to, i->second.references); + writeStorePaths(*store, to, i->second.referencesPossiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -618,7 +618,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - writeStorePaths(*store, to, i.second.references); + writeStorePaths(*store, to, i.second.referencesPossiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -647,7 +647,7 @@ static void performOp(TunnelLogger * logger, ref store, to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") << info->narHash.to_string(Base16, false); - writeStorePaths(*store, to, info->references); + writeStorePaths(*store, to, info->referencesPossiblyToSelf()); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate @@ -707,7 +707,7 @@ static void performOp(TunnelLogger * logger, ref store, if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths(*store, from); + info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..5a5c76f7f 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -62,7 +62,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) hashAndWriteSink << exportMagic << printStorePath(path); - writeStorePaths(*this, hashAndWriteSink, info->references); + writeStorePaths(*this, hashAndWriteSink, info->referencesPossiblyToSelf()); hashAndWriteSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -88,7 +88,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths(*this, source); + info.setReferencesPossiblyToSelf(readStorePaths(*this, source)); auto deriver = readString(source); if (deriver != "") diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5657aa593..f01e642a0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -103,11 +103,10 @@ struct LegacySSHStore : public Store auto info = std::make_shared(parseStorePath(p)); assert(path == info->path); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = readStorePaths(*this, conn->from); + info->setReferencesPossiblyToSelf(readStorePaths(*this, conn->from)); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -140,7 +139,7 @@ struct LegacySSHStore : public Store << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -169,7 +168,7 @@ struct LegacySSHStore : public Store conn->to << exportMagic << printStorePath(info.path); - writeStorePaths(*this, conn->to, info.references); + writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0dfbed9fc..02de3aa5e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -671,8 +671,10 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + while (useQueryReferences.next()) { + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); + } return info; })); @@ -856,11 +858,13 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); - infos.insert_or_assign(path, SubstitutablePathInfo{ - info->deriver, + infos.insert_or_assign(path, SubstitutablePathInfo { info->references, + info->hasSelfReference, + info->deriver, narInfo ? narInfo->fileSize : 0, - info->narSize}); + info->narSize, + }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -907,7 +911,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -986,14 +990,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); // text hashing has long been allowed to have non-self-references because it is used for drv files. - bool refersToSelf = info.references.count(info.path) > 0; - if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && !refersToSelf)) + if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && info.hasSelfReference)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr hashSink; - if (!info.ca.has_value() || !info.references.count(info.path)) + if (!info.ca.has_value() || !info.hasSelfReference) hashSink = std::make_unique(htSHA256); else hashSink = std::make_unique(htSHA256, std::string(info.path.hashPart())); @@ -1254,7 +1257,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr hashSink; - if (!info->ca || !info->references.count(info->path)) + if (!info->ca || !info->hasSelfReference) hashSink = std::make_unique(*info->narHash.type); else hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e68edb38c..5214a7bf4 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -61,8 +61,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths, } else { for (auto & ref : info->references) - if (ref != path) - enqueue(printStorePath(ref)); + enqueue(printStorePath(ref)); if (includeOutputs && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) @@ -268,7 +267,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) for (auto & i : references) /* Don't traverse into paths that don't exist. That can happen due to substitutes for non-existent paths. */ - if (i != path && paths.count(i)) + if (paths.count(i)) dfsVisit(i, &path); sorted.push_back(path); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 012dea6ea..c543f6ea2 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -198,7 +198,7 @@ public: narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a980e1243..27efe5ae9 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -24,8 +24,43 @@ template struct PathReferences { std::set references; + bool hasSelfReference = false; + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); }; +template +StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet references { references }; + if (hasSelfReference) + references.insert(self); + return references; +} + +template +void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} + +template +void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + struct ValidPathInfo : PathReferences { StorePath path; @@ -64,6 +99,7 @@ struct ValidPathInfo : PathReferences return path == i.path && narHash == i.narHash + && hasSelfReference == i.hasSelfReference && references == i.references; } @@ -80,6 +116,12 @@ struct ValidPathInfo : PathReferences /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; + void insertReferencePossiblyToSelf(StorePath && ref); + void setReferencesPossiblyToSelf(StorePathSet && refs); + static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were @@ -101,4 +143,14 @@ struct ValidPathInfo : PathReferences }; typedef list ValidPathInfos; + + +struct SubstitutablePathInfo : PathReferences +{ + std::optional deriver; + unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ +}; + +typedef std::map SubstitutablePathInfos; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b7cc7a5fc..f84b62f2e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -326,7 +326,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.setReferencesPossiblyToSelf(i, readStorePaths(*this, conn->from)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i, std::move(info)); @@ -339,11 +339,12 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.setReferencesPossiblyToSelf(path, readStorePaths(*this, conn->from)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -376,7 +377,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths(*this, conn->from); + info->setReferencesPossiblyToSelf(readStorePaths(*this, conn->from)); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -455,7 +456,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = readStorePaths(*this, conn->from); - assert(importedPaths.size() <= 1); + assert(importedPaths.empty() == 0); // doesn't include possible self reference } else { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e4a4ae11e..95b1c1c3b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -715,7 +715,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!string2Int(s, n)) throw Error("number expected"); while (n--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -738,6 +738,20 @@ string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return PathReferences::referencesPossiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); +} std::string ValidPathInfo::fingerprint(const Store & store) const { @@ -748,7 +762,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -767,16 +781,11 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const auto caPath = std::visit(overloaded { [&](TextHash th) { + assert(!hasSelfReference); return store.makeTextPath(path.name(), th.hash, references); }, [&](FixedOutputHash fsh) { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); + return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference); } }, *ca); @@ -810,7 +819,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 00b9c385c..420ffebbe 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -102,17 +102,6 @@ struct GCResults }; -struct SubstitutablePathInfo -{ - std::optional deriver; - StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ -}; - -typedef std::map SubstitutablePathInfos; - - enum BuildMode { bmNormal, bmRepair, bmCheck }; diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 8b699f39b..45abe0405 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -58,7 +58,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 8ca5c9c8d..1cd974e41 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7d81bf54f..c4ca89c85 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -245,7 +245,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -328,7 +328,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -859,7 +859,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - writeStorePaths(*store, out, info->references); + writeStorePaths(*store, out, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -949,7 +949,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(in), htSHA256); - info.references = readStorePaths(*store, in); + info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index fb36fc410..5267948ee 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -79,7 +79,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); + info.hasSelfReference = std::move(hasSelfReference); info.narHash = narHash; info.narSize = sink.s->size(); info.ca = FixedOutputHash { diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 6c9b9a792..a40975982 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -65,7 +65,8 @@ struct CmdCopySigs : StorePathsCommand binary. */ if (info->narHash != info2->narHash || info->narSize != info2->narSize || - info->references != info2->references) + info->references != info2->references || + info->hasSelfReference != info2->hasSelfReference) continue; for (auto & sig : info2->sigs) From a9c0ea30bf81a42dfb7ccca04bb98649a6c34d07 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 17:59:27 +0000 Subject: [PATCH 03/32] Backport fix from #3754 branch --- src/libstore/path-info.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 27efe5ae9..a67c36bb6 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -36,10 +36,10 @@ struct PathReferences template StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const { - StorePathSet references { references }; + StorePathSet refs { references }; if (hasSelfReference) - references.insert(self); - return references; + refs.insert(self); + return refs; } template From 70ed47c1cb9d04a5a350cac664921a194d93d329 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 19:21:46 +0000 Subject: [PATCH 04/32] Fix some things in remote store --- src/libstore/remote-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f84b62f2e..912a3b70c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -445,7 +445,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - writeStorePaths(*this, sink, info.references); + writeStorePaths(*this, sink, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -464,7 +464,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; From 66834068432d316ee558717765851835ceec2dcc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 19:58:31 +0000 Subject: [PATCH 05/32] Fix nar info parsing --- src/libstore/nar-info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ef04bc859..0796de466 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -57,7 +57,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") From e61061c88e0dfcce9329ea9f0b041a35270dfa1a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 23:17:11 +0000 Subject: [PATCH 06/32] Remove stray tabs --- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d7aef3ea5..273455bae 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -386,7 +386,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - auto path = parseStorePath(readString(conn->from)); + auto path = parseStorePath(readString(conn->from)); SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1a28386ef..0fee5559f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -913,17 +913,17 @@ string showPaths(const PathSet & paths) StorePathSet ValidPathInfo::referencesPossiblyToSelf() const { - return PathReferences::referencesPossiblyToSelf(path); + return PathReferences::referencesPossiblyToSelf(path); } void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) { - return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); + return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); } void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) { - return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); + return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); } std::string ValidPathInfo::fingerprint(const Store & store) const From f8d562c0a7cef27c65d3cff96ad8ef384f05b331 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 7 Oct 2020 13:52:20 +0000 Subject: [PATCH 07/32] Use PathReferences more widely --- perl/lib/Nix/Store.xs | 10 +- src/libexpr/primops.cc | 16 ++- src/libexpr/value-to-xml.hh | 2 +- src/libfetchers/fetchers.cc | 8 +- src/libfetchers/tarball.cc | 16 ++- src/libstore/binary-cache-store.cc | 43 ++++-- src/libstore/build.cc | 30 ++-- src/libstore/content-address.cc | 24 +++- src/libstore/content-address.hh | 102 +++++++++++++- src/libstore/derivations.cc | 4 +- src/libstore/derivations.hh | 4 +- src/libstore/local-store.cc | 30 +++- src/libstore/nar-info.hh | 3 + src/libstore/path-info.hh | 47 +------ src/libstore/path.hh | 5 +- src/libstore/store-api.cc | 171 +++++++++++++++-------- src/libstore/store-api.hh | 14 +- src/libutil/args.cc | 2 +- src/libutil/error.hh | 3 +- src/libutil/fmt.hh | 2 +- src/libutil/tests/logging.cc | 2 +- src/libutil/types.hh | 1 + src/nix-prefetch-url/nix-prefetch-url.cc | 10 +- src/nix-store/nix-store.cc | 12 +- src/nix/add-to-store.cc | 16 ++- src/nix/bundle.cc | 2 +- src/nix/make-content-addressable.cc | 38 ++--- src/nix/profile.cc | 13 +- src/nix/verify.cc | 6 +- 29 files changed, 431 insertions(+), 205 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 599921151..ea8bbaf34 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -111,7 +111,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * arr = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { @@ -287,7 +287,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2b304aab0..c74b67658 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1045,7 +1045,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + { + .method = ingestionMethod, + .hash = h, + }, + {}, + }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputCAFixed { @@ -1764,7 +1770,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..c5f327bd8 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 49851f7bc..67bb77d3e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -198,7 +198,13 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for mutable input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); + return store.makeFixedOutputPath("source", FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + {}, + }); } std::string Input::getType() const diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ca49482a9..b3ee84810 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -71,14 +71,20 @@ DownloadFileResult downloadFile( dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + { + .name = name, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + {}, + }, + }, hashString(htSHA256, *sink.s), }; info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index d592f16dd..2d92e1c50 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -322,7 +322,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; @@ -412,14 +422,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -428,17 +444,26 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { textHash, references }); if (!repair && isValidPath(path)) return path; auto source = StringSource { s }; return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + { + .name = name, + .info = TextInfo { + { .hash = textHash }, + references, + }, + }, + nar.first, + }; info.narSize = nar.second; info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 97a832c6b..12ce6f2ec 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4056,25 +4056,24 @@ void DerivationGoal::registerOutputs() break; } auto got = caSink.finish().first; - auto refs = rewriteRefs(); HashModuloSink narSink { htSHA256, oldHashPart }; dumpPath(actualPath, narSink); auto narHashAndSize = narSink.finish(); ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.references, - refs.hasSelfReference), + worker.store, + { + .name = outputPathName(drv->name, outputName), + .info = FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, + }, + rewriteRefs(), + }, + }, narHashAndSize.first, }; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - static_cast &>(newInfo0) = refs; assert(newInfo0.ca); return newInfo0; @@ -4861,7 +4860,10 @@ void SubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA({ + .name = std::string { storePath.name() }, + .info = caWithoutRefs(*ca), + }); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -4891,7 +4893,7 @@ void SubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { + if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { auto info2 = std::make_shared(*info); info2->path = storePath; info = info2; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 90a3ad1f5..d68c60f4f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,6 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } + std::string makeFileIngestionPrefix(const FileIngestionMethod m) { switch (m) { @@ -16,9 +17,8 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; - default: - throw Error("impossible, caught both cases"); } + assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) @@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { - return "text:" + th.hash.to_string(Base32, true); + return "text:" + + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } @@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca) }, [](FixedOutputHash fsh) { return fsh.hash; - } + }, + }, ca); +} + +ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { + return std::visit(overloaded { + [&](TextHash h) -> ContentAddressWithReferences { + return TextInfo { h, {}}; + }, + [&](FixedOutputHash h) -> ContentAddressWithReferences { + return FixedOutputInfo { h, {}}; + }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..e15d76bd7 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -2,14 +2,20 @@ #include #include "hash.hh" +#include "path.hh" namespace nix { +/* + * Mini content address + */ + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; + struct TextHash { Hash hash; }; @@ -41,10 +47,6 @@ typedef std::variant< ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional ca); @@ -74,4 +76,96 @@ ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); std::string renderContentAddressMethod(ContentAddressMethod caMethod); +/* + * References set + */ + +template +struct PathReferences +{ + std::set references; + bool hasSelfReference = false; + + bool operator == (const PathReferences & other) const + { + return references == other.references + && hasSelfReference == other.hasSelfReference; + } + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); +}; + +template +StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet refs { references }; + if (hasSelfReference) + refs.insert(self); + return refs; +} + +template +void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} + +template +void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +// This matches the additional info that we need for makeTextPath +struct TextInfo : TextHash { + // References for the paths, self references disallowed + StorePathSet references; +}; + +struct FixedOutputInfo : FixedOutputHash { + // References for the paths + PathReferences references; +}; + +typedef std::variant< + TextInfo, + FixedOutputInfo +> ContentAddressWithReferences; + +ContentAddressWithReferences caWithoutRefs(const ContentAddress &); + +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + bool operator == (const StorePathDescriptor & other) const + { + return name == other.name; + // FIXME second field + } + + bool operator < (const StorePathDescriptor & other) const + { + return name < other.name; + // FIXME second field + } +}; + } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 07b4e772b..925a78083 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -27,8 +27,8 @@ std::optional DerivationOutput::path(const Store & store, std::string StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + outputPathName(drvName, outputName), + { hash, {} }); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d48266774..be19aa300 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -130,8 +130,8 @@ struct Derivation : BasicDerivation /* Return the underlying basic derivation but with these changes: - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. + 1. Input drvs are emptied, but the outputs of them that were used are + added directly to input sources. 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 18545f659..e6b02cce6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(doia.path, i.first); }, [&](DerivationOutputCAFixed dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} }); envHasRightPath(path, i.first); }, [&](DerivationOutputCAFloating _) { @@ -923,7 +923,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // recompute store path so that we can use a different store root if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + subPath = makeFixedOutputPathFromCA({ + .name = std::string { path.first.name() }, + .info = caWithoutRefs(*path.second), + }); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1164,7 +1167,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto desc = StorePathDescriptor { + name, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); @@ -1184,7 +1198,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1209,9 +1223,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, optimisePath(realPath); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1226,7 +1239,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 39ced76e5..fd37b85db 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 509f100d7..8c4791ac0 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -13,47 +13,6 @@ namespace nix { class Store; -template -struct PathReferences -{ - std::set references; - bool hasSelfReference = false; - - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; - void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); - void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); -}; - -template -StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const -{ - StorePathSet refs { references }; - if (hasSelfReference) - refs.insert(self); - return refs; -} - -template -void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) -{ - if (ref == self) - hasSelfReference = true; - else - references.insert(std::move(ref)); -} - -template -void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) -{ - if (refs.count(self)) - hasSelfReference = true; - refs.erase(self); - - references = refs; -} - struct SubstitutablePathInfo : PathReferences { @@ -68,7 +27,6 @@ struct ValidPathInfo : PathReferences { StorePath path; std::optional deriver; - // TODO document this Hash narHash; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown @@ -117,6 +75,8 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); + std::optional fullStorePathDescriptorOpt() const; + /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -143,6 +103,9 @@ struct ValidPathInfo : PathReferences ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + StorePathDescriptor && ca, Hash narHash); + virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index b03a0f69d..5f239ceb6 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once -#include "content-address.hh" +#include + #include "types.hh" namespace nix { @@ -64,8 +65,6 @@ typedef std::set StorePathSet; typedef std::vector StorePaths; typedef std::map OutputPathMap; -typedef std::map> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7041edbe5..5d63b8e3c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "thread-pool.hh" #include "json.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" @@ -163,63 +164,61 @@ StorePath Store::makeOutputPath(std::string_view id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ static std::string makeType( const Store & store, string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const PathReferences & references) { - for (auto & i : references) { + for (auto & i : references.references) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.hasSelfReference) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.empty()); + assert(info.references.references.size() == 0); + assert(!info.references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", PathReferences { info.references }), + info.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { - [&](TextHash th) { - return makeTextPath(name, th.hash, references); + [&](TextInfo ti) { + return makeTextPath(desc.name, ti); }, - [&](FixedOutputHash fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](FixedOutputInfo foi) { + return makeFixedOutputPath(desc.name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, desc.info); } @@ -229,14 +228,24 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + { + .method = method, + .hash = h, + }, + {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -326,11 +335,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -496,7 +514,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -509,11 +527,9 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref(info)); @@ -536,13 +552,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -554,7 +570,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -737,7 +753,8 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -799,7 +816,8 @@ std::map copyPaths(ref srcStore, ref dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -826,7 +844,8 @@ std::map copyPaths(ref srcStore, ref dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -947,19 +966,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const +{ + if (! ca) + return std::nullopt; + + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](TextHash th) { + TextInfo info { th }; + assert(!hasSelfReference); + info.references = references; + return ContentAddressWithReferences { info }; + }, + [&](FixedOutputHash foh) { + FixedOutputInfo info { foh }; + info.references = static_cast>(*this); + return ContentAddressWithReferences { info }; + }, + }, *ca), + }; +} + bool ValidPathInfo::isContentAddressed(const Store & store) const { - if (! ca) return false; + auto fullCaOpt = fullStorePathDescriptorOpt(); - auto caPath = std::visit(overloaded { - [&](TextHash th) { - assert(!hasSelfReference); - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference); - } - }, *ca); + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -997,6 +1034,26 @@ Strings ValidPathInfo::shortRefs() const } +ValidPathInfo::ValidPathInfo( + const Store & store, + StorePathDescriptor && info, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(info)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo ti) { + this->references = ti.references; + this->ca = TextHash { std::move(ti) }; + }, + [this](FixedOutputInfo foi) { + *(static_cast *>(this)) = foi.references; + this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + }, + }, std::move(info.info)); +} + + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 854446987..e6a6053a3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -170,6 +170,8 @@ struct BuildResult } }; +typedef std::map> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; @@ -313,17 +315,11 @@ public: StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 147602415..453fe60f9 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -244,7 +244,7 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f3babcbde..260ed3cf8 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -201,9 +201,8 @@ public: template SysError(const Args & ... args) - : Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 6e69bdce2..11dbef9db 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -103,7 +103,7 @@ class hintformat public: hintformat(const string &format) :fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 7e53f17c6..d33bd7c1f 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -370,7 +370,7 @@ namespace nix { // constructing without access violation. ErrPos ep(invalid); - + // assignment without access violation. ep = invalid; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 55d02bcf9..6c4c5ab74 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 377ae03a8..99cc0cdec 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -161,8 +161,14 @@ static int _main(int argc, char * * argv) std::optional storePath; if (args.size() == 2) { expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); if (store->isValidPath(*storePath)) hash = *expectedHash; else diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9092dbd80..7981bbbdd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -195,10 +195,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -209,7 +209,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + {}, + }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 023ffa4ed..86616d66b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,14 +69,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + StorePathDescriptor { + .name = *namePart, + .info = FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + {}, + }, + }, narHash, }; info.narSize = sink.s->size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fc41da9e4..510df7504 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -91,7 +91,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 7737f6d91..7695c98f8 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + PathReferences refs; + refs.hasSelfReference = oldInfo->hasSelfReference; for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + refs.references.insert(std::move(replacement)); } *sink.s = rewriteStrings(*sink.s, rewrites); @@ -78,16 +74,20 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + *store, + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + std::move(refs), + }, + }, narHash, }; - info.references = std::move(references); - info.hasSelfReference = std::move(hasSelfReference); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 7ce4dfe4c..41a4857fc 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -130,12 +130,21 @@ struct ProfileManifest auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + StorePathDescriptor { + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + { references }, + }, + }, narHash, }; info.references = std::move(references); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; auto source = StringSource { *sink.s }; store->addToStore(info, source); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 26f755fd9..d189a2fd3 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); From 39c11c5c01de9c18bf1b0bc3928fc28393fd0ca9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 13 Oct 2020 03:30:14 +0000 Subject: [PATCH 08/32] Organize content-address.hh a bit better --- src/libstore/content-address.hh | 53 ++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index e15d76bd7..126244ab5 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -7,7 +7,7 @@ namespace nix { /* - * Mini content address + * Content addressing method */ enum struct FileIngestionMethod : uint8_t { @@ -15,6 +15,34 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; +/* + We only have one way to hash text with references, so this is single-value + type is only useful in std::variant. +*/ +struct TextHashMethod { }; + +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; + +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(const FileIngestionMethod m); + + +typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > ContentAddressMethod; + +ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + +std::string renderContentAddressMethod(ContentAddressMethod caMethod); + +/* + * Mini content address + */ struct TextHash { Hash hash; @@ -43,10 +71,6 @@ typedef std::variant< FixedOutputHash // for path computed by makeFixedOutputPath > ContentAddress; -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional ca); @@ -57,25 +81,6 @@ std::optional parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); -/* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; -}; - -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; - -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); - -std::string renderContentAddressMethod(ContentAddressMethod caMethod); - /* * References set */ From 10e81bf871551901ff0383bdede0f79325e93867 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 02:21:28 +0000 Subject: [PATCH 09/32] Fix conditions for ca-references --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e6b02cce6..36ef7acf7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1071,7 +1071,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); // text hashing has long been allowed to have non-self-references because it is used for drv files. - if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && info.hasSelfReference)) + if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && !info.hasSelfReference)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash From 2c21cb672043fcf3c3fd19f89618b37693c0dc62 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 22:40:40 +0000 Subject: [PATCH 10/32] Fill in missing comparison operators for content addresses --- src/libstore/content-address.hh | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 126244ab5..acdb4f023 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -3,6 +3,7 @@ #include #include "hash.hh" #include "path.hh" +#include "comparator.hh" namespace nix { @@ -46,6 +47,8 @@ std::string renderContentAddressMethod(ContentAddressMethod caMethod); struct TextHash { Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; /// Pair of a hash, and how the file system was ingested @@ -53,6 +56,8 @@ struct FixedOutputHash { FileIngestionMethod method; Hash hash; std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; /* @@ -91,17 +96,13 @@ struct PathReferences std::set references; bool hasSelfReference = false; - bool operator == (const PathReferences & other) const - { - return references == other.references - && hasSelfReference == other.hasSelfReference; - } - /* Functions to view references + hasSelfReference as one set, mainly for compatibility's sake. */ StorePathSet referencesPossiblyToSelf(const Ref & self) const; void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(PathReferences, me->references, me->hasSelfReference); }; template @@ -142,11 +143,15 @@ void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set struct TextInfo : TextHash { // References for the paths, self references disallowed StorePathSet references; + + GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); }; struct FixedOutputInfo : FixedOutputHash { // References for the paths PathReferences references; + + GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); }; typedef std::variant< @@ -160,17 +165,7 @@ struct StorePathDescriptor { std::string name; ContentAddressWithReferences info; - bool operator == (const StorePathDescriptor & other) const - { - return name == other.name; - // FIXME second field - } - - bool operator < (const StorePathDescriptor & other) const - { - return name < other.name; - // FIXME second field - } + GENERATE_CMP(StorePathDescriptor, me->name, me->info); }; } From 13c669105ca93d28ca1a78321f07fd4ddbb445b1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 22:25:21 +0000 Subject: [PATCH 11/32] Slight cleanups --- src/libstore/content-address.cc | 2 +- src/libstore/content-address.hh | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e4ba855d5..2e6c435ce 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -10,7 +10,7 @@ std::string FixedOutputHash::printMethodAlgo() const } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) +std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index acdb4f023..a275800f9 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -11,17 +11,16 @@ namespace nix { * Content addressing method */ +/* We only have one way to hash text with references, so this is a single-value + type, mainly useful with std::variant. +*/ +struct TextHashMethod : std::monostate { }; + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; -/* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; - struct FixedOutputHashMethod { FileIngestionMethod fileIngestionMethod; HashType hashType; @@ -29,9 +28,13 @@ struct FixedOutputHashMethod { /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); +std::string makeFileIngestionPrefix(FileIngestionMethod m); +/* Just the type of a content address. Combine with the hash itself, and we + have a `ContentAddress` as defined below. Combine that, in turn, with info + on references, and we have `ContentAddressWithReferences`, as defined + further below. */ typedef std::variant< TextHashMethod, FixedOutputHashMethod @@ -86,6 +89,7 @@ std::optional parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); + /* * References set */ From 8623143921f8683b88d46aaebe9f707e5b9a912b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 11:18:14 -0500 Subject: [PATCH 12/32] Make formatting consistent --- src/libstore/content-address.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index a275800f9..6be4be4c5 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -38,7 +38,7 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m); typedef std::variant< TextHashMethod, FixedOutputHashMethod - > ContentAddressMethod; +> ContentAddressMethod; ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); From 6a168254ce068c067259c913ee7d6ee2e0d1dc7e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 12:24:20 -0500 Subject: [PATCH 13/32] Use named field initialization for references --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 10 ++++++++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/store-api.cc | 22 +++++++++++---------- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 15 files changed, 34 insertions(+), 26 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9cb078660..3ccd3c722 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -299,7 +299,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - {}, + .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8e9e5630d..8a19eab8f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1249,7 +1249,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * .method = method, .hash = h, }, - {}, + .references = {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 4181f0b7d..560a086f0 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -235,7 +235,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - {} + .references = {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 735d9fc93..3936eadfe 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - {}, + .references = {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index d52d19797..155b86cc4 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -79,7 +79,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - {}, + .references = {}, }, }, hashString(htSHA256, sink.s), diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3bbf4c8ac..aac5e7b88 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - { + .references = { .references = references, .hasSelfReference = false, }, @@ -433,7 +433,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - { + .references = { .references = references, .hasSelfReference = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 110a6a301..3d8299bbf 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2482,7 +2482,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - rewriteRefs(), + .references = rewriteRefs(), }, }, Hash::dummy, diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 2e6c435ce..3b8a773b7 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -154,10 +154,16 @@ Hash getContentAddressHash(const ContentAddress & ca) ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { - return TextInfo { h, {}}; + return TextInfo { + h, + .references = {}, + }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { - return FixedOutputInfo { h, {}}; + return FixedOutputInfo { + h, + .references = {}, + }; }, }, ca); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a8f060768..9f3a6db24 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1426,7 +1426,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - { + .references = { .references = references, .hasSelfReference = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 9655a0555..d6b6e87c9 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -55,7 +55,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - std::move(refs), + .references = std::move(refs), }, }, Hash::dummy, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 592afebd8..4b89465e7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -232,7 +232,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - {}, + .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -442,7 +442,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - {}, + .references = {}, }, }, narHash, @@ -1270,16 +1270,18 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c return StorePathDescriptor { .name = std::string { path.name() }, .info = std::visit(overloaded { - [&](const TextHash & th) { - TextInfo info { th }; + [&](const TextHash & th) -> ContentAddressWithReferences { assert(!hasSelfReference); - info.references = references; - return ContentAddressWithReferences { info }; + return TextInfo { + th, + .references = references, + }; }, - [&](const FixedOutputHash & foh) { - FixedOutputInfo info { foh }; - info.references = static_cast>(*this); - return ContentAddressWithReferences { info }; + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + return FixedOutputInfo { + foh, + .references = static_cast>(*this), + }; }, }, *ca), }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 75fa08551..5cb5aa53a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - {}, + .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index c2494dc9f..0b58818c3 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -50,7 +50,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - {}, + .references = {}, }, }, narHash, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index aa302efc1..df9933d29 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - {}, + .references = {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 024849e3b..614a37eba 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -205,7 +205,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - { references }, + .references = { references }, }, }, narHash, From 9cfa78e58a92b4bf034867bc1296a200bdc3f12a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 12:26:15 -0500 Subject: [PATCH 14/32] Optimize `ValidPathInfo` construction a bit better --- src/libstore/store-api.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4b89465e7..cd48d616b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1340,13 +1340,13 @@ ValidPathInfo::ValidPathInfo( , narHash(narHash) { std::visit(overloaded { - [this](const TextInfo & ti) { - this->references = ti.references; - this->ca = TextHash { std::move(ti) }; + [this](TextInfo && ti) { + this->references = std::move(ti.references); + this->ca = std::move((TextHash &&) ti); }, - [this](const FixedOutputInfo & foi) { - *(static_cast *>(this)) = foi.references; - this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + [this](FixedOutputInfo && foi) { + *(static_cast *>(this)) = std::move(foi.references); + this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); } From 46e942ff9e65755689ee72f93846d7118e1b8d45 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 15:36:05 -0500 Subject: [PATCH 15/32] Do big rename to clean up code - `PathReferences` -> `References` - `PathReferences` -> `StoreReference` - `references` -> `others` - `hasSelfReference` -> `self` And get rid of silly subclassing --- src/libexpr/primops.cc | 7 ++- src/libstore/binary-cache-store.cc | 13 ++-- src/libstore/build/local-derivation-goal.cc | 14 ++--- src/libstore/build/substitution-goal.cc | 6 +- src/libstore/content-address.hh | 46 +------------- src/libstore/daemon.cc | 4 +- src/libstore/local-store.cc | 18 +++--- src/libstore/make-content-addressed.cc | 8 +-- src/libstore/misc.cc | 16 ++--- src/libstore/path-info.hh | 7 ++- src/libstore/remote-store.cc | 4 +- src/libstore/store-api.cc | 36 +++++------ src/libutil/reference-set.hh | 68 +++++++++++++++++++++ src/nix/profile.cc | 6 +- src/nix/sigs.cc | 3 +- src/nix/why-depends.cc | 2 +- 16 files changed, 146 insertions(+), 112 deletions(-) create mode 100644 src/libutil/reference-set.hh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a19eab8f..0113659d1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1535,7 +1535,8 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V StorePathSet refs; if (state.store->isInStore(path)) { try { - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + // FIXME: Are self references becoming non-self references OK? + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf(); } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1971,7 +1972,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->references; + refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } @@ -2010,7 +2011,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - {}, + .references = {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index aac5e7b88..aa5aafdbf 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,8 +180,9 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) + reads, but typically they'll already be cached. Note that + self-references are always valid. */ + for (auto & ref : info.references.others) try { queryPathInfo(ref); } catch (InvalidPath &) { @@ -314,8 +315,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .hash = nar.first, }, .references = { - .references = references, - .hasSelfReference = false, + .others = references, + .self = false, }, }, }, @@ -434,8 +435,8 @@ StorePath BinaryCacheStore::addToStore( .hash = h, }, .references = { - .references = references, - .hasSelfReference = false, + .others = references, + .self = false, }, }, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3d8299bbf..ff24bd088 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2421,26 +2421,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } }; - auto rewriteRefs = [&]() -> PathReferences { + auto rewriteRefs = [&]() -> StoreReferences { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - PathReferences res { - .hasSelfReference = false, + StoreReferences res { + .self = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; if (r == *scratchPath) { - res.hasSelfReference = true; + res.self = true; } else if (auto outputRewrite = get(outputRewrites, origHash)) { std::string newRef = *outputRewrite; newRef += '-'; newRef += name; - res.references.insert(StorePath { newRef }); + res.others.insert(StorePath { newRef }); } else { - res.references.insert(r); + res.others.insert(r); } } return res; @@ -2523,7 +2523,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - static_cast &>(newInfo0) = rewriteRefs(); + newInfo0.references = rewriteRefs(); return newInfo0; }, diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 307183505..36b0ea7a7 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -128,7 +128,7 @@ void PathSubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { + if (info->isContentAddressed(*sub) && info->references.empty()) { auto info2 = std::make_shared(*info); info2->path = storePath; info = info2; @@ -165,7 +165,7 @@ void PathSubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - for (auto & i : info->references) + for (auto & i : info->references.others) addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -187,7 +187,7 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references) + for (auto & i : info->references.others) assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 6be4be4c5..f8a4d5370 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,6 +4,7 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" +#include "reference-set.hh" namespace nix { @@ -94,48 +95,7 @@ Hash getContentAddressHash(const ContentAddress & ca); * References set */ -template -struct PathReferences -{ - std::set references; - bool hasSelfReference = false; - - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; - void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); - void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); - - GENERATE_CMP(PathReferences, me->references, me->hasSelfReference); -}; - -template -StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const -{ - StorePathSet refs { references }; - if (hasSelfReference) - refs.insert(self); - return refs; -} - -template -void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) -{ - if (ref == self) - hasSelfReference = true; - else - references.insert(std::move(ref)); -} - -template -void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) -{ - if (refs.count(self)) - hasSelfReference = true; - refs.erase(self); - - references = refs; -} +typedef References StoreReferences; /* * Full content address @@ -153,7 +113,7 @@ struct TextInfo : TextHash { struct FixedOutputInfo : FixedOutputHash { // References for the paths - PathReferences references; + StoreReferences references; GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6407e575a..605f871fc 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -758,7 +758,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.referencesPossiblyToSelf(path)); + worker_proto::write(*store, to, i->second.references.possiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -781,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.referencesPossiblyToSelf(i.first)); + worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9f3a6db24..b32953f3f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1157,11 +1157,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->references, - info->hasSelfReference, - info->deriver, - narInfo ? narInfo->fileSize : 0, - info->narSize, + .deriver = info->deriver, + .references = info->references, + .downloadSize = narInfo ? narInfo->fileSize : 0, + .narSize = info->narSize, }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { @@ -1228,7 +1227,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references; + return i == infos.end() ? StorePathSet() : i->second.references.others; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1427,8 +1426,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .hash = hash, }, .references = { - .references = references, - .hasSelfReference = false, + .others = references, + .self = false, }, }, }; @@ -1526,7 +1525,8 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - info.references = references; + // No self reference allowed with text-hashing + info.references.others = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index d6b6e87c9..5d7945eb1 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -27,15 +27,15 @@ std::map makeContentAddressed( StringMap rewrites; - PathReferences refs; - refs.hasSelfReference = oldInfo->hasSelfReference; - for (auto & ref : oldInfo->references) { + StoreReferences refs; + refs.self = oldInfo->references.self; + for (auto & ref : oldInfo->references.others) { auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? if (replacement != ref) { rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.references.insert(std::move(replacement)); + refs.others.insert(std::move(replacement)); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e0bb1fab0..87f85c3cc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -21,16 +21,16 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; StorePathSet referrers; queryReferrers(path, referrers); - for (auto& ref : referrers) + for (auto & ref : referrers) if (ref != path) res.insert(ref); if (includeOutputs) - for (auto& i : queryValidDerivers(path)) + for (auto & i : queryValidDerivers(path)) res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; @@ -40,11 +40,11 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto& ref : info->references) + for (auto & ref : info->references.others) res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); @@ -223,7 +223,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references) + for (auto & ref : info->second.references.others) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -241,7 +241,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references; + return queryPathInfo(path)->references.others; } catch (InvalidPath &) { return StorePathSet(); } @@ -297,7 +297,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); } } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 89886873a..9254835b7 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -14,20 +14,22 @@ namespace nix { class Store; -struct SubstitutablePathInfo : PathReferences +struct SubstitutablePathInfo { std::optional deriver; + StoreReferences references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; -struct ValidPathInfo : PathReferences +struct ValidPathInfo { StorePath path; std::optional deriver; Hash narHash; + StoreReferences references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -61,7 +63,6 @@ struct ValidPathInfo : PathReferences return path == i.path && narHash == i.narHash - && hasSelfReference == i.hasSelfReference && references == i.references; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 689ad3fbe..1f8098b85 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); + info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -426,7 +426,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); + info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cd48d616b..5490df292 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -169,13 +169,13 @@ StorePath Store::makeOutputPath(std::string_view id, static std::string makeType( const Store & store, std::string && type, - const PathReferences & references) + const StoreReferences & references) { - for (auto & i : references.references) { + for (auto & i : references.others) { type += ":"; type += store.printStorePath(i); } - if (references.hasSelfReference) type += ":self"; + if (references.self) type += ":self"; return std::move(type); } @@ -185,8 +185,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(info.references.references.size() == 0); - assert(!info.references.hasSelfReference); + assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" @@ -201,7 +200,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", PathReferences { info.references }), + makeType(*this, "text", StoreReferences { info.references }), info.hash, name); } @@ -311,7 +310,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references; + return info.references.others; }, [&](const StorePath & path) { @@ -816,7 +815,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info->references) + for (auto & j : info->referencesPossiblyToSelf()) s += printStorePath(j) + "\n"; } @@ -878,7 +877,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1231,17 +1230,17 @@ std::string showPaths(const PathSet & paths) StorePathSet ValidPathInfo::referencesPossiblyToSelf() const { - return PathReferences::referencesPossiblyToSelf(path); + return references.possiblyToSelf(path); } void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) { - return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); + return references.insertPossiblyToSelf(path, std::move(ref)); } void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) { - return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); + return references.setPossiblyToSelf(path, std::move(refs)); } std::string ValidPathInfo::fingerprint(const Store & store) const @@ -1271,16 +1270,16 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c .name = std::string { path.name() }, .info = std::visit(overloaded { [&](const TextHash & th) -> ContentAddressWithReferences { - assert(!hasSelfReference); + assert(!references.self); return TextInfo { th, - .references = references, + .references = references.others, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { return FixedOutputInfo { foh, - .references = static_cast>(*this), + .references = references, }; }, }, *ca), @@ -1341,11 +1340,14 @@ ValidPathInfo::ValidPathInfo( { std::visit(overloaded { [this](TextInfo && ti) { - this->references = std::move(ti.references); + this->references = { + .others = std::move(ti.references), + .self = false, + }; this->ca = std::move((TextHash &&) ti); }, [this](FixedOutputInfo && foi) { - *(static_cast *>(this)) = std::move(foi.references); + this->references = std::move(foi.references); this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh new file mode 100644 index 000000000..ac4a9994e --- /dev/null +++ b/src/libutil/reference-set.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "comparator.hh" + +#include + +namespace nix { + +template +struct References +{ + std::set others; + bool self = false; + + bool empty() const; + size_t size() const; + + /* Functions to view references + self as one set, mainly for + compatibility's sake. */ + std::set possiblyToSelf(const Ref & self) const; + void insertPossiblyToSelf(const Ref & self, Ref && ref); + void setPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(References, me->others, me->self); +}; + +template +bool References::empty() const +{ + return !self && others.empty(); +} + +template +size_t References::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +template +std::set References::possiblyToSelf(const Ref & selfRef) const +{ + std::set refs { others }; + if (self) + refs.insert(selfRef); + return refs; +} + +template +void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) +{ + if (ref == selfRef) + self = true; + else + others.insert(std::move(ref)); +} + +template +void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) +{ + if (refs.count(selfRef)) { + self = true; + refs.erase(selfRef); + } + + others = refs; +} + +} diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 614a37eba..8a0f06435 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -205,12 +205,14 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - .references = { references }, + .references = { + .others = std::move(references), + .self = false, + }, }, }, narHash, }; - info.references = std::move(references); info.narSize = sink.s.size(); StringSource source(sink.s); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index a08314a25..3d659d6d2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -63,8 +63,7 @@ struct CmdCopySigs : StorePathsCommand binary. */ if (info->narHash != info2->narHash || info->narSize != info2->narSize || - info->references != info2->references || - info->hasSelfReference != info2->hasSelfReference) + info->references != info2->references) continue; for (auto & sig : info2->sigs) diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 76125e5e4..33cd13600 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references, + .refs = store->queryPathInfo(path)->references.others, .dist = path == dependencyPath ? 0 : inf }); From 91617f80ec03ff4580a656310959ce2e31e0d177 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 16:00:10 -0500 Subject: [PATCH 16/32] Fix perl bindings --- perl/lib/Nix/Store.xs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 3ccd3c722..bdb4fa655 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -69,7 +69,7 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->referencesPossiblyToSelf()) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); From 2e7be46e73293f729358eefc5b464dcb7e2d76bf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 13 Jan 2023 15:06:07 -0500 Subject: [PATCH 17/32] Move new `ValidPathInfo` methods to path-info.cc We'll move the old ones separately, so as not to clutter the diff. --- src/libstore/path-info.cc | 15 +++++++++++++++ src/libstore/store-api.cc | 15 --------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 003685604..cb3077c61 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,6 +3,21 @@ namespace nix { +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return references.possiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return references.insertPossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return references.setPossiblyToSelf(path, std::move(refs)); +} + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5490df292..a4e98d66b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1228,21 +1228,6 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } -StorePathSet ValidPathInfo::referencesPossiblyToSelf() const -{ - return references.possiblyToSelf(path); -} - -void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) -{ - return references.insertPossiblyToSelf(path, std::move(ref)); -} - -void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) -{ - return references.setPossiblyToSelf(path, std::move(refs)); -} - std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) From b3d91239ae9f21a60057b278ceeff663fb786246 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 14 Jan 2023 16:38:43 -0500 Subject: [PATCH 18/32] Make `ValidPathInfo` have plain `StorePathSet` references like before This change can wait for another PR. --- perl/lib/Nix/Store.xs | 4 +- src/libexpr/primops.cc | 5 +- src/libstore/binary-cache-store.cc | 8 +-- src/libstore/build/local-derivation-goal.cc | 11 ++-- src/libstore/build/substitution-goal.cc | 10 +-- src/libstore/content-address.cc | 10 +++ src/libstore/content-address.hh | 11 +++- src/libstore/daemon.cc | 8 +-- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 6 +- src/libstore/local-store.cc | 10 ++- src/libstore/make-content-addressed.cc | 5 +- src/libstore/misc.cc | 11 ++-- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/path-info.cc | 48 ++++++--------- src/libstore/path-info.hh | 11 +--- src/libstore/remote-store.cc | 13 ++-- src/libstore/store-api.cc | 14 +++-- src/libutil/reference-set.hh | 68 --------------------- src/nix-store/dotgraph.cc | 2 +- src/nix-store/graphml.cc | 2 +- src/nix-store/nix-store.cc | 8 +-- src/nix/why-depends.cc | 2 +- 24 files changed, 109 insertions(+), 166 deletions(-) delete mode 100644 src/libutil/reference-set.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index bdb4fa655..f19fb20bf 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -69,7 +69,7 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->referencesPossiblyToSelf()) + for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -110,7 +110,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->referencesPossiblyToSelf()) + for (auto & i : info->references) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ae573cf4d..3b32625b1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1544,8 +1544,7 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V StorePathSet refs; if (state.store->isInStore(path)) { try { - // FIXME: Are self references becoming non-self references OK? - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf(); + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1980,7 +1979,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); + refs = state.store->queryPathInfo(storePath)->references; path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 087b37655..ac41add2c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,11 +180,11 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. Note that - self-references are always valid. */ - for (auto & ref : info.references.others) + reads, but typically they'll already be cached. */ + for (auto & ref : info.references) try { - queryPathInfo(ref); + if (ref != info.path) + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d96858fc0..bb4f92989 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2523,7 +2523,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - newInfo0.references = rewriteRefs(); + auto refs = rewriteRefs(); + newInfo0.references = std::move(refs.others); + if (refs.self) + newInfo0.references.insert(newInfo0.path); return newInfo0; }, @@ -2774,12 +2777,12 @@ void LocalDerivationGoal::checkOutputs(const std::mapsecond.narSize; - for (auto & ref : i->second.referencesPossiblyToSelf()) + for (auto & ref : i->second.references) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->referencesPossiblyToSelf()) + for (auto & ref : info->references) pathsLeft.push(ref); } } @@ -2819,7 +2822,7 @@ void LocalDerivationGoal::checkOutputs(const std::mapreferences.others) - addWaitee(worker.makePathSubstitutionGoal(i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -187,8 +188,9 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references.others) - assert(worker.store.isValidPath(i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3b8a773b7..a98e34cb8 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -151,6 +151,16 @@ Hash getContentAddressHash(const ContentAddress & ca) }, ca); } +bool StoreReferences::empty() const +{ + return !self && others.empty(); +} + +size_t StoreReferences::size() const +{ + return (self ? 1 : 0) + others.size(); +} + ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f8a4d5370..4a50bbee0 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,7 +4,6 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" -#include "reference-set.hh" namespace nix { @@ -95,7 +94,15 @@ Hash getContentAddressHash(const ContentAddress & ca); * References set */ -typedef References StoreReferences; +struct StoreReferences { + StorePathSet others; + bool self = false; + + bool empty() const; + size_t size() const; + + GENERATE_CMP(StoreReferences, me->self, me->others); +}; /* * Full content address diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 605f871fc..12596ba49 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -336,7 +336,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) + for (auto & i : store->queryPathInfo(path)->references) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -758,7 +758,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references.possiblyToSelf(path)); + worker_proto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -781,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first)); + worker_proto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -863,7 +863,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom {})); + info.references = worker_proto::read(*store, from, Phantom {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4adf51573..9875da909 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf()); + worker_proto::write(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(std::move(references)); + info.references = references; info.narSize = saved.s.size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 6a694f034..2c9dd2680 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -137,7 +137,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom {})); + info->references = worker_proto::read(*this, conn->from, Phantom {}); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -171,7 +171,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -200,7 +200,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b32953f3f..2d03d2d8b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -938,8 +938,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->insertReferencePossiblyToSelf( - parseStorePath(useQueryReferences.getStr(0))); + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -1206,7 +1205,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.referencesPossiblyToSelf()) + for (auto & j : i.references) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1227,7 +1226,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references.others; + return i == infos.end() ? StorePathSet() : i->second.references; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1525,8 +1524,7 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - // No self reference allowed with text-hashing - info.references.others = references; + info.references = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 5d7945eb1..09f615439 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -28,8 +28,9 @@ std::map makeContentAddressed( StringMap rewrites; StoreReferences refs; - refs.self = oldInfo->references.self; - for (auto & ref : oldInfo->references.others) { + for (auto & ref : oldInfo->references) { + if (ref == path) + refs.self = true; auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 70e97569a..da96dcebc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -40,8 +40,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references.others) - res.insert(ref); + for (auto & ref : info->references) + if (ref != path) + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) @@ -223,7 +224,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references.others) + for (auto & ref : info->second.references) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -241,7 +242,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references.others; + return queryPathInfo(path)->references; } catch (InvalidPath &) { return StorePathSet(); } @@ -297,7 +298,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8d4a21daf..3e0689534 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -248,7 +248,7 @@ public: narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->insertReferencePossiblyToSelf(StorePath(r)); + narInfo->references.insert(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index f54e8f1fc..071d8355e 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -63,7 +63,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) throw corrupt(); for (auto & r : refs) - insertReferencePossiblyToSelf(StorePath(r)); + references.insert(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 2972c0bbe..93f91e702 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -12,7 +12,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); + + concatStringsSep(",", store.printStorePathSet(references)); } @@ -30,16 +30,25 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c .name = std::string { path.name() }, .info = std::visit(overloaded { [&](const TextHash & th) -> ContentAddressWithReferences { - assert(!references.self); + assert(references.count(path) == 0); return TextInfo { th, - .references = references.others, + .references = references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } return FixedOutputInfo { foh, - .references = references, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, }; }, }, *ca), @@ -85,7 +94,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : referencesPossiblyToSelf()) + for (auto & r : references) refs.push_back(std::string(r.to_string())); return refs; } @@ -100,36 +109,19 @@ ValidPathInfo::ValidPathInfo( { std::visit(overloaded { [this](TextInfo && ti) { - this->references = { - .others = std::move(ti.references), - .self = false, - }; + this->references = std::move(ti.references); this->ca = std::move((TextHash &&) ti); }, [this](FixedOutputInfo && foi) { - this->references = std::move(foi.references); + this->references = std::move(foi.references.others); + if (foi.references.self) + this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); } -StorePathSet ValidPathInfo::referencesPossiblyToSelf() const -{ - return references.possiblyToSelf(path); -} - -void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) -{ - return references.insertPossiblyToSelf(path, std::move(ref)); -} - -void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) -{ - return references.setPossiblyToSelf(path, std::move(refs)); -} - - ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -141,7 +133,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(store, source, Phantom {})); + info.references = worker_proto::read(store, source, Phantom {}); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -162,7 +154,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, referencesPossiblyToSelf()); + worker_proto::write(store, sink, references); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 9254835b7..476df79c2 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -17,19 +17,20 @@ class Store; struct SubstitutablePathInfo { std::optional deriver; - StoreReferences references; + StorePathSet references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; + struct ValidPathInfo { StorePath path; std::optional deriver; Hash narHash; - StoreReferences references; + StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -81,12 +82,6 @@ struct ValidPathInfo /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; - void insertReferencePossiblyToSelf(StorePath && ref); - void setReferencesPossiblyToSelf(StorePathSet && refs); - static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8ea126c65..ff57a77ca 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -421,12 +421,11 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - auto path = parseStorePath(readString(conn->from)); - SubstitutablePathInfo & info { infos[path] }; + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -634,7 +633,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.referencesPossiblyToSelf()); + worker_proto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -645,7 +644,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); - assert(importedPaths.empty() == 0); // doesn't include possible self reference + assert(importedPaths.size() <= 1); } else { @@ -653,7 +652,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9446ad132..c39e50d14 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -200,7 +200,10 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", StoreReferences { info.references }), + makeType(*this, "text", StoreReferences { + .others = info.references, + .self = false, + }), info.hash, name); } @@ -310,7 +313,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references.others; + return info.references; }, [&](const StorePath & path) { @@ -815,7 +818,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info->referencesPossiblyToSelf()) + for (auto & j : info->references) s += printStorePath(j) + "\n"; } @@ -877,7 +880,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->referencesPossiblyToSelf()) + for (auto & ref : info->references) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1205,7 +1208,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.insertReferencePossiblyToSelf(store.parseStorePath(s)); + info.references.insert(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -1228,6 +1231,7 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh deleted file mode 100644 index ac4a9994e..000000000 --- a/src/libutil/reference-set.hh +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once - -#include "comparator.hh" - -#include - -namespace nix { - -template -struct References -{ - std::set others; - bool self = false; - - bool empty() const; - size_t size() const; - - /* Functions to view references + self as one set, mainly for - compatibility's sake. */ - std::set possiblyToSelf(const Ref & self) const; - void insertPossiblyToSelf(const Ref & self, Ref && ref); - void setPossiblyToSelf(const Ref & self, std::set && refs); - - GENERATE_CMP(References, me->others, me->self); -}; - -template -bool References::empty() const -{ - return !self && others.empty(); -} - -template -size_t References::size() const -{ - return (self ? 1 : 0) + others.size(); -} - -template -std::set References::possiblyToSelf(const Ref & selfRef) const -{ - std::set refs { others }; - if (self) - refs.insert(selfRef); - return refs; -} - -template -void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) -{ - if (ref == selfRef) - self = true; - else - others.insert(std::move(ref)); -} - -template -void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) -{ - if (refs.count(selfRef)) { - self = true; - refs.erase(selfRef); - } - - others = refs; -} - -} diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 36d774dca..577cadceb 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -56,7 +56,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { + for (auto & p : store->queryPathInfo(path)->references) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index d2eebca7a..425d61e53 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->referencesPossiblyToSelf()) { + for (auto & p : info->references) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5cb5aa53a..5b261ecc6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -263,7 +263,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); + auto sorted = store->topoSortPaths(info->references); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -344,7 +344,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) + for (auto & p : store->queryPathInfo(j)->references) paths.insert(p); } else if (query == qReferrers) { @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->referencesPossiblyToSelf()); + worker_proto::write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -964,7 +964,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom {})); + info.references = worker_proto::read(*store, in, Phantom {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 33cd13600..76125e5e4 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references.others, + .refs = store->queryPathInfo(path)->references, .dist = path == dependencyPath ? 0 : inf }); From 4540e7b940ca56db821fe7c7d7d79fafa488f55e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Jan 2023 12:58:11 -0500 Subject: [PATCH 19/32] Don't add `StorePathDescriptor` for now We don't need it yet, we can add it back later. --- src/libfetchers/tarball.cc | 14 ++--- src/libstore/binary-cache-store.cc | 50 +++++++--------- src/libstore/build/local-derivation-goal.cc | 14 ++--- src/libstore/build/substitution-goal.cc | 7 +-- src/libstore/content-address.hh | 7 --- src/libstore/local-store.cc | 35 +++++------ src/libstore/make-content-addressed.cc | 14 ++--- src/libstore/nar-info.hh | 4 +- src/libstore/path-info.cc | 64 ++++++++++----------- src/libstore/path-info.hh | 4 +- src/libstore/store-api.cc | 28 ++++----- src/libstore/store-api.hh | 2 +- src/nix/add-to-store.cc | 14 ++--- src/nix/profile.cc | 20 +++---- 14 files changed, 126 insertions(+), 151 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 155b86cc4..302046610 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -72,15 +72,13 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, res.data); ValidPathInfo info { *store, - { - .name = name, - .info = FixedOutputInfo { - { - .method = FileIngestionMethod::Flat, - .hash = hash, - }, - .references = {}, + name, + FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, }, + .references = {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ac41add2c..9058bb8b1 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -307,17 +307,15 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - { - .name = std::string { name }, - .info = FixedOutputInfo { - { - .method = method, - .hash = nar.first, - }, - .references = { - .others = references, - .self = false, - }, + name, + FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + .references = { + .others = references, + .self = false, }, }, nar.first, @@ -427,17 +425,15 @@ StorePath BinaryCacheStore::addToStore( return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - { - .name = std::string { name }, - .info = FixedOutputInfo { - { - .method = method, - .hash = h, - }, - .references = { - .others = references, - .self = false, - }, + name, + FixedOutputInfo { + { + .method = method, + .hash = h, + }, + .references = { + .others = references, + .self = false, }, }, nar.first, @@ -465,12 +461,10 @@ StorePath BinaryCacheStore::addTextToStore( return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - { - .name = std::string { name }, - .info = TextInfo { - { .hash = textHash }, - references, - }, + std::string { name }, + TextInfo { + { .hash = textHash }, + references, }, nar.first, }; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index bb4f92989..98f8cb061 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2475,15 +2475,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto got = caSink.finish().first; ValidPathInfo newInfo0 { worker.store, - { - .name = outputPathName(drv->name, outputName), - .info = FixedOutputInfo { - { - .method = outputHash.method, - .hash = got, - }, - .references = rewriteRefs(), + outputPathName(drv->name, outputName), + FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, }, + .references = rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 994cb4ac2..87fed495c 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -95,10 +95,9 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA({ - .name = std::string { storePath.name() }, - .info = caWithoutRefs(*ca), - }); + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + caWithoutRefs(*ca)); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 4a50bbee0..c49ab269f 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -132,11 +132,4 @@ typedef std::variant< ContentAddressWithReferences caWithoutRefs(const ContentAddress &); -struct StorePathDescriptor { - std::string name; - ContentAddressWithReferences info; - - GENERATE_CMP(StorePathDescriptor, me->name, me->info); -}; - } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2d03d2d8b..e55ccab84 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1136,10 +1136,9 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // Recompute store path so that we can use a different store root. if (path.second) { - subPath = makeFixedOutputPathFromCA({ - .name = std::string { path.first.name() }, - .info = caWithoutRefs(*path.second), - }); + subPath = makeFixedOutputPathFromCA( + path.first.name(), + caWithoutRefs(*path.second)); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1417,21 +1416,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); - auto desc = StorePathDescriptor { - std::string { name }, - FixedOutputInfo { - { - .method = method, - .hash = hash, - }, - .references = { - .others = references, - .self = false, - }, + ContentAddressWithReferences desc = FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + .references = { + .others = references, + .self = false, }, }; - auto dstPath = makeFixedOutputPathFromCA(desc); + auto dstPath = makeFixedOutputPathFromCA(name, desc); addTempRoot(dstPath); @@ -1475,7 +1471,12 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { *this, std::move(desc), narHash.first }; + ValidPathInfo info { + *this, + name, + std::move(desc), + narHash.first + }; info.narSize = narHash.second; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 09f615439..3ee64c77a 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -49,15 +49,13 @@ std::map makeContentAddressed( ValidPathInfo info { dstStore, - StorePathDescriptor { - .name = std::string { path.name() }, - .info = FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }, - .references = std::move(refs), + path.name(), + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, }, + .references = std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index f1e3aabbd..a4dccb397 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -16,8 +16,8 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) - : ValidPathInfo(store, std::move(ca), narHash) + NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + : ValidPathInfo(store, std::move(name), std::move(ca), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 93f91e702..5944afd06 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,48 +21,45 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullStorePathDescriptorOpt() const +std::optional ValidPathInfo::contentAddressWithReferenences() const { if (! ca) return std::nullopt; - return StorePathDescriptor { - .name = std::string { path.name() }, - .info = std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { - assert(references.count(path) == 0); - return TextInfo { - th, - .references = references, - }; - }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return FixedOutputInfo { - foh, - .references = { - .others = std::move(refs), - .self = hasSelfReference, - }, - }; - }, - }, *ca), - }; + return std::visit(overloaded { + [&](const TextHash & th) -> ContentAddressWithReferences { + assert(references.count(path) == 0); + return TextInfo { + th, + .references = references, + }; + }, + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return FixedOutputInfo { + foh, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + }, + }, *ca); } bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = fullStorePathDescriptorOpt(); + auto fullCaOpt = contentAddressWithReferenences(); if (! fullCaOpt) return false; - auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); + auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); bool res = caPath == path; @@ -102,9 +99,10 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - StorePathDescriptor && info, + std::string_view name, + ContentAddressWithReferences && ca, Hash narHash) - : path(store.makeFixedOutputPathFromCA(info)) + : path(store.makeFixedOutputPathFromCA(name, ca)) , narHash(narHash) { std::visit(overloaded { @@ -118,7 +116,7 @@ ValidPathInfo::ValidPathInfo( this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, - }, std::move(info.info)); + }, std::move(ca)); } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 476df79c2..663d94540 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -77,7 +77,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional fullStorePathDescriptorOpt() const; + std::optional contentAddressWithReferenences() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -100,7 +100,7 @@ struct ValidPathInfo ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const Store & store, - StorePathDescriptor && ca, Hash narHash); + std::string_view name, ContentAddressWithReferences && ca, Hash narHash); virtual ~ValidPathInfo() { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c39e50d14..3c0c26706 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -209,17 +209,17 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { [&](const TextInfo & ti) { - return makeTextPath(desc.name, ti); + return makeTextPath(name, ti); }, [&](const FixedOutputInfo & foi) { - return makeFixedOutputPath(desc.name, foi); + return makeFixedOutputPath(name, foi); } - }, desc.info); + }, ca); } @@ -437,15 +437,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, ValidPathInfo info { *this, - StorePathDescriptor { - std::string { name }, - FixedOutputInfo { - { - .method = method, - .hash = hash, - }, - .references = {}, + name, + FixedOutputInfo { + { + .method = method, + .hash = hash, }, + .references = {}, }, narHash, }; @@ -997,7 +995,8 @@ void copyStorePath( if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); info2->path = dstStore.makeFixedOutputPathFromCA( - info->fullStorePathDescriptorOpt().value()); + info->path.name(), + info->contentAddressWithReferenences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1110,7 +1109,8 @@ std::map copyPaths( auto storePathForDst = storePathForSrc; if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( - currentPathInfo.fullStorePathDescriptorOpt().value()); + currentPathInfo.path.name(), + currentPathInfo.contentAddressWithReferenences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d77aea338..2d252db84 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -216,7 +216,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 0b58818c3..81dbc09a6 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -43,15 +43,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - StorePathDescriptor { - .name = *namePart, - .info = FixedOutputInfo { - { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - }, - .references = {}, + std::move(*namePart), + FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), }, + .references = {}, }, narHash, }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index aac8e5c81..345505532 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -200,17 +200,15 @@ struct ProfileManifest ValidPathInfo info { *store, - StorePathDescriptor { - "profile", - FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, - .references = { - .others = std::move(references), - .self = false, - }, + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + .references = { + .others = std::move(references), + .self = false, }, }, narHash, From 974a983351283a644228b10731e4f9d2ff01e533 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 09:59:55 -0500 Subject: [PATCH 20/32] Shrink diff in two places Stuff crept in there. --- src/libstore/content-address.cc | 3 ++- src/libstore/make-content-addressed.cc | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a98e34cb8..a51646d0f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -17,8 +17,9 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; + default: + throw Error("impossible, caught both cases"); } - assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 3ee64c77a..ff9f5cdaa 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -31,12 +31,14 @@ std::map makeContentAddressed( for (auto & ref : oldInfo->references) { if (ref == path) refs.self = true; - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) { - rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.others.insert(std::move(replacement)); + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) { + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + refs.others.insert(std::move(replacement)); + } } } From 0983a0bd3050d02659f7c58555b8cbcfffed2c3b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:04:28 -0500 Subject: [PATCH 21/32] Shrink diff in one place --- src/libstore/make-content-addressed.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index ff9f5cdaa..42de79226 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -35,10 +35,9 @@ std::map makeContentAddressed( auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? - if (replacement != ref) { + if (replacement != ref) rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.others.insert(std::move(replacement)); - } + refs.others.insert(std::move(replacement)); } } From db759b1bc23c64b2aa6bdd0c5444a6d864488671 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:07:54 -0500 Subject: [PATCH 22/32] Undo style change `&` without space before is far more common on this codebase than I thought, so it is not worth changing just this one file. Maybe we will adopt a formatter someday but until then this is fine. --- src/libstore/misc.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1c187535d..b28768459 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -21,16 +21,16 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; StorePathSet referrers; queryReferrers(path, referrers); - for (auto & ref : referrers) + for (auto& ref : referrers) if (ref != path) res.insert(ref); if (includeOutputs) - for (auto & i : queryValidDerivers(path)) + for (auto& i : queryValidDerivers(path)) res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; @@ -40,12 +40,12 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references) + for (auto& ref : info->references) if (ref != path) res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); From 59d3175649a6bbdde76d1dfcf476c11392add827 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:09:25 -0500 Subject: [PATCH 23/32] Put back TODO I don't think the `narHash` is in need of documentation more than the other undocumented fields, but regardless this change has nothing to do with that field and so we should leave the comment as is. --- src/libstore/path-info.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 663d94540..35aced472 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,6 +29,7 @@ struct ValidPathInfo { StorePath path; std::optional deriver; + // TODO document this Hash narHash; StorePathSet references; time_t registrationTime = 0; From ee9eb83a842eb97d0180fd9d349d30ff27fdb485 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 11:25:56 -0500 Subject: [PATCH 24/32] Remove some designated initializers With the switch to C++20, the rules became more strict, and we can no longer initialize base classes. Make them comments instead. (BTW https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2287r1.html this offers some new syntax for this use-case. Hopefully this will be adopted and we can eventually use it.) --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 4 ++-- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index f19fb20bf..314183383 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -299,7 +299,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8b54c4477..4e2f92276 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1287,7 +1287,7 @@ drvName, Bindings * attrs, Value & v) .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", @@ -2103,7 +2103,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - .references = {}, + /* .references = */ {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e194462e4..69395ad3d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -240,7 +240,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - .references = {} + /* .references = */ {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 3936eadfe..dae4998f9 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - .references = {}, + /* .references = */ {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 302046610..b6f72bb1f 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -78,7 +78,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - .references = {}, + /* .references = */ {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9cb0f74f6..5617e2c42 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - .references = { + /* .references = */ { .others = references, .self = false, }, @@ -431,7 +431,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - .references = { + /* .references = */ { .others = references, .self = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8fdc9dce1..cfd5db3b4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2498,7 +2498,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - .references = rewriteRefs(), + /* .references = */ rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a51646d0f..39a31f0a0 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -167,13 +167,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { h, - .references = {}, + /* .references = */ {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { h, - .references = {}, + /* .references = */ {}, }; }, }, ca); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6f870dde..9d2ea7156 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1419,7 +1419,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - .references = { + /* .references = */ { .others = references, .self = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 42de79226..59a452918 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -56,7 +56,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - .references = std::move(refs), + /* .references = */ std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5944afd06..ff85b3932 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -31,7 +31,7 @@ std::optional ValidPathInfo::contentAddressWithRef assert(references.count(path) == 0); return TextInfo { th, - .references = references, + /* .references = */ references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { @@ -43,7 +43,7 @@ std::optional ValidPathInfo::contentAddressWithRef } return FixedOutputInfo { foh, - .references = { + /* .references = */ { .others = std::move(refs), .self = hasSelfReference, }, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3c0c26706..295ce4953 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -234,7 +234,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -443,7 +443,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - .references = {}, + /* .references = */ {}, }, narHash, }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6..28ddf2f4a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - .references = {}, + /* .references = */ {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6..5de1aebfc 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -49,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - .references = {}, + /* .references = */ {}, }, narHash, }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index df9933d29..bc270f66d 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - .references = {}, + /* .references = */ {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 345505532..e552e8975 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -206,7 +206,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - .references = { + /* .references = */ { .others = std::move(references), .self = false, }, From c36b584f8eb103afa152ef4304cf9fd5c3ebaaf0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 11:34:18 -0500 Subject: [PATCH 25/32] Fix typo in the method name --- src/libstore/path-info.cc | 4 ++-- src/libstore/path-info.hh | 2 +- src/libstore/store-api.cc | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ff85b3932..074b50818 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,7 +21,7 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::contentAddressWithReferenences() const +std::optional ValidPathInfo::contentAddressWithReferences() const { if (! ca) return std::nullopt; @@ -54,7 +54,7 @@ std::optional ValidPathInfo::contentAddressWithRef bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = contentAddressWithReferenences(); + auto fullCaOpt = contentAddressWithReferences(); if (! fullCaOpt) return false; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 35aced472..97eb6638b 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -78,7 +78,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional contentAddressWithReferenences() const; + std::optional contentAddressWithReferences() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 06d746a0b..73dcaf150 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -996,7 +996,7 @@ void copyStorePath( auto info2 = make_ref(*info); info2->path = dstStore.makeFixedOutputPathFromCA( info->path.name(), - info->contentAddressWithReferenences().value()); + info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1110,7 +1110,7 @@ std::map copyPaths( if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( currentPathInfo.path.name(), - currentPathInfo.contentAddressWithReferenences().value()); + currentPathInfo.contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) From 123b11ff83da0cbcef6e4aae276bfbdd7a183656 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 11:49:13 -0500 Subject: [PATCH 26/32] Clarify store path grammar and improve comment on `makeType` --- src/libstore/store-api.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 73dcaf150..2ff92c0e6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -99,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const silly, but it's done that way for compatibility). is the name of the output (usually, "out"). -

= base-16 representation of a SHA-256 hash of: +

= base-16 representation of a SHA-256 hash of + + = if = "text:...": the string written to the resulting store path - if = "source": + if = "source:...": the serialisation of the path from which this store path is copied, as returned by hashPath() if = "output:": @@ -164,8 +166,8 @@ StorePath Store::makeOutputPath(std::string_view id, /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ + hacky, but we can't put them in, say, (per the grammar above) + since that would be ambiguous. */ static std::string makeType( const Store & store, std::string && type, From 85bb865d200f04b73f183af722757c78d5a3be76 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 11:57:20 -0500 Subject: [PATCH 27/32] Revert "Remove some designated initializers" This reverts commit ee9eb83a842eb97d0180fd9d349d30ff27fdb485. --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 4 ++-- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index db733ce40..fca7607d3 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -300,7 +300,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - /* .references = */ {}, + .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 762ff8249..a54cca5ab 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1287,7 +1287,7 @@ drvName, Bindings * attrs, Value & v) .method = method, .hash = h, }, - /* .references = */ {}, + .references = {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", @@ -2103,7 +2103,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - /* .references = */ {}, + .references = {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index ffc6f5859..93592290f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -250,7 +250,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - /* .references = */ {} + .references = {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index dae4998f9..3936eadfe 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - /* .references = */ {}, + .references = {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b6f72bb1f..302046610 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -78,7 +78,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - /* .references = */ {}, + .references = {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5617e2c42..9cb0f74f6 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - /* .references = */ { + .references = { .others = references, .self = false, }, @@ -431,7 +431,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - /* .references = */ { + .references = { .others = references, .self = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 472188fea..765bb8f35 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2446,7 +2446,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - /* .references = */ rewriteRefs(), + .references = rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 39a31f0a0..a51646d0f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -167,13 +167,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { h, - /* .references = */ {}, + .references = {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { h, - /* .references = */ {}, + .references = {}, }; }, }, ca); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9d2ea7156..c6f870dde 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1419,7 +1419,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - /* .references = */ { + .references = { .others = references, .self = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 59a452918..42de79226 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -56,7 +56,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - /* .references = */ std::move(refs), + .references = std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 074b50818..2a03e9dfa 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -31,7 +31,7 @@ std::optional ValidPathInfo::contentAddressWithRef assert(references.count(path) == 0); return TextInfo { th, - /* .references = */ references, + .references = references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { @@ -43,7 +43,7 @@ std::optional ValidPathInfo::contentAddressWithRef } return FixedOutputInfo { foh, - /* .references = */ { + .references = { .others = std::move(refs), .self = hasSelfReference, }, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2ff92c0e6..1c01c9cd8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -236,7 +236,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - /* .references = */ {}, + .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -445,7 +445,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - /* .references = */ {}, + .references = {}, }, narHash, }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 28ddf2f4a..5b261ecc6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - /* .references = */ {}, + .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5de1aebfc..81dbc09a6 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -49,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - /* .references = */ {}, + .references = {}, }, narHash, }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index bc270f66d..df9933d29 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - /* .references = */ {}, + .references = {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 28c7fe32d..5505d8bdd 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -207,7 +207,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - /* .references = */ { + .references = { .others = std::move(references), .self = false, }, From d381248ec0847cacd918480e83a99287f814456a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 12:13:43 -0500 Subject: [PATCH 28/32] No inheritance for `TextInfo` and `FixedOutputInfo` --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/content-address.hh | 10 ++++++---- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 16 ++++++++-------- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 17 files changed, 33 insertions(+), 31 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index fca7607d3..bfe00d3e2 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -296,7 +296,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a54cca5ab..7af796aa6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1283,7 +1283,7 @@ drvName, Bindings * attrs, Value & v) auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, @@ -2099,7 +2099,7 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = *expectedHash, }, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 93592290f..f3dce2214 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -246,7 +246,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - { + .hash = { .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 3936eadfe..91db3a9eb 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -211,7 +211,7 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = *narHash, }, diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 302046610..96fe5faca 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -74,7 +74,7 @@ DownloadFileResult downloadFile( *store, name, FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Flat, .hash = hash, }, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9cb0f74f6..9eae8d534 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -309,7 +309,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = nar.first, }, @@ -427,7 +427,7 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 765bb8f35..87eac6e19 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2442,7 +2442,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() worker.store, outputPathName(drv->name, outputName), FixedOutputInfo { - { + .hash = { .method = outputHash.method, .hash = got, }, diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a51646d0f..d9a8a4535 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -166,13 +166,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { - h, + .hash = h, .references = {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { - h, + .hash = h, .references = {}, }; }, diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c49ab269f..9fae288d8 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -111,18 +111,20 @@ struct StoreReferences { */ // This matches the additional info that we need for makeTextPath -struct TextInfo : TextHash { +struct TextInfo { + TextHash hash; // References for the paths, self references disallowed StorePathSet references; - GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); + GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo : FixedOutputHash { +struct FixedOutputInfo { + FixedOutputHash hash; // References for the paths StoreReferences references; - GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); + GENERATE_CMP(FixedOutputInfo, me->hash, me->references); }; typedef std::variant< diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6f870dde..8b33b2da5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1415,7 +1415,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 42de79226..53fe04704 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,7 +52,7 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 2a03e9dfa..76cab63e0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -30,7 +30,7 @@ std::optional ValidPathInfo::contentAddressWithRef [&](const TextHash & th) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - th, + .hash = th, .references = references, }; }, @@ -42,7 +42,7 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - foh, + .hash = foh, .references = { .others = std::move(refs), .self = hasSelfReference, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1c01c9cd8..b8a77b324 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -184,15 +184,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash, name); + if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.hash.method) + + info.hash.hash.to_string(Base16, true) + ":"), name); } } @@ -200,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.type == htSHA256); + assert(info.hash.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash, + info.hash.hash, name); } @@ -232,7 +232,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); FixedOutputInfo caInfo { - { + .hash = { .method = method, .hash = h, }, @@ -441,7 +441,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6..735d6a592 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -216,7 +216,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6..16e48a39b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -45,7 +45,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand *store, std::move(*namePart), FixedOutputInfo { - { + .hash = { .method = std::move(ingestionMethod), .hash = std::move(hash), }, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index df9933d29..209517b21 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -68,7 +68,7 @@ std::tuple prefetchFile( if (expectedHash) { hashType = expectedHash->type; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - { + .hash = { .method = ingestionMethod, .hash = *expectedHash, }, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 5505d8bdd..04ac48f00 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -203,7 +203,7 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = narHash, }, From a6d00a7bfb18e7ec461ac1d54203cc628aca5c66 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Mar 2023 16:29:13 -0400 Subject: [PATCH 29/32] Fix warning --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9eae8d534..628e9b9db 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -450,7 +450,7 @@ StorePath BinaryCacheStore::addTextToStore( RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, TextInfo { textHash, references }); + auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) return path; From c51d554c933b5fe294da41fcdf5afe0d4f33f586 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Mar 2023 17:12:49 -0400 Subject: [PATCH 30/32] Use "raw pattern" for content address types We weren't because this ancient PR predated it! This is actually a new version of the pattern which addresses some issues identified in #7479. --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/content-address.cc | 47 +++++---- src/libstore/content-address.hh | 110 +++++++++++++------- src/libstore/daemon.cc | 10 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 8 +- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/path-info.cc | 6 +- src/libstore/remote-store.cc | 8 +- src/libstore/store-api.cc | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix/prefetch.cc | 2 +- 14 files changed, 118 insertions(+), 87 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index caa15ab04..4fb7aa9d8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2508,7 +2508,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* Check wanted hash */ const Hash & wanted = dof.hash.hash; assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); + auto got = newInfo0.ca->getHash(); if (wanted != got) { /* Throw an error after registering the path as valid. */ diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 87fed495c..190fb455a 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -97,7 +97,7 @@ void PathSubstitutionGoal::tryNext() if (ca) { subPath = sub->makeFixedOutputPathFromCA( std::string { storePath.name() }, - caWithoutRefs(*ca)); + ContentAddressWithReferences::withoutRefs(*ca)); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 64daea0d4..055b216db 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,7 +9,6 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } - std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { @@ -22,35 +21,35 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) } } -std::string renderContentAddress(ContentAddress ca) +std::string ContentAddress::render() const { return std::visit(overloaded { - [](TextHash & th) { + [](const TextHash & th) { return "text:" + th.hash.to_string(Base32, true); }, - [](FixedOutputHash & fsh) { + [](const FixedOutputHash & fsh) { return "fixed:" + makeFileIngestionPrefix(fsh.method) + fsh.hash.to_string(Base32, true); } - }, ca); + }, raw); } -std::string renderContentAddressMethod(ContentAddressMethod cam) +std::string ContentAddressMethod::render() const { return std::visit(overloaded { - [](TextHashMethod & th) { + [](const TextHashMethod & th) { return std::string{"text:"} + printHashType(htSHA256); }, - [](FixedOutputHashMethod & fshm) { + [](const FixedOutputHashMethod & fshm) { return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); } - }, cam); + }, raw); } -/* - Parses content address strings up to the hash. +/** + * Parses content address strings up to the hash. */ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) { @@ -94,7 +93,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); } -ContentAddress parseContentAddress(std::string_view rawCa) { +ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); @@ -112,10 +111,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), }); }, - }, caMethod); + }, caMethod.raw); } -ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -123,26 +122,28 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) return parseContentAddressMethodPrefix(asPrefixView); } -std::optional parseContentAddressOpt(std::string_view rawCaOpt) +std::optional ContentAddress::parseOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional() : parseContentAddress(rawCaOpt); + return rawCaOpt == "" + ? std::nullopt + : std::optional { ContentAddress::parse(rawCaOpt) }; }; std::string renderContentAddress(std::optional ca) { - return ca ? renderContentAddress(*ca) : ""; + return ca ? ca->render() : ""; } -Hash getContentAddressHash(const ContentAddress & ca) +const Hash & ContentAddress::getHash() const { return std::visit(overloaded { - [](const TextHash & th) { + [](const TextHash & th) -> auto & { return th.hash; }, - [](const FixedOutputHash & fsh) { + [](const FixedOutputHash & fsh) -> auto & { return fsh.hash; }, - }, ca); + }, raw); } bool StoreReferences::empty() const @@ -155,7 +156,7 @@ size_t StoreReferences::size() const return (self ? 1 : 0) + others.size(); } -ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { +ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { @@ -169,7 +170,7 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { .references = {}, }; }, - }, ca); + }, ca.raw); } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index d74d1ff4b..d1dd1256c 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -39,17 +39,16 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; -}; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. */ std::string makeFileIngestionPrefix(FileIngestionMethod m); +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; /** * An enumeration of all the ways we can serialize file system objects. @@ -59,14 +58,25 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m); * with info on references, and we have `ContentAddressWithReferences`, * as defined further below. */ -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod -> ContentAddressMethod; +struct ContentAddressMethod +{ + typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > Raw; -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + Raw raw; + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressMethod(auto &&... arg) + : raw(std::forward(arg)...) + { } + + static ContentAddressMethod parse(std::string_view rawCaMethod); + + std::string render() const; +}; -std::string renderContentAddressMethod(ContentAddressMethod caMethod); /* * Mini content address @@ -115,25 +125,41 @@ struct FixedOutputHash { * - ‘fixed:::’: For paths computed by * Store::makeFixedOutputPath() / Store::addToStore(). */ -typedef std::variant< - TextHash, - FixedOutputHash -> ContentAddress; +struct ContentAddress +{ + typedef std::variant< + TextHash, + FixedOutputHash + > Raw; -/** - * Compute the content-addressability assertion (ValidPathInfo::ca) for - * paths created by Store::makeFixedOutputPath() / Store::addToStore(). - */ -std::string renderContentAddress(ContentAddress ca); + Raw raw; + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddress(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Compute the content-addressability assertion (ValidPathInfo::ca) for + * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + */ + std::string render() const; + + static ContentAddress parse(std::string_view rawCa); + + static std::optional parseOpt(std::string_view rawCaOpt); + + const Hash & getHash() const; +}; std::string renderContentAddress(std::optional ca); -ContentAddress parseContentAddress(std::string_view rawCa); - -std::optional parseContentAddressOpt(std::string_view rawCaOpt); - -Hash getContentAddressHash(const ContentAddress & ca); +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ /** * A set of references to other store objects. @@ -167,12 +193,6 @@ struct StoreReferences { GENERATE_CMP(StoreReferences, me->self, me->others); }; -/* - * Full content address - * - * See the schema for store paths in store-api.cc - */ - // This matches the additional info that we need for makeTextPath struct TextInfo { TextHash hash; @@ -200,15 +220,25 @@ struct FixedOutputInfo { * * A ContentAddress without a Hash. */ -typedef std::variant< - TextInfo, - FixedOutputInfo -> ContentAddressWithReferences; +struct ContentAddressWithReferences +{ + typedef std::variant< + TextInfo, + FixedOutputInfo + > Raw; -/** - * Create a ContentAddressWithReferences from a mere ContentAddress, by - * assuming no references in all cases. - */ -ContentAddressWithReferences caWithoutRefs(const ContentAddress &); + Raw raw; + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressWithReferences(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Create a ContentAddressWithReferences from a mere ContentAddress, by + * assuming no references in all cases. + */ + static ContentAddressWithReferences withoutRefs(const ContentAddress &); +}; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 656ad4587..0169eef1a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -401,21 +401,21 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr); FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](TextHashMethod &) { + [&](const TextHashMethod &) { // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, - [&](FixedOutputHashMethod & fohm) { + [&](const FixedOutputHashMethod & fohm) { auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, - }, contentAddressMethod); + }, contentAddressMethod.raw); }(); logger->stopWork(); @@ -880,7 +880,7 @@ static void performOp(TunnelLogger * logger, ref store, info.references = worker_proto::read(*store, from, Phantom {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseContentAddressOpt(readString(from)); + info.ca = ContentAddress::parseOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 98322b045..a1c38d180 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -156,7 +156,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor throw Error("NAR hash is now mandatory"); info->narHash = Hash::parseAnyPrefixed(s); } - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = ContentAddress::parseOpt(readString(conn->from)); info->sigs = readStrings(conn->from); auto s = readString(conn->from); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e1c7e387a..b49d5462b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -944,7 +944,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = ContentAddress::parseOpt(s); /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); @@ -1150,7 +1150,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst if (path.second) { subPath = makeFixedOutputPathFromCA( path.first.name(), - caWithoutRefs(*path.second)); + ContentAddressWithReferences::withoutRefs(*path.second)); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1329,7 +1329,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&*info.ca)) { + if (auto foHash = std::get_if(&info.ca->raw)) { auto actualFoHash = hashCAPath( foHash->method, foHash->hash.type, @@ -1342,7 +1342,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, actualFoHash.hash.to_string(Base32, true)); } } - if (auto textHash = std::get_if(&*info.ca)) { + if (auto textHash = std::get_if(&info.ca->raw)) { auto actualTextHash = hashString(htSHA256, readFile(realPath)); if (textHash->hash != actualTextHash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 2645f468b..c7176d30f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -273,7 +273,7 @@ public: narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 071d8355e..274cd861c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -74,7 +74,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseContentAddressOpt(value); + ca = ContentAddress::parseOpt(value); } pos = eol + 1; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 76cab63e0..e60d7abe0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -49,7 +49,7 @@ std::optional ValidPathInfo::contentAddressWithRef }, }; }, - }, *ca); + }, ca->raw); } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -116,7 +116,7 @@ ValidPathInfo::ValidPathInfo( this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, - }, std::move(ca)); + }, std::move(ca).raw); } @@ -136,7 +136,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned if (format >= 16) { source >> info.ultimate; info.sigs = readStrings(source); - info.ca = parseContentAddressOpt(readString(source)); + info.ca = ContentAddress::parseOpt(readString(source)); } return info; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d24d83117..ac98e76d2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -44,7 +44,7 @@ void write(const Store & store, Sink & out, const StorePath & storePath) ContentAddress read(const Store & store, Source & from, Phantom _) { - return parseContentAddress(readString(from)); + return ContentAddress::parse(readString(from)); } void write(const Store & store, Sink & out, const ContentAddress & ca) @@ -134,7 +134,7 @@ void write(const Store & store, Sink & out, const std::optional & sto std::optional read(const Store & store, Source & from, Phantom> _) { - return parseContentAddressOpt(readString(from)); + return ContentAddress::parseOpt(readString(from)); } void write(const Store & store, Sink & out, const std::optional & caOpt) @@ -545,7 +545,7 @@ ref RemoteStore::addCAToStore( conn->to << wopAddToStore << name - << renderContentAddressMethod(caMethod); + << caMethod.render(); worker_proto::write(*this, conn->to, references); conn->to << repair; @@ -603,7 +603,7 @@ ref RemoteStore::addCAToStore( } } - }, caMethod); + }, caMethod.raw); auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fed38e2dd..78b0d907e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -221,7 +221,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA [&](const FixedOutputInfo & foi) { return makeFixedOutputPath(name, foi); } - }, ca); + }, ca.raw); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 26febb6e3..3d2dc49fd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -970,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.references = worker_proto::read(*store, in, Phantom {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseContentAddressOpt(readString(in)); + info.ca = ContentAddress::parseOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b06b8a320..56e7bbb6e 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -124,7 +124,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = getContentAddressHash(*info.ca); + hash = info.ca->getHash(); } return {storePath.value(), hash.value()}; From 5d56e2daf70788fae532d1875edbd4e9bdb5afef Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Apr 2023 16:52:23 -0400 Subject: [PATCH 31/32] Add comparison methods for content addresses --- src/libstore/content-address.hh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index d1dd1256c..737bf9a41 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -46,8 +46,10 @@ enum struct FileIngestionMethod : uint8_t { std::string makeFileIngestionPrefix(FileIngestionMethod m); struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; + FileIngestionMethod fileIngestionMethod; + HashType hashType; + + GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType); }; /** @@ -67,6 +69,8 @@ struct ContentAddressMethod Raw raw; + GENERATE_CMP(ContentAddressMethod, me->raw); + /* The moral equivalent of `using Raw::Raw;` */ ContentAddressMethod(auto &&... arg) : raw(std::forward(arg)...) @@ -134,6 +138,8 @@ struct ContentAddress Raw raw; + GENERATE_CMP(ContentAddress, me->raw); + /* The moral equivalent of `using Raw::Raw;` */ ContentAddress(auto &&... arg) : raw(std::forward(arg)...) @@ -229,6 +235,8 @@ struct ContentAddressWithReferences Raw raw; + GENERATE_CMP(ContentAddressWithReferences, me->raw); + /* The moral equivalent of `using Raw::Raw;` */ ContentAddressWithReferences(auto &&... arg) : raw(std::forward(arg)...) From 537e8719f2ca8e18312bd8dcc37124fb1b25d4d3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 09:15:11 -0400 Subject: [PATCH 32/32] Explain various `.self = false,` Co-authored-by: Robert Hensing --- src/libstore/binary-cache-store.cc | 2 ++ src/libstore/local-store.cc | 1 + src/nix/profile.cc | 1 + 3 files changed, 4 insertions(+) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 628e9b9db..fcd763a9d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -315,6 +315,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, @@ -433,6 +434,7 @@ StorePath BinaryCacheStore::addToStore( }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0072a16dc..7fb312c37 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1328,6 +1328,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 5aa87a313..fd63b3519 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -209,6 +209,7 @@ struct ProfileManifest }, .references = { .others = std::move(references), + // profiles never refer to themselves .self = false, }, },