From 1722ae6ecee54e14164d215ba3d767ea6c352fc3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jun 2020 05:41:18 +0000 Subject: [PATCH 01/61] Pull out PathReferences super class --- src/libstore/path-info.hh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index f5dee00a6..a980e1243 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -20,12 +20,17 @@ namespace nix { class Store; -struct ValidPathInfo +template +struct PathReferences +{ + std::set references; +}; + +struct ValidPathInfo : PathReferences { StorePath path; std::optional deriver; Hash narHash; - StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only From 71e4c9c505f2418084643c1a68da5c89b82038dd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jun 2020 22:46:27 +0000 Subject: [PATCH 02/61] WIP: store separate `hasValidPath` bool --- src/libstore/binary-cache-store.cc | 3 +- src/libstore/build.cc | 14 ++++---- src/libstore/daemon.cc | 10 +++--- src/libstore/export-import.cc | 4 +-- src/libstore/legacy-ssh-store.cc | 7 ++-- src/libstore/local-store.cc | 23 +++++++------ src/libstore/misc.cc | 5 ++- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/path-info.hh | 52 +++++++++++++++++++++++++++++ src/libstore/remote-store.cc | 11 +++--- src/libstore/store-api.cc | 29 ++++++++++------ src/libstore/store-api.hh | 11 ------ src/nix-store/dotgraph.cc | 2 +- src/nix-store/graphml.cc | 2 +- src/nix-store/nix-store.cc | 8 ++--- src/nix/make-content-addressable.cc | 2 +- src/nix/sigs.cc | 3 +- 17 files changed, 119 insertions(+), 69 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..7167ec900 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -125,8 +125,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource reads, but typically they'll already be cached. */ for (auto & ref : info.references) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0c25897f8..c01b2ddaf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3834,7 +3834,7 @@ void DerivationGoal::registerOutputs() ValidPathInfo info(worker.store.parseStorePath(path)); info.narHash = hash.first; info.narSize = hash.second; - info.references = std::move(references); + info.setReferencesPossiblyToSelf(std::move(references)); info.deriver = drvPath; info.ultimate = true; info.ca = ca; @@ -3963,12 +3963,12 @@ void DerivationGoal::checkOutputs(const std::map & outputs) auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -3997,7 +3997,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) auto used = recursive ? getClosure(info.path).first - : info.references; + : info.referencesPossiblyToSelf(); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); @@ -4466,8 +4466,7 @@ void SubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); + addWaitee(worker.makeSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -4487,8 +4486,7 @@ void SubstitutionGoal::referencesValid() } for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + assert(worker.store.isValidPath(i)); state = &SubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 842aef20c..c6b70f6f3 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -326,7 +326,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -601,7 +601,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - writeStorePaths(*store, to, i->second.references); + writeStorePaths(*store, to, i->second.referencesPossiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -618,7 +618,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - writeStorePaths(*store, to, i.second.references); + writeStorePaths(*store, to, i.second.referencesPossiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -647,7 +647,7 @@ static void performOp(TunnelLogger * logger, ref store, to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") << info->narHash.to_string(Base16, false); - writeStorePaths(*store, to, info->references); + writeStorePaths(*store, to, info->referencesPossiblyToSelf()); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate @@ -707,7 +707,7 @@ static void performOp(TunnelLogger * logger, ref store, if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths(*store, from); + info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..5a5c76f7f 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -62,7 +62,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) hashAndWriteSink << exportMagic << printStorePath(path); - writeStorePaths(*this, hashAndWriteSink, info->references); + writeStorePaths(*this, hashAndWriteSink, info->referencesPossiblyToSelf()); hashAndWriteSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -88,7 +88,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths(*this, source); + info.setReferencesPossiblyToSelf(readStorePaths(*this, source)); auto deriver = readString(source); if (deriver != "") diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5657aa593..f01e642a0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -103,11 +103,10 @@ struct LegacySSHStore : public Store auto info = std::make_shared(parseStorePath(p)); assert(path == info->path); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = readStorePaths(*this, conn->from); + info->setReferencesPossiblyToSelf(readStorePaths(*this, conn->from)); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -140,7 +139,7 @@ struct LegacySSHStore : public Store << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -169,7 +168,7 @@ struct LegacySSHStore : public Store conn->to << exportMagic << printStorePath(info.path); - writeStorePaths(*this, conn->to, info.references); + writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0dfbed9fc..02de3aa5e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -671,8 +671,10 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + while (useQueryReferences.next()) { + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); + } return info; })); @@ -856,11 +858,13 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); - infos.insert_or_assign(path, SubstitutablePathInfo{ - info->deriver, + infos.insert_or_assign(path, SubstitutablePathInfo { info->references, + info->hasSelfReference, + info->deriver, narInfo ? narInfo->fileSize : 0, - info->narSize}); + info->narSize, + }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -907,7 +911,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -986,14 +990,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); // text hashing has long been allowed to have non-self-references because it is used for drv files. - bool refersToSelf = info.references.count(info.path) > 0; - if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && !refersToSelf)) + if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && info.hasSelfReference)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr hashSink; - if (!info.ca.has_value() || !info.references.count(info.path)) + if (!info.ca.has_value() || !info.hasSelfReference) hashSink = std::make_unique(htSHA256); else hashSink = std::make_unique(htSHA256, std::string(info.path.hashPart())); @@ -1254,7 +1257,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr hashSink; - if (!info->ca || !info->references.count(info->path)) + if (!info->ca || !info->hasSelfReference) hashSink = std::make_unique(*info->narHash.type); else hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e68edb38c..5214a7bf4 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -61,8 +61,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths, } else { for (auto & ref : info->references) - if (ref != path) - enqueue(printStorePath(ref)); + enqueue(printStorePath(ref)); if (includeOutputs && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) @@ -268,7 +267,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) for (auto & i : references) /* Don't traverse into paths that don't exist. That can happen due to substitutes for non-existent paths. */ - if (i != path && paths.count(i)) + if (paths.count(i)) dfsVisit(i, &path); sorted.push_back(path); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 012dea6ea..c543f6ea2 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -198,7 +198,7 @@ public: narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a980e1243..27efe5ae9 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -24,8 +24,43 @@ template struct PathReferences { std::set references; + bool hasSelfReference = false; + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); }; +template +StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet references { references }; + if (hasSelfReference) + references.insert(self); + return references; +} + +template +void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} + +template +void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + struct ValidPathInfo : PathReferences { StorePath path; @@ -64,6 +99,7 @@ struct ValidPathInfo : PathReferences return path == i.path && narHash == i.narHash + && hasSelfReference == i.hasSelfReference && references == i.references; } @@ -80,6 +116,12 @@ struct ValidPathInfo : PathReferences /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; + void insertReferencePossiblyToSelf(StorePath && ref); + void setReferencesPossiblyToSelf(StorePathSet && refs); + static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were @@ -101,4 +143,14 @@ struct ValidPathInfo : PathReferences }; typedef list ValidPathInfos; + + +struct SubstitutablePathInfo : PathReferences +{ + std::optional deriver; + unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ +}; + +typedef std::map SubstitutablePathInfos; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b7cc7a5fc..f84b62f2e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -326,7 +326,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.setReferencesPossiblyToSelf(i, readStorePaths(*this, conn->from)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i, std::move(info)); @@ -339,11 +339,12 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.setReferencesPossiblyToSelf(path, readStorePaths(*this, conn->from)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -376,7 +377,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths(*this, conn->from); + info->setReferencesPossiblyToSelf(readStorePaths(*this, conn->from)); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -455,7 +456,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = readStorePaths(*this, conn->from); - assert(importedPaths.size() <= 1); + assert(importedPaths.empty() == 0); // doesn't include possible self reference } else { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e4a4ae11e..95b1c1c3b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -715,7 +715,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!string2Int(s, n)) throw Error("number expected"); while (n--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -738,6 +738,20 @@ string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return PathReferences::referencesPossiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); +} std::string ValidPathInfo::fingerprint(const Store & store) const { @@ -748,7 +762,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -767,16 +781,11 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const auto caPath = std::visit(overloaded { [&](TextHash th) { + assert(!hasSelfReference); return store.makeTextPath(path.name(), th.hash, references); }, [&](FixedOutputHash fsh) { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); + return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference); } }, *ca); @@ -810,7 +819,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 00b9c385c..420ffebbe 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -102,17 +102,6 @@ struct GCResults }; -struct SubstitutablePathInfo -{ - std::optional deriver; - StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ -}; - -typedef std::map SubstitutablePathInfos; - - enum BuildMode { bmNormal, bmRepair, bmCheck }; diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 8b699f39b..45abe0405 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -58,7 +58,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 8ca5c9c8d..1cd974e41 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7d81bf54f..c4ca89c85 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -245,7 +245,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -328,7 +328,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -859,7 +859,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - writeStorePaths(*store, out, info->references); + writeStorePaths(*store, out, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -949,7 +949,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(in), htSHA256); - info.references = readStorePaths(*store, in); + info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index fb36fc410..5267948ee 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -79,7 +79,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); + info.hasSelfReference = std::move(hasSelfReference); info.narHash = narHash; info.narSize = sink.s->size(); info.ca = FixedOutputHash { diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 6c9b9a792..a40975982 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -65,7 +65,8 @@ struct CmdCopySigs : StorePathsCommand binary. */ if (info->narHash != info2->narHash || info->narSize != info2->narSize || - info->references != info2->references) + info->references != info2->references || + info->hasSelfReference != info2->hasSelfReference) continue; for (auto & sig : info2->sigs) From a9c0ea30bf81a42dfb7ccca04bb98649a6c34d07 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 17:59:27 +0000 Subject: [PATCH 03/61] Backport fix from #3754 branch --- src/libstore/path-info.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 27efe5ae9..a67c36bb6 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -36,10 +36,10 @@ struct PathReferences template StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const { - StorePathSet references { references }; + StorePathSet refs { references }; if (hasSelfReference) - references.insert(self); - return references; + refs.insert(self); + return refs; } template From 70ed47c1cb9d04a5a350cac664921a194d93d329 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 19:21:46 +0000 Subject: [PATCH 04/61] Fix some things in remote store --- src/libstore/remote-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f84b62f2e..912a3b70c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -445,7 +445,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - writeStorePaths(*this, sink, info.references); + writeStorePaths(*this, sink, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -464,7 +464,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; From 66834068432d316ee558717765851835ceec2dcc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 19:58:31 +0000 Subject: [PATCH 05/61] Fix nar info parsing --- src/libstore/nar-info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ef04bc859..0796de466 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -57,7 +57,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") From e61061c88e0dfcce9329ea9f0b041a35270dfa1a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 23:17:11 +0000 Subject: [PATCH 06/61] Remove stray tabs --- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d7aef3ea5..273455bae 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -386,7 +386,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - auto path = parseStorePath(readString(conn->from)); + auto path = parseStorePath(readString(conn->from)); SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1a28386ef..0fee5559f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -913,17 +913,17 @@ string showPaths(const PathSet & paths) StorePathSet ValidPathInfo::referencesPossiblyToSelf() const { - return PathReferences::referencesPossiblyToSelf(path); + return PathReferences::referencesPossiblyToSelf(path); } void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) { - return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); + return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); } void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) { - return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); + return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); } std::string ValidPathInfo::fingerprint(const Store & store) const From f8d562c0a7cef27c65d3cff96ad8ef384f05b331 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 7 Oct 2020 13:52:20 +0000 Subject: [PATCH 07/61] Use PathReferences more widely --- perl/lib/Nix/Store.xs | 10 +- src/libexpr/primops.cc | 16 ++- src/libexpr/value-to-xml.hh | 2 +- src/libfetchers/fetchers.cc | 8 +- src/libfetchers/tarball.cc | 16 ++- src/libstore/binary-cache-store.cc | 43 ++++-- src/libstore/build.cc | 30 ++-- src/libstore/content-address.cc | 24 +++- src/libstore/content-address.hh | 102 +++++++++++++- src/libstore/derivations.cc | 4 +- src/libstore/derivations.hh | 4 +- src/libstore/local-store.cc | 30 +++- src/libstore/nar-info.hh | 3 + src/libstore/path-info.hh | 47 +------ src/libstore/path.hh | 5 +- src/libstore/store-api.cc | 171 +++++++++++++++-------- src/libstore/store-api.hh | 14 +- src/libutil/args.cc | 2 +- src/libutil/error.hh | 3 +- src/libutil/fmt.hh | 2 +- src/libutil/tests/logging.cc | 2 +- src/libutil/types.hh | 1 + src/nix-prefetch-url/nix-prefetch-url.cc | 10 +- src/nix-store/nix-store.cc | 12 +- src/nix/add-to-store.cc | 16 ++- src/nix/bundle.cc | 2 +- src/nix/make-content-addressable.cc | 38 ++--- src/nix/profile.cc | 13 +- src/nix/verify.cc | 6 +- 29 files changed, 431 insertions(+), 205 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 599921151..ea8bbaf34 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -111,7 +111,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * arr = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { @@ -287,7 +287,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2b304aab0..c74b67658 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1045,7 +1045,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + { + .method = ingestionMethod, + .hash = h, + }, + {}, + }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputCAFixed { @@ -1764,7 +1770,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..c5f327bd8 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 49851f7bc..67bb77d3e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -198,7 +198,13 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for mutable input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); + return store.makeFixedOutputPath("source", FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + {}, + }); } std::string Input::getType() const diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ca49482a9..b3ee84810 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -71,14 +71,20 @@ DownloadFileResult downloadFile( dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + { + .name = name, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + {}, + }, + }, hashString(htSHA256, *sink.s), }; info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index d592f16dd..2d92e1c50 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -322,7 +322,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; @@ -412,14 +422,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -428,17 +444,26 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { textHash, references }); if (!repair && isValidPath(path)) return path; auto source = StringSource { s }; return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + { + .name = name, + .info = TextInfo { + { .hash = textHash }, + references, + }, + }, + nar.first, + }; info.narSize = nar.second; info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 97a832c6b..12ce6f2ec 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4056,25 +4056,24 @@ void DerivationGoal::registerOutputs() break; } auto got = caSink.finish().first; - auto refs = rewriteRefs(); HashModuloSink narSink { htSHA256, oldHashPart }; dumpPath(actualPath, narSink); auto narHashAndSize = narSink.finish(); ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.references, - refs.hasSelfReference), + worker.store, + { + .name = outputPathName(drv->name, outputName), + .info = FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, + }, + rewriteRefs(), + }, + }, narHashAndSize.first, }; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - static_cast &>(newInfo0) = refs; assert(newInfo0.ca); return newInfo0; @@ -4861,7 +4860,10 @@ void SubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA({ + .name = std::string { storePath.name() }, + .info = caWithoutRefs(*ca), + }); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -4891,7 +4893,7 @@ void SubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { + if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { auto info2 = std::make_shared(*info); info2->path = storePath; info = info2; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 90a3ad1f5..d68c60f4f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,6 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } + std::string makeFileIngestionPrefix(const FileIngestionMethod m) { switch (m) { @@ -16,9 +17,8 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; - default: - throw Error("impossible, caught both cases"); } + assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) @@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { - return "text:" + th.hash.to_string(Base32, true); + return "text:" + + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } @@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca) }, [](FixedOutputHash fsh) { return fsh.hash; - } + }, + }, ca); +} + +ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { + return std::visit(overloaded { + [&](TextHash h) -> ContentAddressWithReferences { + return TextInfo { h, {}}; + }, + [&](FixedOutputHash h) -> ContentAddressWithReferences { + return FixedOutputInfo { h, {}}; + }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..e15d76bd7 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -2,14 +2,20 @@ #include #include "hash.hh" +#include "path.hh" namespace nix { +/* + * Mini content address + */ + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; + struct TextHash { Hash hash; }; @@ -41,10 +47,6 @@ typedef std::variant< ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional ca); @@ -74,4 +76,96 @@ ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); std::string renderContentAddressMethod(ContentAddressMethod caMethod); +/* + * References set + */ + +template +struct PathReferences +{ + std::set references; + bool hasSelfReference = false; + + bool operator == (const PathReferences & other) const + { + return references == other.references + && hasSelfReference == other.hasSelfReference; + } + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); +}; + +template +StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet refs { references }; + if (hasSelfReference) + refs.insert(self); + return refs; +} + +template +void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} + +template +void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +// This matches the additional info that we need for makeTextPath +struct TextInfo : TextHash { + // References for the paths, self references disallowed + StorePathSet references; +}; + +struct FixedOutputInfo : FixedOutputHash { + // References for the paths + PathReferences references; +}; + +typedef std::variant< + TextInfo, + FixedOutputInfo +> ContentAddressWithReferences; + +ContentAddressWithReferences caWithoutRefs(const ContentAddress &); + +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + bool operator == (const StorePathDescriptor & other) const + { + return name == other.name; + // FIXME second field + } + + bool operator < (const StorePathDescriptor & other) const + { + return name < other.name; + // FIXME second field + } +}; + } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 07b4e772b..925a78083 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -27,8 +27,8 @@ std::optional DerivationOutput::path(const Store & store, std::string StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + outputPathName(drvName, outputName), + { hash, {} }); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d48266774..be19aa300 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -130,8 +130,8 @@ struct Derivation : BasicDerivation /* Return the underlying basic derivation but with these changes: - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. + 1. Input drvs are emptied, but the outputs of them that were used are + added directly to input sources. 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 18545f659..e6b02cce6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(doia.path, i.first); }, [&](DerivationOutputCAFixed dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} }); envHasRightPath(path, i.first); }, [&](DerivationOutputCAFloating _) { @@ -923,7 +923,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // recompute store path so that we can use a different store root if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + subPath = makeFixedOutputPathFromCA({ + .name = std::string { path.first.name() }, + .info = caWithoutRefs(*path.second), + }); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1164,7 +1167,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto desc = StorePathDescriptor { + name, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); @@ -1184,7 +1198,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1209,9 +1223,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, optimisePath(realPath); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1226,7 +1239,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 39ced76e5..fd37b85db 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 509f100d7..8c4791ac0 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -13,47 +13,6 @@ namespace nix { class Store; -template -struct PathReferences -{ - std::set references; - bool hasSelfReference = false; - - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; - void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); - void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); -}; - -template -StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const -{ - StorePathSet refs { references }; - if (hasSelfReference) - refs.insert(self); - return refs; -} - -template -void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) -{ - if (ref == self) - hasSelfReference = true; - else - references.insert(std::move(ref)); -} - -template -void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) -{ - if (refs.count(self)) - hasSelfReference = true; - refs.erase(self); - - references = refs; -} - struct SubstitutablePathInfo : PathReferences { @@ -68,7 +27,6 @@ struct ValidPathInfo : PathReferences { StorePath path; std::optional deriver; - // TODO document this Hash narHash; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown @@ -117,6 +75,8 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); + std::optional fullStorePathDescriptorOpt() const; + /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -143,6 +103,9 @@ struct ValidPathInfo : PathReferences ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + StorePathDescriptor && ca, Hash narHash); + virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index b03a0f69d..5f239ceb6 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once -#include "content-address.hh" +#include + #include "types.hh" namespace nix { @@ -64,8 +65,6 @@ typedef std::set StorePathSet; typedef std::vector StorePaths; typedef std::map OutputPathMap; -typedef std::map> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7041edbe5..5d63b8e3c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "thread-pool.hh" #include "json.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" @@ -163,63 +164,61 @@ StorePath Store::makeOutputPath(std::string_view id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ static std::string makeType( const Store & store, string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const PathReferences & references) { - for (auto & i : references) { + for (auto & i : references.references) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.hasSelfReference) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.empty()); + assert(info.references.references.size() == 0); + assert(!info.references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", PathReferences { info.references }), + info.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { - [&](TextHash th) { - return makeTextPath(name, th.hash, references); + [&](TextInfo ti) { + return makeTextPath(desc.name, ti); }, - [&](FixedOutputHash fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](FixedOutputInfo foi) { + return makeFixedOutputPath(desc.name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, desc.info); } @@ -229,14 +228,24 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + { + .method = method, + .hash = h, + }, + {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -326,11 +335,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -496,7 +514,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -509,11 +527,9 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref(info)); @@ -536,13 +552,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -554,7 +570,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -737,7 +753,8 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -799,7 +816,8 @@ std::map copyPaths(ref srcStore, ref dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -826,7 +844,8 @@ std::map copyPaths(ref srcStore, ref dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -947,19 +966,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const +{ + if (! ca) + return std::nullopt; + + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](TextHash th) { + TextInfo info { th }; + assert(!hasSelfReference); + info.references = references; + return ContentAddressWithReferences { info }; + }, + [&](FixedOutputHash foh) { + FixedOutputInfo info { foh }; + info.references = static_cast>(*this); + return ContentAddressWithReferences { info }; + }, + }, *ca), + }; +} + bool ValidPathInfo::isContentAddressed(const Store & store) const { - if (! ca) return false; + auto fullCaOpt = fullStorePathDescriptorOpt(); - auto caPath = std::visit(overloaded { - [&](TextHash th) { - assert(!hasSelfReference); - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference); - } - }, *ca); + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -997,6 +1034,26 @@ Strings ValidPathInfo::shortRefs() const } +ValidPathInfo::ValidPathInfo( + const Store & store, + StorePathDescriptor && info, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(info)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo ti) { + this->references = ti.references; + this->ca = TextHash { std::move(ti) }; + }, + [this](FixedOutputInfo foi) { + *(static_cast *>(this)) = foi.references; + this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + }, + }, std::move(info.info)); +} + + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 854446987..e6a6053a3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -170,6 +170,8 @@ struct BuildResult } }; +typedef std::map> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; @@ -313,17 +315,11 @@ public: StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 147602415..453fe60f9 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -244,7 +244,7 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f3babcbde..260ed3cf8 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -201,9 +201,8 @@ public: template SysError(const Args & ... args) - : Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 6e69bdce2..11dbef9db 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -103,7 +103,7 @@ class hintformat public: hintformat(const string &format) :fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 7e53f17c6..d33bd7c1f 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -370,7 +370,7 @@ namespace nix { // constructing without access violation. ErrPos ep(invalid); - + // assignment without access violation. ep = invalid; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 55d02bcf9..6c4c5ab74 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 377ae03a8..99cc0cdec 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -161,8 +161,14 @@ static int _main(int argc, char * * argv) std::optional storePath; if (args.size() == 2) { expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); if (store->isValidPath(*storePath)) hash = *expectedHash; else diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9092dbd80..7981bbbdd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -195,10 +195,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -209,7 +209,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + {}, + }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 023ffa4ed..86616d66b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,14 +69,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + StorePathDescriptor { + .name = *namePart, + .info = FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + {}, + }, + }, narHash, }; info.narSize = sink.s->size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fc41da9e4..510df7504 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -91,7 +91,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 7737f6d91..7695c98f8 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + PathReferences refs; + refs.hasSelfReference = oldInfo->hasSelfReference; for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + refs.references.insert(std::move(replacement)); } *sink.s = rewriteStrings(*sink.s, rewrites); @@ -78,16 +74,20 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + *store, + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + std::move(refs), + }, + }, narHash, }; - info.references = std::move(references); - info.hasSelfReference = std::move(hasSelfReference); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 7ce4dfe4c..41a4857fc 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -130,12 +130,21 @@ struct ProfileManifest auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + StorePathDescriptor { + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + { references }, + }, + }, narHash, }; info.references = std::move(references); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; auto source = StringSource { *sink.s }; store->addToStore(info, source); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 26f755fd9..d189a2fd3 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); From 39c11c5c01de9c18bf1b0bc3928fc28393fd0ca9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 13 Oct 2020 03:30:14 +0000 Subject: [PATCH 08/61] Organize content-address.hh a bit better --- src/libstore/content-address.hh | 53 ++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index e15d76bd7..126244ab5 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -7,7 +7,7 @@ namespace nix { /* - * Mini content address + * Content addressing method */ enum struct FileIngestionMethod : uint8_t { @@ -15,6 +15,34 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; +/* + We only have one way to hash text with references, so this is single-value + type is only useful in std::variant. +*/ +struct TextHashMethod { }; + +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; + +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(const FileIngestionMethod m); + + +typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > ContentAddressMethod; + +ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + +std::string renderContentAddressMethod(ContentAddressMethod caMethod); + +/* + * Mini content address + */ struct TextHash { Hash hash; @@ -43,10 +71,6 @@ typedef std::variant< FixedOutputHash // for path computed by makeFixedOutputPath > ContentAddress; -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional ca); @@ -57,25 +81,6 @@ std::optional parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); -/* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; -}; - -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; - -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); - -std::string renderContentAddressMethod(ContentAddressMethod caMethod); - /* * References set */ From 10e81bf871551901ff0383bdede0f79325e93867 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 02:21:28 +0000 Subject: [PATCH 09/61] Fix conditions for ca-references --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e6b02cce6..36ef7acf7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1071,7 +1071,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); // text hashing has long been allowed to have non-self-references because it is used for drv files. - if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && info.hasSelfReference)) + if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && !info.hasSelfReference)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash From 2c21cb672043fcf3c3fd19f89618b37693c0dc62 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 22:40:40 +0000 Subject: [PATCH 10/61] Fill in missing comparison operators for content addresses --- src/libstore/content-address.hh | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 126244ab5..acdb4f023 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -3,6 +3,7 @@ #include #include "hash.hh" #include "path.hh" +#include "comparator.hh" namespace nix { @@ -46,6 +47,8 @@ std::string renderContentAddressMethod(ContentAddressMethod caMethod); struct TextHash { Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; /// Pair of a hash, and how the file system was ingested @@ -53,6 +56,8 @@ struct FixedOutputHash { FileIngestionMethod method; Hash hash; std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; /* @@ -91,17 +96,13 @@ struct PathReferences std::set references; bool hasSelfReference = false; - bool operator == (const PathReferences & other) const - { - return references == other.references - && hasSelfReference == other.hasSelfReference; - } - /* Functions to view references + hasSelfReference as one set, mainly for compatibility's sake. */ StorePathSet referencesPossiblyToSelf(const Ref & self) const; void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(PathReferences, me->references, me->hasSelfReference); }; template @@ -142,11 +143,15 @@ void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set struct TextInfo : TextHash { // References for the paths, self references disallowed StorePathSet references; + + GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); }; struct FixedOutputInfo : FixedOutputHash { // References for the paths PathReferences references; + + GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); }; typedef std::variant< @@ -160,17 +165,7 @@ struct StorePathDescriptor { std::string name; ContentAddressWithReferences info; - bool operator == (const StorePathDescriptor & other) const - { - return name == other.name; - // FIXME second field - } - - bool operator < (const StorePathDescriptor & other) const - { - return name < other.name; - // FIXME second field - } + GENERATE_CMP(StorePathDescriptor, me->name, me->info); }; } From 13c669105ca93d28ca1a78321f07fd4ddbb445b1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 22:25:21 +0000 Subject: [PATCH 11/61] Slight cleanups --- src/libstore/content-address.cc | 2 +- src/libstore/content-address.hh | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e4ba855d5..2e6c435ce 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -10,7 +10,7 @@ std::string FixedOutputHash::printMethodAlgo() const } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) +std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index acdb4f023..a275800f9 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -11,17 +11,16 @@ namespace nix { * Content addressing method */ +/* We only have one way to hash text with references, so this is a single-value + type, mainly useful with std::variant. +*/ +struct TextHashMethod : std::monostate { }; + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; -/* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; - struct FixedOutputHashMethod { FileIngestionMethod fileIngestionMethod; HashType hashType; @@ -29,9 +28,13 @@ struct FixedOutputHashMethod { /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); +std::string makeFileIngestionPrefix(FileIngestionMethod m); +/* Just the type of a content address. Combine with the hash itself, and we + have a `ContentAddress` as defined below. Combine that, in turn, with info + on references, and we have `ContentAddressWithReferences`, as defined + further below. */ typedef std::variant< TextHashMethod, FixedOutputHashMethod @@ -86,6 +89,7 @@ std::optional parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); + /* * References set */ From 8623143921f8683b88d46aaebe9f707e5b9a912b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 11:18:14 -0500 Subject: [PATCH 12/61] Make formatting consistent --- src/libstore/content-address.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index a275800f9..6be4be4c5 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -38,7 +38,7 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m); typedef std::variant< TextHashMethod, FixedOutputHashMethod - > ContentAddressMethod; +> ContentAddressMethod; ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); From 6a168254ce068c067259c913ee7d6ee2e0d1dc7e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 12:24:20 -0500 Subject: [PATCH 13/61] Use named field initialization for references --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 10 ++++++++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/store-api.cc | 22 +++++++++++---------- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 15 files changed, 34 insertions(+), 26 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9cb078660..3ccd3c722 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -299,7 +299,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - {}, + .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8e9e5630d..8a19eab8f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1249,7 +1249,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * .method = method, .hash = h, }, - {}, + .references = {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 4181f0b7d..560a086f0 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -235,7 +235,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - {} + .references = {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 735d9fc93..3936eadfe 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - {}, + .references = {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index d52d19797..155b86cc4 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -79,7 +79,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - {}, + .references = {}, }, }, hashString(htSHA256, sink.s), diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3bbf4c8ac..aac5e7b88 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - { + .references = { .references = references, .hasSelfReference = false, }, @@ -433,7 +433,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - { + .references = { .references = references, .hasSelfReference = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 110a6a301..3d8299bbf 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2482,7 +2482,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - rewriteRefs(), + .references = rewriteRefs(), }, }, Hash::dummy, diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 2e6c435ce..3b8a773b7 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -154,10 +154,16 @@ Hash getContentAddressHash(const ContentAddress & ca) ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { - return TextInfo { h, {}}; + return TextInfo { + h, + .references = {}, + }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { - return FixedOutputInfo { h, {}}; + return FixedOutputInfo { + h, + .references = {}, + }; }, }, ca); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a8f060768..9f3a6db24 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1426,7 +1426,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - { + .references = { .references = references, .hasSelfReference = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 9655a0555..d6b6e87c9 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -55,7 +55,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - std::move(refs), + .references = std::move(refs), }, }, Hash::dummy, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 592afebd8..4b89465e7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -232,7 +232,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - {}, + .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -442,7 +442,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - {}, + .references = {}, }, }, narHash, @@ -1270,16 +1270,18 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c return StorePathDescriptor { .name = std::string { path.name() }, .info = std::visit(overloaded { - [&](const TextHash & th) { - TextInfo info { th }; + [&](const TextHash & th) -> ContentAddressWithReferences { assert(!hasSelfReference); - info.references = references; - return ContentAddressWithReferences { info }; + return TextInfo { + th, + .references = references, + }; }, - [&](const FixedOutputHash & foh) { - FixedOutputInfo info { foh }; - info.references = static_cast>(*this); - return ContentAddressWithReferences { info }; + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + return FixedOutputInfo { + foh, + .references = static_cast>(*this), + }; }, }, *ca), }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 75fa08551..5cb5aa53a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - {}, + .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index c2494dc9f..0b58818c3 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -50,7 +50,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - {}, + .references = {}, }, }, narHash, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index aa302efc1..df9933d29 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - {}, + .references = {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 024849e3b..614a37eba 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -205,7 +205,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - { references }, + .references = { references }, }, }, narHash, From 9cfa78e58a92b4bf034867bc1296a200bdc3f12a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 12:26:15 -0500 Subject: [PATCH 14/61] Optimize `ValidPathInfo` construction a bit better --- src/libstore/store-api.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4b89465e7..cd48d616b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1340,13 +1340,13 @@ ValidPathInfo::ValidPathInfo( , narHash(narHash) { std::visit(overloaded { - [this](const TextInfo & ti) { - this->references = ti.references; - this->ca = TextHash { std::move(ti) }; + [this](TextInfo && ti) { + this->references = std::move(ti.references); + this->ca = std::move((TextHash &&) ti); }, - [this](const FixedOutputInfo & foi) { - *(static_cast *>(this)) = foi.references; - this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + [this](FixedOutputInfo && foi) { + *(static_cast *>(this)) = std::move(foi.references); + this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); } From 46e942ff9e65755689ee72f93846d7118e1b8d45 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 15:36:05 -0500 Subject: [PATCH 15/61] Do big rename to clean up code - `PathReferences` -> `References` - `PathReferences` -> `StoreReference` - `references` -> `others` - `hasSelfReference` -> `self` And get rid of silly subclassing --- src/libexpr/primops.cc | 7 ++- src/libstore/binary-cache-store.cc | 13 ++-- src/libstore/build/local-derivation-goal.cc | 14 ++--- src/libstore/build/substitution-goal.cc | 6 +- src/libstore/content-address.hh | 46 +------------- src/libstore/daemon.cc | 4 +- src/libstore/local-store.cc | 18 +++--- src/libstore/make-content-addressed.cc | 8 +-- src/libstore/misc.cc | 16 ++--- src/libstore/path-info.hh | 7 ++- src/libstore/remote-store.cc | 4 +- src/libstore/store-api.cc | 36 +++++------ src/libutil/reference-set.hh | 68 +++++++++++++++++++++ src/nix/profile.cc | 6 +- src/nix/sigs.cc | 3 +- src/nix/why-depends.cc | 2 +- 16 files changed, 146 insertions(+), 112 deletions(-) create mode 100644 src/libutil/reference-set.hh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a19eab8f..0113659d1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1535,7 +1535,8 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V StorePathSet refs; if (state.store->isInStore(path)) { try { - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + // FIXME: Are self references becoming non-self references OK? + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf(); } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1971,7 +1972,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->references; + refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } @@ -2010,7 +2011,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - {}, + .references = {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index aac5e7b88..aa5aafdbf 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,8 +180,9 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) + reads, but typically they'll already be cached. Note that + self-references are always valid. */ + for (auto & ref : info.references.others) try { queryPathInfo(ref); } catch (InvalidPath &) { @@ -314,8 +315,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .hash = nar.first, }, .references = { - .references = references, - .hasSelfReference = false, + .others = references, + .self = false, }, }, }, @@ -434,8 +435,8 @@ StorePath BinaryCacheStore::addToStore( .hash = h, }, .references = { - .references = references, - .hasSelfReference = false, + .others = references, + .self = false, }, }, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3d8299bbf..ff24bd088 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2421,26 +2421,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } }; - auto rewriteRefs = [&]() -> PathReferences { + auto rewriteRefs = [&]() -> StoreReferences { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - PathReferences res { - .hasSelfReference = false, + StoreReferences res { + .self = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; if (r == *scratchPath) { - res.hasSelfReference = true; + res.self = true; } else if (auto outputRewrite = get(outputRewrites, origHash)) { std::string newRef = *outputRewrite; newRef += '-'; newRef += name; - res.references.insert(StorePath { newRef }); + res.others.insert(StorePath { newRef }); } else { - res.references.insert(r); + res.others.insert(r); } } return res; @@ -2523,7 +2523,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - static_cast &>(newInfo0) = rewriteRefs(); + newInfo0.references = rewriteRefs(); return newInfo0; }, diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 307183505..36b0ea7a7 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -128,7 +128,7 @@ void PathSubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { + if (info->isContentAddressed(*sub) && info->references.empty()) { auto info2 = std::make_shared(*info); info2->path = storePath; info = info2; @@ -165,7 +165,7 @@ void PathSubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - for (auto & i : info->references) + for (auto & i : info->references.others) addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -187,7 +187,7 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references) + for (auto & i : info->references.others) assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 6be4be4c5..f8a4d5370 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,6 +4,7 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" +#include "reference-set.hh" namespace nix { @@ -94,48 +95,7 @@ Hash getContentAddressHash(const ContentAddress & ca); * References set */ -template -struct PathReferences -{ - std::set references; - bool hasSelfReference = false; - - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; - void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); - void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); - - GENERATE_CMP(PathReferences, me->references, me->hasSelfReference); -}; - -template -StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const -{ - StorePathSet refs { references }; - if (hasSelfReference) - refs.insert(self); - return refs; -} - -template -void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) -{ - if (ref == self) - hasSelfReference = true; - else - references.insert(std::move(ref)); -} - -template -void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) -{ - if (refs.count(self)) - hasSelfReference = true; - refs.erase(self); - - references = refs; -} +typedef References StoreReferences; /* * Full content address @@ -153,7 +113,7 @@ struct TextInfo : TextHash { struct FixedOutputInfo : FixedOutputHash { // References for the paths - PathReferences references; + StoreReferences references; GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6407e575a..605f871fc 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -758,7 +758,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.referencesPossiblyToSelf(path)); + worker_proto::write(*store, to, i->second.references.possiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -781,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.referencesPossiblyToSelf(i.first)); + worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9f3a6db24..b32953f3f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1157,11 +1157,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->references, - info->hasSelfReference, - info->deriver, - narInfo ? narInfo->fileSize : 0, - info->narSize, + .deriver = info->deriver, + .references = info->references, + .downloadSize = narInfo ? narInfo->fileSize : 0, + .narSize = info->narSize, }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { @@ -1228,7 +1227,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references; + return i == infos.end() ? StorePathSet() : i->second.references.others; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1427,8 +1426,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .hash = hash, }, .references = { - .references = references, - .hasSelfReference = false, + .others = references, + .self = false, }, }, }; @@ -1526,7 +1525,8 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - info.references = references; + // No self reference allowed with text-hashing + info.references.others = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index d6b6e87c9..5d7945eb1 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -27,15 +27,15 @@ std::map makeContentAddressed( StringMap rewrites; - PathReferences refs; - refs.hasSelfReference = oldInfo->hasSelfReference; - for (auto & ref : oldInfo->references) { + StoreReferences refs; + refs.self = oldInfo->references.self; + for (auto & ref : oldInfo->references.others) { auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? if (replacement != ref) { rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.references.insert(std::move(replacement)); + refs.others.insert(std::move(replacement)); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e0bb1fab0..87f85c3cc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -21,16 +21,16 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; StorePathSet referrers; queryReferrers(path, referrers); - for (auto& ref : referrers) + for (auto & ref : referrers) if (ref != path) res.insert(ref); if (includeOutputs) - for (auto& i : queryValidDerivers(path)) + for (auto & i : queryValidDerivers(path)) res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; @@ -40,11 +40,11 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto& ref : info->references) + for (auto & ref : info->references.others) res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); @@ -223,7 +223,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references) + for (auto & ref : info->second.references.others) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -241,7 +241,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references; + return queryPathInfo(path)->references.others; } catch (InvalidPath &) { return StorePathSet(); } @@ -297,7 +297,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); } } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 89886873a..9254835b7 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -14,20 +14,22 @@ namespace nix { class Store; -struct SubstitutablePathInfo : PathReferences +struct SubstitutablePathInfo { std::optional deriver; + StoreReferences references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; -struct ValidPathInfo : PathReferences +struct ValidPathInfo { StorePath path; std::optional deriver; Hash narHash; + StoreReferences references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -61,7 +63,6 @@ struct ValidPathInfo : PathReferences return path == i.path && narHash == i.narHash - && hasSelfReference == i.hasSelfReference && references == i.references; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 689ad3fbe..1f8098b85 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); + info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -426,7 +426,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); + info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cd48d616b..5490df292 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -169,13 +169,13 @@ StorePath Store::makeOutputPath(std::string_view id, static std::string makeType( const Store & store, std::string && type, - const PathReferences & references) + const StoreReferences & references) { - for (auto & i : references.references) { + for (auto & i : references.others) { type += ":"; type += store.printStorePath(i); } - if (references.hasSelfReference) type += ":self"; + if (references.self) type += ":self"; return std::move(type); } @@ -185,8 +185,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(info.references.references.size() == 0); - assert(!info.references.hasSelfReference); + assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" @@ -201,7 +200,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", PathReferences { info.references }), + makeType(*this, "text", StoreReferences { info.references }), info.hash, name); } @@ -311,7 +310,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references; + return info.references.others; }, [&](const StorePath & path) { @@ -816,7 +815,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info->references) + for (auto & j : info->referencesPossiblyToSelf()) s += printStorePath(j) + "\n"; } @@ -878,7 +877,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1231,17 +1230,17 @@ std::string showPaths(const PathSet & paths) StorePathSet ValidPathInfo::referencesPossiblyToSelf() const { - return PathReferences::referencesPossiblyToSelf(path); + return references.possiblyToSelf(path); } void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) { - return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); + return references.insertPossiblyToSelf(path, std::move(ref)); } void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) { - return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); + return references.setPossiblyToSelf(path, std::move(refs)); } std::string ValidPathInfo::fingerprint(const Store & store) const @@ -1271,16 +1270,16 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c .name = std::string { path.name() }, .info = std::visit(overloaded { [&](const TextHash & th) -> ContentAddressWithReferences { - assert(!hasSelfReference); + assert(!references.self); return TextInfo { th, - .references = references, + .references = references.others, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { return FixedOutputInfo { foh, - .references = static_cast>(*this), + .references = references, }; }, }, *ca), @@ -1341,11 +1340,14 @@ ValidPathInfo::ValidPathInfo( { std::visit(overloaded { [this](TextInfo && ti) { - this->references = std::move(ti.references); + this->references = { + .others = std::move(ti.references), + .self = false, + }; this->ca = std::move((TextHash &&) ti); }, [this](FixedOutputInfo && foi) { - *(static_cast *>(this)) = std::move(foi.references); + this->references = std::move(foi.references); this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh new file mode 100644 index 000000000..ac4a9994e --- /dev/null +++ b/src/libutil/reference-set.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "comparator.hh" + +#include + +namespace nix { + +template +struct References +{ + std::set others; + bool self = false; + + bool empty() const; + size_t size() const; + + /* Functions to view references + self as one set, mainly for + compatibility's sake. */ + std::set possiblyToSelf(const Ref & self) const; + void insertPossiblyToSelf(const Ref & self, Ref && ref); + void setPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(References, me->others, me->self); +}; + +template +bool References::empty() const +{ + return !self && others.empty(); +} + +template +size_t References::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +template +std::set References::possiblyToSelf(const Ref & selfRef) const +{ + std::set refs { others }; + if (self) + refs.insert(selfRef); + return refs; +} + +template +void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) +{ + if (ref == selfRef) + self = true; + else + others.insert(std::move(ref)); +} + +template +void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) +{ + if (refs.count(selfRef)) { + self = true; + refs.erase(selfRef); + } + + others = refs; +} + +} diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 614a37eba..8a0f06435 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -205,12 +205,14 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - .references = { references }, + .references = { + .others = std::move(references), + .self = false, + }, }, }, narHash, }; - info.references = std::move(references); info.narSize = sink.s.size(); StringSource source(sink.s); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index a08314a25..3d659d6d2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -63,8 +63,7 @@ struct CmdCopySigs : StorePathsCommand binary. */ if (info->narHash != info2->narHash || info->narSize != info2->narSize || - info->references != info2->references || - info->hasSelfReference != info2->hasSelfReference) + info->references != info2->references) continue; for (auto & sig : info2->sigs) diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 76125e5e4..33cd13600 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references, + .refs = store->queryPathInfo(path)->references.others, .dist = path == dependencyPath ? 0 : inf }); From 91617f80ec03ff4580a656310959ce2e31e0d177 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 16:00:10 -0500 Subject: [PATCH 16/61] Fix perl bindings --- perl/lib/Nix/Store.xs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 3ccd3c722..bdb4fa655 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -69,7 +69,7 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->referencesPossiblyToSelf()) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); From 2e7be46e73293f729358eefc5b464dcb7e2d76bf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 13 Jan 2023 15:06:07 -0500 Subject: [PATCH 17/61] Move new `ValidPathInfo` methods to path-info.cc We'll move the old ones separately, so as not to clutter the diff. --- src/libstore/path-info.cc | 15 +++++++++++++++ src/libstore/store-api.cc | 15 --------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 003685604..cb3077c61 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,6 +3,21 @@ namespace nix { +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return references.possiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return references.insertPossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return references.setPossiblyToSelf(path, std::move(refs)); +} + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5490df292..a4e98d66b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1228,21 +1228,6 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } -StorePathSet ValidPathInfo::referencesPossiblyToSelf() const -{ - return references.possiblyToSelf(path); -} - -void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) -{ - return references.insertPossiblyToSelf(path, std::move(ref)); -} - -void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) -{ - return references.setPossiblyToSelf(path, std::move(refs)); -} - std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) From b3d91239ae9f21a60057b278ceeff663fb786246 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 14 Jan 2023 16:38:43 -0500 Subject: [PATCH 18/61] Make `ValidPathInfo` have plain `StorePathSet` references like before This change can wait for another PR. --- perl/lib/Nix/Store.xs | 4 +- src/libexpr/primops.cc | 5 +- src/libstore/binary-cache-store.cc | 8 +-- src/libstore/build/local-derivation-goal.cc | 11 ++-- src/libstore/build/substitution-goal.cc | 10 +-- src/libstore/content-address.cc | 10 +++ src/libstore/content-address.hh | 11 +++- src/libstore/daemon.cc | 8 +-- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 6 +- src/libstore/local-store.cc | 10 ++- src/libstore/make-content-addressed.cc | 5 +- src/libstore/misc.cc | 11 ++-- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/path-info.cc | 48 ++++++--------- src/libstore/path-info.hh | 11 +--- src/libstore/remote-store.cc | 13 ++-- src/libstore/store-api.cc | 14 +++-- src/libutil/reference-set.hh | 68 --------------------- src/nix-store/dotgraph.cc | 2 +- src/nix-store/graphml.cc | 2 +- src/nix-store/nix-store.cc | 8 +-- src/nix/why-depends.cc | 2 +- 24 files changed, 109 insertions(+), 166 deletions(-) delete mode 100644 src/libutil/reference-set.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index bdb4fa655..f19fb20bf 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -69,7 +69,7 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->referencesPossiblyToSelf()) + for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -110,7 +110,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->referencesPossiblyToSelf()) + for (auto & i : info->references) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ae573cf4d..3b32625b1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1544,8 +1544,7 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V StorePathSet refs; if (state.store->isInStore(path)) { try { - // FIXME: Are self references becoming non-self references OK? - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf(); + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1980,7 +1979,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); + refs = state.store->queryPathInfo(storePath)->references; path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 087b37655..ac41add2c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,11 +180,11 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. Note that - self-references are always valid. */ - for (auto & ref : info.references.others) + reads, but typically they'll already be cached. */ + for (auto & ref : info.references) try { - queryPathInfo(ref); + if (ref != info.path) + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d96858fc0..bb4f92989 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2523,7 +2523,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - newInfo0.references = rewriteRefs(); + auto refs = rewriteRefs(); + newInfo0.references = std::move(refs.others); + if (refs.self) + newInfo0.references.insert(newInfo0.path); return newInfo0; }, @@ -2774,12 +2777,12 @@ void LocalDerivationGoal::checkOutputs(const std::mapsecond.narSize; - for (auto & ref : i->second.referencesPossiblyToSelf()) + for (auto & ref : i->second.references) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->referencesPossiblyToSelf()) + for (auto & ref : info->references) pathsLeft.push(ref); } } @@ -2819,7 +2822,7 @@ void LocalDerivationGoal::checkOutputs(const std::mapreferences.others) - addWaitee(worker.makePathSubstitutionGoal(i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -187,8 +188,9 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references.others) - assert(worker.store.isValidPath(i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3b8a773b7..a98e34cb8 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -151,6 +151,16 @@ Hash getContentAddressHash(const ContentAddress & ca) }, ca); } +bool StoreReferences::empty() const +{ + return !self && others.empty(); +} + +size_t StoreReferences::size() const +{ + return (self ? 1 : 0) + others.size(); +} + ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f8a4d5370..4a50bbee0 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,7 +4,6 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" -#include "reference-set.hh" namespace nix { @@ -95,7 +94,15 @@ Hash getContentAddressHash(const ContentAddress & ca); * References set */ -typedef References StoreReferences; +struct StoreReferences { + StorePathSet others; + bool self = false; + + bool empty() const; + size_t size() const; + + GENERATE_CMP(StoreReferences, me->self, me->others); +}; /* * Full content address diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 605f871fc..12596ba49 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -336,7 +336,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) + for (auto & i : store->queryPathInfo(path)->references) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -758,7 +758,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references.possiblyToSelf(path)); + worker_proto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -781,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first)); + worker_proto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -863,7 +863,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom {})); + info.references = worker_proto::read(*store, from, Phantom {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4adf51573..9875da909 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf()); + worker_proto::write(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(std::move(references)); + info.references = references; info.narSize = saved.s.size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 6a694f034..2c9dd2680 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -137,7 +137,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom {})); + info->references = worker_proto::read(*this, conn->from, Phantom {}); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -171,7 +171,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -200,7 +200,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b32953f3f..2d03d2d8b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -938,8 +938,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->insertReferencePossiblyToSelf( - parseStorePath(useQueryReferences.getStr(0))); + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -1206,7 +1205,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.referencesPossiblyToSelf()) + for (auto & j : i.references) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1227,7 +1226,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references.others; + return i == infos.end() ? StorePathSet() : i->second.references; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1525,8 +1524,7 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - // No self reference allowed with text-hashing - info.references.others = references; + info.references = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 5d7945eb1..09f615439 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -28,8 +28,9 @@ std::map makeContentAddressed( StringMap rewrites; StoreReferences refs; - refs.self = oldInfo->references.self; - for (auto & ref : oldInfo->references.others) { + for (auto & ref : oldInfo->references) { + if (ref == path) + refs.self = true; auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 70e97569a..da96dcebc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -40,8 +40,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references.others) - res.insert(ref); + for (auto & ref : info->references) + if (ref != path) + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) @@ -223,7 +224,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references.others) + for (auto & ref : info->second.references) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -241,7 +242,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references.others; + return queryPathInfo(path)->references; } catch (InvalidPath &) { return StorePathSet(); } @@ -297,7 +298,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8d4a21daf..3e0689534 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -248,7 +248,7 @@ public: narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->insertReferencePossiblyToSelf(StorePath(r)); + narInfo->references.insert(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index f54e8f1fc..071d8355e 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -63,7 +63,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) throw corrupt(); for (auto & r : refs) - insertReferencePossiblyToSelf(StorePath(r)); + references.insert(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 2972c0bbe..93f91e702 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -12,7 +12,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); + + concatStringsSep(",", store.printStorePathSet(references)); } @@ -30,16 +30,25 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c .name = std::string { path.name() }, .info = std::visit(overloaded { [&](const TextHash & th) -> ContentAddressWithReferences { - assert(!references.self); + assert(references.count(path) == 0); return TextInfo { th, - .references = references.others, + .references = references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } return FixedOutputInfo { foh, - .references = references, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, }; }, }, *ca), @@ -85,7 +94,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : referencesPossiblyToSelf()) + for (auto & r : references) refs.push_back(std::string(r.to_string())); return refs; } @@ -100,36 +109,19 @@ ValidPathInfo::ValidPathInfo( { std::visit(overloaded { [this](TextInfo && ti) { - this->references = { - .others = std::move(ti.references), - .self = false, - }; + this->references = std::move(ti.references); this->ca = std::move((TextHash &&) ti); }, [this](FixedOutputInfo && foi) { - this->references = std::move(foi.references); + this->references = std::move(foi.references.others); + if (foi.references.self) + this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); } -StorePathSet ValidPathInfo::referencesPossiblyToSelf() const -{ - return references.possiblyToSelf(path); -} - -void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) -{ - return references.insertPossiblyToSelf(path, std::move(ref)); -} - -void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) -{ - return references.setPossiblyToSelf(path, std::move(refs)); -} - - ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -141,7 +133,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(store, source, Phantom {})); + info.references = worker_proto::read(store, source, Phantom {}); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -162,7 +154,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, referencesPossiblyToSelf()); + worker_proto::write(store, sink, references); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 9254835b7..476df79c2 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -17,19 +17,20 @@ class Store; struct SubstitutablePathInfo { std::optional deriver; - StoreReferences references; + StorePathSet references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; + struct ValidPathInfo { StorePath path; std::optional deriver; Hash narHash; - StoreReferences references; + StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -81,12 +82,6 @@ struct ValidPathInfo /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; - void insertReferencePossiblyToSelf(StorePath && ref); - void setReferencesPossiblyToSelf(StorePathSet && refs); - static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8ea126c65..ff57a77ca 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -421,12 +421,11 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - auto path = parseStorePath(readString(conn->from)); - SubstitutablePathInfo & info { infos[path] }; + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -634,7 +633,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.referencesPossiblyToSelf()); + worker_proto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -645,7 +644,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); - assert(importedPaths.empty() == 0); // doesn't include possible self reference + assert(importedPaths.size() <= 1); } else { @@ -653,7 +652,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9446ad132..c39e50d14 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -200,7 +200,10 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", StoreReferences { info.references }), + makeType(*this, "text", StoreReferences { + .others = info.references, + .self = false, + }), info.hash, name); } @@ -310,7 +313,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references.others; + return info.references; }, [&](const StorePath & path) { @@ -815,7 +818,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info->referencesPossiblyToSelf()) + for (auto & j : info->references) s += printStorePath(j) + "\n"; } @@ -877,7 +880,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->referencesPossiblyToSelf()) + for (auto & ref : info->references) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1205,7 +1208,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.insertReferencePossiblyToSelf(store.parseStorePath(s)); + info.references.insert(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -1228,6 +1231,7 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh deleted file mode 100644 index ac4a9994e..000000000 --- a/src/libutil/reference-set.hh +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once - -#include "comparator.hh" - -#include - -namespace nix { - -template -struct References -{ - std::set others; - bool self = false; - - bool empty() const; - size_t size() const; - - /* Functions to view references + self as one set, mainly for - compatibility's sake. */ - std::set possiblyToSelf(const Ref & self) const; - void insertPossiblyToSelf(const Ref & self, Ref && ref); - void setPossiblyToSelf(const Ref & self, std::set && refs); - - GENERATE_CMP(References, me->others, me->self); -}; - -template -bool References::empty() const -{ - return !self && others.empty(); -} - -template -size_t References::size() const -{ - return (self ? 1 : 0) + others.size(); -} - -template -std::set References::possiblyToSelf(const Ref & selfRef) const -{ - std::set refs { others }; - if (self) - refs.insert(selfRef); - return refs; -} - -template -void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) -{ - if (ref == selfRef) - self = true; - else - others.insert(std::move(ref)); -} - -template -void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) -{ - if (refs.count(selfRef)) { - self = true; - refs.erase(selfRef); - } - - others = refs; -} - -} diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 36d774dca..577cadceb 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -56,7 +56,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { + for (auto & p : store->queryPathInfo(path)->references) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index d2eebca7a..425d61e53 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->referencesPossiblyToSelf()) { + for (auto & p : info->references) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5cb5aa53a..5b261ecc6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -263,7 +263,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); + auto sorted = store->topoSortPaths(info->references); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -344,7 +344,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) + for (auto & p : store->queryPathInfo(j)->references) paths.insert(p); } else if (query == qReferrers) { @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->referencesPossiblyToSelf()); + worker_proto::write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -964,7 +964,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom {})); + info.references = worker_proto::read(*store, in, Phantom {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 33cd13600..76125e5e4 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references.others, + .refs = store->queryPathInfo(path)->references, .dist = path == dependencyPath ? 0 : inf }); From 4540e7b940ca56db821fe7c7d7d79fafa488f55e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Jan 2023 12:58:11 -0500 Subject: [PATCH 19/61] Don't add `StorePathDescriptor` for now We don't need it yet, we can add it back later. --- src/libfetchers/tarball.cc | 14 ++--- src/libstore/binary-cache-store.cc | 50 +++++++--------- src/libstore/build/local-derivation-goal.cc | 14 ++--- src/libstore/build/substitution-goal.cc | 7 +-- src/libstore/content-address.hh | 7 --- src/libstore/local-store.cc | 35 +++++------ src/libstore/make-content-addressed.cc | 14 ++--- src/libstore/nar-info.hh | 4 +- src/libstore/path-info.cc | 64 ++++++++++----------- src/libstore/path-info.hh | 4 +- src/libstore/store-api.cc | 28 ++++----- src/libstore/store-api.hh | 2 +- src/nix/add-to-store.cc | 14 ++--- src/nix/profile.cc | 20 +++---- 14 files changed, 126 insertions(+), 151 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 155b86cc4..302046610 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -72,15 +72,13 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, res.data); ValidPathInfo info { *store, - { - .name = name, - .info = FixedOutputInfo { - { - .method = FileIngestionMethod::Flat, - .hash = hash, - }, - .references = {}, + name, + FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, }, + .references = {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ac41add2c..9058bb8b1 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -307,17 +307,15 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - { - .name = std::string { name }, - .info = FixedOutputInfo { - { - .method = method, - .hash = nar.first, - }, - .references = { - .others = references, - .self = false, - }, + name, + FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + .references = { + .others = references, + .self = false, }, }, nar.first, @@ -427,17 +425,15 @@ StorePath BinaryCacheStore::addToStore( return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - { - .name = std::string { name }, - .info = FixedOutputInfo { - { - .method = method, - .hash = h, - }, - .references = { - .others = references, - .self = false, - }, + name, + FixedOutputInfo { + { + .method = method, + .hash = h, + }, + .references = { + .others = references, + .self = false, }, }, nar.first, @@ -465,12 +461,10 @@ StorePath BinaryCacheStore::addTextToStore( return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - { - .name = std::string { name }, - .info = TextInfo { - { .hash = textHash }, - references, - }, + std::string { name }, + TextInfo { + { .hash = textHash }, + references, }, nar.first, }; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index bb4f92989..98f8cb061 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2475,15 +2475,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto got = caSink.finish().first; ValidPathInfo newInfo0 { worker.store, - { - .name = outputPathName(drv->name, outputName), - .info = FixedOutputInfo { - { - .method = outputHash.method, - .hash = got, - }, - .references = rewriteRefs(), + outputPathName(drv->name, outputName), + FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, }, + .references = rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 994cb4ac2..87fed495c 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -95,10 +95,9 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA({ - .name = std::string { storePath.name() }, - .info = caWithoutRefs(*ca), - }); + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + caWithoutRefs(*ca)); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 4a50bbee0..c49ab269f 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -132,11 +132,4 @@ typedef std::variant< ContentAddressWithReferences caWithoutRefs(const ContentAddress &); -struct StorePathDescriptor { - std::string name; - ContentAddressWithReferences info; - - GENERATE_CMP(StorePathDescriptor, me->name, me->info); -}; - } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2d03d2d8b..e55ccab84 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1136,10 +1136,9 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // Recompute store path so that we can use a different store root. if (path.second) { - subPath = makeFixedOutputPathFromCA({ - .name = std::string { path.first.name() }, - .info = caWithoutRefs(*path.second), - }); + subPath = makeFixedOutputPathFromCA( + path.first.name(), + caWithoutRefs(*path.second)); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1417,21 +1416,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); - auto desc = StorePathDescriptor { - std::string { name }, - FixedOutputInfo { - { - .method = method, - .hash = hash, - }, - .references = { - .others = references, - .self = false, - }, + ContentAddressWithReferences desc = FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + .references = { + .others = references, + .self = false, }, }; - auto dstPath = makeFixedOutputPathFromCA(desc); + auto dstPath = makeFixedOutputPathFromCA(name, desc); addTempRoot(dstPath); @@ -1475,7 +1471,12 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { *this, std::move(desc), narHash.first }; + ValidPathInfo info { + *this, + name, + std::move(desc), + narHash.first + }; info.narSize = narHash.second; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 09f615439..3ee64c77a 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -49,15 +49,13 @@ std::map makeContentAddressed( ValidPathInfo info { dstStore, - StorePathDescriptor { - .name = std::string { path.name() }, - .info = FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }, - .references = std::move(refs), + path.name(), + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, }, + .references = std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index f1e3aabbd..a4dccb397 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -16,8 +16,8 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) - : ValidPathInfo(store, std::move(ca), narHash) + NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + : ValidPathInfo(store, std::move(name), std::move(ca), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 93f91e702..5944afd06 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,48 +21,45 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullStorePathDescriptorOpt() const +std::optional ValidPathInfo::contentAddressWithReferenences() const { if (! ca) return std::nullopt; - return StorePathDescriptor { - .name = std::string { path.name() }, - .info = std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { - assert(references.count(path) == 0); - return TextInfo { - th, - .references = references, - }; - }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return FixedOutputInfo { - foh, - .references = { - .others = std::move(refs), - .self = hasSelfReference, - }, - }; - }, - }, *ca), - }; + return std::visit(overloaded { + [&](const TextHash & th) -> ContentAddressWithReferences { + assert(references.count(path) == 0); + return TextInfo { + th, + .references = references, + }; + }, + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return FixedOutputInfo { + foh, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + }, + }, *ca); } bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = fullStorePathDescriptorOpt(); + auto fullCaOpt = contentAddressWithReferenences(); if (! fullCaOpt) return false; - auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); + auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); bool res = caPath == path; @@ -102,9 +99,10 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - StorePathDescriptor && info, + std::string_view name, + ContentAddressWithReferences && ca, Hash narHash) - : path(store.makeFixedOutputPathFromCA(info)) + : path(store.makeFixedOutputPathFromCA(name, ca)) , narHash(narHash) { std::visit(overloaded { @@ -118,7 +116,7 @@ ValidPathInfo::ValidPathInfo( this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, - }, std::move(info.info)); + }, std::move(ca)); } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 476df79c2..663d94540 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -77,7 +77,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional fullStorePathDescriptorOpt() const; + std::optional contentAddressWithReferenences() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -100,7 +100,7 @@ struct ValidPathInfo ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const Store & store, - StorePathDescriptor && ca, Hash narHash); + std::string_view name, ContentAddressWithReferences && ca, Hash narHash); virtual ~ValidPathInfo() { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c39e50d14..3c0c26706 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -209,17 +209,17 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { [&](const TextInfo & ti) { - return makeTextPath(desc.name, ti); + return makeTextPath(name, ti); }, [&](const FixedOutputInfo & foi) { - return makeFixedOutputPath(desc.name, foi); + return makeFixedOutputPath(name, foi); } - }, desc.info); + }, ca); } @@ -437,15 +437,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, ValidPathInfo info { *this, - StorePathDescriptor { - std::string { name }, - FixedOutputInfo { - { - .method = method, - .hash = hash, - }, - .references = {}, + name, + FixedOutputInfo { + { + .method = method, + .hash = hash, }, + .references = {}, }, narHash, }; @@ -997,7 +995,8 @@ void copyStorePath( if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); info2->path = dstStore.makeFixedOutputPathFromCA( - info->fullStorePathDescriptorOpt().value()); + info->path.name(), + info->contentAddressWithReferenences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1110,7 +1109,8 @@ std::map copyPaths( auto storePathForDst = storePathForSrc; if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( - currentPathInfo.fullStorePathDescriptorOpt().value()); + currentPathInfo.path.name(), + currentPathInfo.contentAddressWithReferenences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d77aea338..2d252db84 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -216,7 +216,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 0b58818c3..81dbc09a6 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -43,15 +43,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - StorePathDescriptor { - .name = *namePart, - .info = FixedOutputInfo { - { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - }, - .references = {}, + std::move(*namePart), + FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), }, + .references = {}, }, narHash, }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index aac8e5c81..345505532 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -200,17 +200,15 @@ struct ProfileManifest ValidPathInfo info { *store, - StorePathDescriptor { - "profile", - FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, - .references = { - .others = std::move(references), - .self = false, - }, + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + .references = { + .others = std::move(references), + .self = false, }, }, narHash, From 974a983351283a644228b10731e4f9d2ff01e533 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 09:59:55 -0500 Subject: [PATCH 20/61] Shrink diff in two places Stuff crept in there. --- src/libstore/content-address.cc | 3 ++- src/libstore/make-content-addressed.cc | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a98e34cb8..a51646d0f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -17,8 +17,9 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; + default: + throw Error("impossible, caught both cases"); } - assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 3ee64c77a..ff9f5cdaa 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -31,12 +31,14 @@ std::map makeContentAddressed( for (auto & ref : oldInfo->references) { if (ref == path) refs.self = true; - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) { - rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.others.insert(std::move(replacement)); + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) { + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + refs.others.insert(std::move(replacement)); + } } } From 0983a0bd3050d02659f7c58555b8cbcfffed2c3b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:04:28 -0500 Subject: [PATCH 21/61] Shrink diff in one place --- src/libstore/make-content-addressed.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index ff9f5cdaa..42de79226 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -35,10 +35,9 @@ std::map makeContentAddressed( auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? - if (replacement != ref) { + if (replacement != ref) rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.others.insert(std::move(replacement)); - } + refs.others.insert(std::move(replacement)); } } From db759b1bc23c64b2aa6bdd0c5444a6d864488671 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:07:54 -0500 Subject: [PATCH 22/61] Undo style change `&` without space before is far more common on this codebase than I thought, so it is not worth changing just this one file. Maybe we will adopt a formatter someday but until then this is fine. --- src/libstore/misc.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1c187535d..b28768459 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -21,16 +21,16 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; StorePathSet referrers; queryReferrers(path, referrers); - for (auto & ref : referrers) + for (auto& ref : referrers) if (ref != path) res.insert(ref); if (includeOutputs) - for (auto & i : queryValidDerivers(path)) + for (auto& i : queryValidDerivers(path)) res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; @@ -40,12 +40,12 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references) + for (auto& ref : info->references) if (ref != path) res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); From 59d3175649a6bbdde76d1dfcf476c11392add827 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:09:25 -0500 Subject: [PATCH 23/61] Put back TODO I don't think the `narHash` is in need of documentation more than the other undocumented fields, but regardless this change has nothing to do with that field and so we should leave the comment as is. --- src/libstore/path-info.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 663d94540..35aced472 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,6 +29,7 @@ struct ValidPathInfo { StorePath path; std::optional deriver; + // TODO document this Hash narHash; StorePathSet references; time_t registrationTime = 0; From ee9eb83a842eb97d0180fd9d349d30ff27fdb485 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 11:25:56 -0500 Subject: [PATCH 24/61] Remove some designated initializers With the switch to C++20, the rules became more strict, and we can no longer initialize base classes. Make them comments instead. (BTW https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2287r1.html this offers some new syntax for this use-case. Hopefully this will be adopted and we can eventually use it.) --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 4 ++-- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index f19fb20bf..314183383 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -299,7 +299,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8b54c4477..4e2f92276 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1287,7 +1287,7 @@ drvName, Bindings * attrs, Value & v) .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", @@ -2103,7 +2103,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - .references = {}, + /* .references = */ {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e194462e4..69395ad3d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -240,7 +240,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - .references = {} + /* .references = */ {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 3936eadfe..dae4998f9 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - .references = {}, + /* .references = */ {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 302046610..b6f72bb1f 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -78,7 +78,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - .references = {}, + /* .references = */ {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9cb0f74f6..5617e2c42 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - .references = { + /* .references = */ { .others = references, .self = false, }, @@ -431,7 +431,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - .references = { + /* .references = */ { .others = references, .self = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8fdc9dce1..cfd5db3b4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2498,7 +2498,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - .references = rewriteRefs(), + /* .references = */ rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a51646d0f..39a31f0a0 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -167,13 +167,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { h, - .references = {}, + /* .references = */ {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { h, - .references = {}, + /* .references = */ {}, }; }, }, ca); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6f870dde..9d2ea7156 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1419,7 +1419,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - .references = { + /* .references = */ { .others = references, .self = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 42de79226..59a452918 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -56,7 +56,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - .references = std::move(refs), + /* .references = */ std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5944afd06..ff85b3932 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -31,7 +31,7 @@ std::optional ValidPathInfo::contentAddressWithRef assert(references.count(path) == 0); return TextInfo { th, - .references = references, + /* .references = */ references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { @@ -43,7 +43,7 @@ std::optional ValidPathInfo::contentAddressWithRef } return FixedOutputInfo { foh, - .references = { + /* .references = */ { .others = std::move(refs), .self = hasSelfReference, }, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3c0c26706..295ce4953 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -234,7 +234,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -443,7 +443,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - .references = {}, + /* .references = */ {}, }, narHash, }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6..28ddf2f4a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - .references = {}, + /* .references = */ {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6..5de1aebfc 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -49,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - .references = {}, + /* .references = */ {}, }, narHash, }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index df9933d29..bc270f66d 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - .references = {}, + /* .references = */ {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 345505532..e552e8975 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -206,7 +206,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - .references = { + /* .references = */ { .others = std::move(references), .self = false, }, From c36b584f8eb103afa152ef4304cf9fd5c3ebaaf0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 11:34:18 -0500 Subject: [PATCH 25/61] Fix typo in the method name --- src/libstore/path-info.cc | 4 ++-- src/libstore/path-info.hh | 2 +- src/libstore/store-api.cc | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ff85b3932..074b50818 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,7 +21,7 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::contentAddressWithReferenences() const +std::optional ValidPathInfo::contentAddressWithReferences() const { if (! ca) return std::nullopt; @@ -54,7 +54,7 @@ std::optional ValidPathInfo::contentAddressWithRef bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = contentAddressWithReferenences(); + auto fullCaOpt = contentAddressWithReferences(); if (! fullCaOpt) return false; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 35aced472..97eb6638b 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -78,7 +78,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional contentAddressWithReferenences() const; + std::optional contentAddressWithReferences() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 06d746a0b..73dcaf150 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -996,7 +996,7 @@ void copyStorePath( auto info2 = make_ref(*info); info2->path = dstStore.makeFixedOutputPathFromCA( info->path.name(), - info->contentAddressWithReferenences().value()); + info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1110,7 +1110,7 @@ std::map copyPaths( if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( currentPathInfo.path.name(), - currentPathInfo.contentAddressWithReferenences().value()); + currentPathInfo.contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) From 123b11ff83da0cbcef6e4aae276bfbdd7a183656 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 11:49:13 -0500 Subject: [PATCH 26/61] Clarify store path grammar and improve comment on `makeType` --- src/libstore/store-api.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 73dcaf150..2ff92c0e6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -99,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const silly, but it's done that way for compatibility). is the name of the output (usually, "out"). -

= base-16 representation of a SHA-256 hash of: +

= base-16 representation of a SHA-256 hash of + + = if = "text:...": the string written to the resulting store path - if = "source": + if = "source:...": the serialisation of the path from which this store path is copied, as returned by hashPath() if = "output:": @@ -164,8 +166,8 @@ StorePath Store::makeOutputPath(std::string_view id, /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ + hacky, but we can't put them in, say, (per the grammar above) + since that would be ambiguous. */ static std::string makeType( const Store & store, std::string && type, From 85bb865d200f04b73f183af722757c78d5a3be76 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 11:57:20 -0500 Subject: [PATCH 27/61] Revert "Remove some designated initializers" This reverts commit ee9eb83a842eb97d0180fd9d349d30ff27fdb485. --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 4 ++-- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index db733ce40..fca7607d3 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -300,7 +300,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - /* .references = */ {}, + .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 762ff8249..a54cca5ab 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1287,7 +1287,7 @@ drvName, Bindings * attrs, Value & v) .method = method, .hash = h, }, - /* .references = */ {}, + .references = {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", @@ -2103,7 +2103,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - /* .references = */ {}, + .references = {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index ffc6f5859..93592290f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -250,7 +250,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - /* .references = */ {} + .references = {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index dae4998f9..3936eadfe 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - /* .references = */ {}, + .references = {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b6f72bb1f..302046610 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -78,7 +78,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - /* .references = */ {}, + .references = {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5617e2c42..9cb0f74f6 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - /* .references = */ { + .references = { .others = references, .self = false, }, @@ -431,7 +431,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - /* .references = */ { + .references = { .others = references, .self = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 472188fea..765bb8f35 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2446,7 +2446,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - /* .references = */ rewriteRefs(), + .references = rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 39a31f0a0..a51646d0f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -167,13 +167,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { h, - /* .references = */ {}, + .references = {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { h, - /* .references = */ {}, + .references = {}, }; }, }, ca); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9d2ea7156..c6f870dde 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1419,7 +1419,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - /* .references = */ { + .references = { .others = references, .self = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 59a452918..42de79226 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -56,7 +56,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - /* .references = */ std::move(refs), + .references = std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 074b50818..2a03e9dfa 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -31,7 +31,7 @@ std::optional ValidPathInfo::contentAddressWithRef assert(references.count(path) == 0); return TextInfo { th, - /* .references = */ references, + .references = references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { @@ -43,7 +43,7 @@ std::optional ValidPathInfo::contentAddressWithRef } return FixedOutputInfo { foh, - /* .references = */ { + .references = { .others = std::move(refs), .self = hasSelfReference, }, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2ff92c0e6..1c01c9cd8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -236,7 +236,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - /* .references = */ {}, + .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -445,7 +445,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - /* .references = */ {}, + .references = {}, }, narHash, }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 28ddf2f4a..5b261ecc6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - /* .references = */ {}, + .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5de1aebfc..81dbc09a6 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -49,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - /* .references = */ {}, + .references = {}, }, narHash, }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index bc270f66d..df9933d29 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - /* .references = */ {}, + .references = {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 28c7fe32d..5505d8bdd 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -207,7 +207,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - /* .references = */ { + .references = { .others = std::move(references), .self = false, }, From d381248ec0847cacd918480e83a99287f814456a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 12:13:43 -0500 Subject: [PATCH 28/61] No inheritance for `TextInfo` and `FixedOutputInfo` --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/content-address.hh | 10 ++++++---- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 16 ++++++++-------- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 17 files changed, 33 insertions(+), 31 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index fca7607d3..bfe00d3e2 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -296,7 +296,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a54cca5ab..7af796aa6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1283,7 +1283,7 @@ drvName, Bindings * attrs, Value & v) auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, @@ -2099,7 +2099,7 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = *expectedHash, }, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 93592290f..f3dce2214 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -246,7 +246,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - { + .hash = { .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 3936eadfe..91db3a9eb 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -211,7 +211,7 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = *narHash, }, diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 302046610..96fe5faca 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -74,7 +74,7 @@ DownloadFileResult downloadFile( *store, name, FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Flat, .hash = hash, }, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9cb0f74f6..9eae8d534 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -309,7 +309,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = nar.first, }, @@ -427,7 +427,7 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 765bb8f35..87eac6e19 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2442,7 +2442,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() worker.store, outputPathName(drv->name, outputName), FixedOutputInfo { - { + .hash = { .method = outputHash.method, .hash = got, }, diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a51646d0f..d9a8a4535 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -166,13 +166,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { - h, + .hash = h, .references = {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { - h, + .hash = h, .references = {}, }; }, diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c49ab269f..9fae288d8 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -111,18 +111,20 @@ struct StoreReferences { */ // This matches the additional info that we need for makeTextPath -struct TextInfo : TextHash { +struct TextInfo { + TextHash hash; // References for the paths, self references disallowed StorePathSet references; - GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); + GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo : FixedOutputHash { +struct FixedOutputInfo { + FixedOutputHash hash; // References for the paths StoreReferences references; - GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); + GENERATE_CMP(FixedOutputInfo, me->hash, me->references); }; typedef std::variant< diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6f870dde..8b33b2da5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1415,7 +1415,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 42de79226..53fe04704 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,7 +52,7 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 2a03e9dfa..76cab63e0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -30,7 +30,7 @@ std::optional ValidPathInfo::contentAddressWithRef [&](const TextHash & th) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - th, + .hash = th, .references = references, }; }, @@ -42,7 +42,7 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - foh, + .hash = foh, .references = { .others = std::move(refs), .self = hasSelfReference, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1c01c9cd8..b8a77b324 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -184,15 +184,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash, name); + if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.hash.method) + + info.hash.hash.to_string(Base16, true) + ":"), name); } } @@ -200,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.type == htSHA256); + assert(info.hash.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash, + info.hash.hash, name); } @@ -232,7 +232,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); FixedOutputInfo caInfo { - { + .hash = { .method = method, .hash = h, }, @@ -441,7 +441,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6..735d6a592 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -216,7 +216,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6..16e48a39b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -45,7 +45,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand *store, std::move(*namePart), FixedOutputInfo { - { + .hash = { .method = std::move(ingestionMethod), .hash = std::move(hash), }, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index df9933d29..209517b21 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -68,7 +68,7 @@ std::tuple prefetchFile( if (expectedHash) { hashType = expectedHash->type; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - { + .hash = { .method = ingestionMethod, .hash = *expectedHash, }, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 5505d8bdd..04ac48f00 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -203,7 +203,7 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = narHash, }, From a6d00a7bfb18e7ec461ac1d54203cc628aca5c66 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Mar 2023 16:29:13 -0400 Subject: [PATCH 29/61] Fix warning --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9eae8d534..628e9b9db 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -450,7 +450,7 @@ StorePath BinaryCacheStore::addTextToStore( RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, TextInfo { textHash, references }); + auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) return path; From c51d554c933b5fe294da41fcdf5afe0d4f33f586 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Mar 2023 17:12:49 -0400 Subject: [PATCH 30/61] Use "raw pattern" for content address types We weren't because this ancient PR predated it! This is actually a new version of the pattern which addresses some issues identified in #7479. --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/content-address.cc | 47 +++++---- src/libstore/content-address.hh | 110 +++++++++++++------- src/libstore/daemon.cc | 10 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 8 +- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/path-info.cc | 6 +- src/libstore/remote-store.cc | 8 +- src/libstore/store-api.cc | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix/prefetch.cc | 2 +- 14 files changed, 118 insertions(+), 87 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index caa15ab04..4fb7aa9d8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2508,7 +2508,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* Check wanted hash */ const Hash & wanted = dof.hash.hash; assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); + auto got = newInfo0.ca->getHash(); if (wanted != got) { /* Throw an error after registering the path as valid. */ diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 87fed495c..190fb455a 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -97,7 +97,7 @@ void PathSubstitutionGoal::tryNext() if (ca) { subPath = sub->makeFixedOutputPathFromCA( std::string { storePath.name() }, - caWithoutRefs(*ca)); + ContentAddressWithReferences::withoutRefs(*ca)); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 64daea0d4..055b216db 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,7 +9,6 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } - std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { @@ -22,35 +21,35 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) } } -std::string renderContentAddress(ContentAddress ca) +std::string ContentAddress::render() const { return std::visit(overloaded { - [](TextHash & th) { + [](const TextHash & th) { return "text:" + th.hash.to_string(Base32, true); }, - [](FixedOutputHash & fsh) { + [](const FixedOutputHash & fsh) { return "fixed:" + makeFileIngestionPrefix(fsh.method) + fsh.hash.to_string(Base32, true); } - }, ca); + }, raw); } -std::string renderContentAddressMethod(ContentAddressMethod cam) +std::string ContentAddressMethod::render() const { return std::visit(overloaded { - [](TextHashMethod & th) { + [](const TextHashMethod & th) { return std::string{"text:"} + printHashType(htSHA256); }, - [](FixedOutputHashMethod & fshm) { + [](const FixedOutputHashMethod & fshm) { return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); } - }, cam); + }, raw); } -/* - Parses content address strings up to the hash. +/** + * Parses content address strings up to the hash. */ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) { @@ -94,7 +93,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); } -ContentAddress parseContentAddress(std::string_view rawCa) { +ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); @@ -112,10 +111,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), }); }, - }, caMethod); + }, caMethod.raw); } -ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -123,26 +122,28 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) return parseContentAddressMethodPrefix(asPrefixView); } -std::optional parseContentAddressOpt(std::string_view rawCaOpt) +std::optional ContentAddress::parseOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional() : parseContentAddress(rawCaOpt); + return rawCaOpt == "" + ? std::nullopt + : std::optional { ContentAddress::parse(rawCaOpt) }; }; std::string renderContentAddress(std::optional ca) { - return ca ? renderContentAddress(*ca) : ""; + return ca ? ca->render() : ""; } -Hash getContentAddressHash(const ContentAddress & ca) +const Hash & ContentAddress::getHash() const { return std::visit(overloaded { - [](const TextHash & th) { + [](const TextHash & th) -> auto & { return th.hash; }, - [](const FixedOutputHash & fsh) { + [](const FixedOutputHash & fsh) -> auto & { return fsh.hash; }, - }, ca); + }, raw); } bool StoreReferences::empty() const @@ -155,7 +156,7 @@ size_t StoreReferences::size() const return (self ? 1 : 0) + others.size(); } -ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { +ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { @@ -169,7 +170,7 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { .references = {}, }; }, - }, ca); + }, ca.raw); } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index d74d1ff4b..d1dd1256c 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -39,17 +39,16 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; -}; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. */ std::string makeFileIngestionPrefix(FileIngestionMethod m); +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; /** * An enumeration of all the ways we can serialize file system objects. @@ -59,14 +58,25 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m); * with info on references, and we have `ContentAddressWithReferences`, * as defined further below. */ -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod -> ContentAddressMethod; +struct ContentAddressMethod +{ + typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > Raw; -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + Raw raw; + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressMethod(auto &&... arg) + : raw(std::forward(arg)...) + { } + + static ContentAddressMethod parse(std::string_view rawCaMethod); + + std::string render() const; +}; -std::string renderContentAddressMethod(ContentAddressMethod caMethod); /* * Mini content address @@ -115,25 +125,41 @@ struct FixedOutputHash { * - ‘fixed:::’: For paths computed by * Store::makeFixedOutputPath() / Store::addToStore(). */ -typedef std::variant< - TextHash, - FixedOutputHash -> ContentAddress; +struct ContentAddress +{ + typedef std::variant< + TextHash, + FixedOutputHash + > Raw; -/** - * Compute the content-addressability assertion (ValidPathInfo::ca) for - * paths created by Store::makeFixedOutputPath() / Store::addToStore(). - */ -std::string renderContentAddress(ContentAddress ca); + Raw raw; + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddress(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Compute the content-addressability assertion (ValidPathInfo::ca) for + * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + */ + std::string render() const; + + static ContentAddress parse(std::string_view rawCa); + + static std::optional parseOpt(std::string_view rawCaOpt); + + const Hash & getHash() const; +}; std::string renderContentAddress(std::optional ca); -ContentAddress parseContentAddress(std::string_view rawCa); - -std::optional parseContentAddressOpt(std::string_view rawCaOpt); - -Hash getContentAddressHash(const ContentAddress & ca); +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ /** * A set of references to other store objects. @@ -167,12 +193,6 @@ struct StoreReferences { GENERATE_CMP(StoreReferences, me->self, me->others); }; -/* - * Full content address - * - * See the schema for store paths in store-api.cc - */ - // This matches the additional info that we need for makeTextPath struct TextInfo { TextHash hash; @@ -200,15 +220,25 @@ struct FixedOutputInfo { * * A ContentAddress without a Hash. */ -typedef std::variant< - TextInfo, - FixedOutputInfo -> ContentAddressWithReferences; +struct ContentAddressWithReferences +{ + typedef std::variant< + TextInfo, + FixedOutputInfo + > Raw; -/** - * Create a ContentAddressWithReferences from a mere ContentAddress, by - * assuming no references in all cases. - */ -ContentAddressWithReferences caWithoutRefs(const ContentAddress &); + Raw raw; + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressWithReferences(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Create a ContentAddressWithReferences from a mere ContentAddress, by + * assuming no references in all cases. + */ + static ContentAddressWithReferences withoutRefs(const ContentAddress &); +}; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 656ad4587..0169eef1a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -401,21 +401,21 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr); FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](TextHashMethod &) { + [&](const TextHashMethod &) { // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, - [&](FixedOutputHashMethod & fohm) { + [&](const FixedOutputHashMethod & fohm) { auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, - }, contentAddressMethod); + }, contentAddressMethod.raw); }(); logger->stopWork(); @@ -880,7 +880,7 @@ static void performOp(TunnelLogger * logger, ref store, info.references = worker_proto::read(*store, from, Phantom {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseContentAddressOpt(readString(from)); + info.ca = ContentAddress::parseOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 98322b045..a1c38d180 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -156,7 +156,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor throw Error("NAR hash is now mandatory"); info->narHash = Hash::parseAnyPrefixed(s); } - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = ContentAddress::parseOpt(readString(conn->from)); info->sigs = readStrings(conn->from); auto s = readString(conn->from); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e1c7e387a..b49d5462b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -944,7 +944,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = ContentAddress::parseOpt(s); /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); @@ -1150,7 +1150,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst if (path.second) { subPath = makeFixedOutputPathFromCA( path.first.name(), - caWithoutRefs(*path.second)); + ContentAddressWithReferences::withoutRefs(*path.second)); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1329,7 +1329,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&*info.ca)) { + if (auto foHash = std::get_if(&info.ca->raw)) { auto actualFoHash = hashCAPath( foHash->method, foHash->hash.type, @@ -1342,7 +1342,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, actualFoHash.hash.to_string(Base32, true)); } } - if (auto textHash = std::get_if(&*info.ca)) { + if (auto textHash = std::get_if(&info.ca->raw)) { auto actualTextHash = hashString(htSHA256, readFile(realPath)); if (textHash->hash != actualTextHash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 2645f468b..c7176d30f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -273,7 +273,7 @@ public: narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 071d8355e..274cd861c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -74,7 +74,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseContentAddressOpt(value); + ca = ContentAddress::parseOpt(value); } pos = eol + 1; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 76cab63e0..e60d7abe0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -49,7 +49,7 @@ std::optional ValidPathInfo::contentAddressWithRef }, }; }, - }, *ca); + }, ca->raw); } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -116,7 +116,7 @@ ValidPathInfo::ValidPathInfo( this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, - }, std::move(ca)); + }, std::move(ca).raw); } @@ -136,7 +136,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned if (format >= 16) { source >> info.ultimate; info.sigs = readStrings(source); - info.ca = parseContentAddressOpt(readString(source)); + info.ca = ContentAddress::parseOpt(readString(source)); } return info; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d24d83117..ac98e76d2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -44,7 +44,7 @@ void write(const Store & store, Sink & out, const StorePath & storePath) ContentAddress read(const Store & store, Source & from, Phantom _) { - return parseContentAddress(readString(from)); + return ContentAddress::parse(readString(from)); } void write(const Store & store, Sink & out, const ContentAddress & ca) @@ -134,7 +134,7 @@ void write(const Store & store, Sink & out, const std::optional & sto std::optional read(const Store & store, Source & from, Phantom> _) { - return parseContentAddressOpt(readString(from)); + return ContentAddress::parseOpt(readString(from)); } void write(const Store & store, Sink & out, const std::optional & caOpt) @@ -545,7 +545,7 @@ ref RemoteStore::addCAToStore( conn->to << wopAddToStore << name - << renderContentAddressMethod(caMethod); + << caMethod.render(); worker_proto::write(*this, conn->to, references); conn->to << repair; @@ -603,7 +603,7 @@ ref RemoteStore::addCAToStore( } } - }, caMethod); + }, caMethod.raw); auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fed38e2dd..78b0d907e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -221,7 +221,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA [&](const FixedOutputInfo & foi) { return makeFixedOutputPath(name, foi); } - }, ca); + }, ca.raw); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 26febb6e3..3d2dc49fd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -970,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.references = worker_proto::read(*store, in, Phantom {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseContentAddressOpt(readString(in)); + info.ca = ContentAddress::parseOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b06b8a320..56e7bbb6e 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -124,7 +124,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = getContentAddressHash(*info.ca); + hash = info.ca->getHash(); } return {storePath.value(), hash.value()}; From 5d56e2daf70788fae532d1875edbd4e9bdb5afef Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Apr 2023 16:52:23 -0400 Subject: [PATCH 31/61] Add comparison methods for content addresses --- src/libstore/content-address.hh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index d1dd1256c..737bf9a41 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -46,8 +46,10 @@ enum struct FileIngestionMethod : uint8_t { std::string makeFileIngestionPrefix(FileIngestionMethod m); struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; + FileIngestionMethod fileIngestionMethod; + HashType hashType; + + GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType); }; /** @@ -67,6 +69,8 @@ struct ContentAddressMethod Raw raw; + GENERATE_CMP(ContentAddressMethod, me->raw); + /* The moral equivalent of `using Raw::Raw;` */ ContentAddressMethod(auto &&... arg) : raw(std::forward(arg)...) @@ -134,6 +138,8 @@ struct ContentAddress Raw raw; + GENERATE_CMP(ContentAddress, me->raw); + /* The moral equivalent of `using Raw::Raw;` */ ContentAddress(auto &&... arg) : raw(std::forward(arg)...) @@ -229,6 +235,8 @@ struct ContentAddressWithReferences Raw raw; + GENERATE_CMP(ContentAddressWithReferences, me->raw); + /* The moral equivalent of `using Raw::Raw;` */ ContentAddressWithReferences(auto &&... arg) : raw(std::forward(arg)...) From b11ae93581065d7fa98c69d26601ef1ecf211c0f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 26 Dec 2022 18:27:12 +0100 Subject: [PATCH 32/61] remove incorrect reference the semantics are not explained in the referenced section any more, they have been moved to the documentation for common options in the new CLI [0]. [0]: 703d863a48f549b2626382eda407ffae779f8725 --- doc/manual/src/command-ref/opt-common.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index a23b87e4e..c94b6aef8 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -203,10 +203,9 @@ Most Nix commands accept the following command-line options: instead. - [`-I`](#opt-I) *path*\ - Add a path to the Nix expression search path. This option may be - given multiple times. See the `NIX_PATH` environment variable for - information on the semantics of the Nix search path. Paths added - through `-I` take precedence over `NIX_PATH`. + Add a path to the Nix expression search path. + This option may be given multiple times. + Paths added through `-I` take precedence over [`NIX_PATH`](./env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value*\ Set the Nix configuration option *name* to *value*. This overrides From 6e0b7109abb40ded327b15599b29f861d9acb3c9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 13:34:32 +0100 Subject: [PATCH 33/61] Move OpenSSL init to initLibUtil Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 22 +--------------------- src/libutil/hash.cc | 23 +++++++++++++++++++++++ src/libutil/util.cc | 4 ++++ src/libutil/util.hh | 3 +++ 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 37664c065..2ed310cba 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -115,22 +115,6 @@ std::string getArg(const std::string & opt, return *i; } - -#if OPENSSL_VERSION_NUMBER < 0x10101000L -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} -#endif - static std::once_flag dns_resolve_flag; static void preloadNSS() { @@ -177,11 +161,7 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif -#if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); -#endif + initLibUtil(); if (sodium_init() == -1) throw Error("could not initialise libsodium"); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 5735e4715..9df8bcfb4 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -16,6 +17,28 @@ namespace nix { +#if OPENSSL_VERSION_NUMBER < 0x10101000L +/* OpenSSL is not thread-safe by default - it will randomly crash + unless the user supplies a mutex locking function. So let's do + that. */ +static std::vector opensslLocks; + +static void opensslLockCallback(int mode, int type, const char * file, int line) +{ + if (mode & CRYPTO_LOCK) + opensslLocks[type].lock(); + else + opensslLocks[type].unlock(); +} +#endif + +void initOpenSSL() { +#if OPENSSL_VERSION_NUMBER < 0x10101000L + /* Initialise OpenSSL locking. */ + opensslLocks = std::vector(CRYPTO_num_locks()); + CRYPTO_set_locking_callback(opensslLockCallback); +#endif +} static size_t regularHashSize(HashType type) { switch (type) { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 843a10eab..0099f7ebc 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -47,6 +47,10 @@ extern char * * environ __attribute__((weak)); namespace nix { +void initLibUtil() { + initOpenSSL(); +} + std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 56160baaf..783a4a601 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -32,6 +32,9 @@ namespace nix { struct Sink; struct Source; +void initLibUtil(); + +void initOpenSSL(); /** * The system for which Nix is compiled. From a692c437298ad59004583f193ef3d73a378fd837 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 16:46:26 +0100 Subject: [PATCH 34/61] Move loadConfFile() to initLibStore Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. Using libstore without loading the config file is risky, as sqlite may then be misconfigured. See https://github.com/cachix/cachix/issues/475 --- perl/lib/Nix/Store.xs | 1 - src/libmain/shared.cc | 2 -- src/libstore/globals.cc | 3 +++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index de91dc28d..b3f192810 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -27,7 +27,6 @@ static ref store() if (!_store) { try { initLibStore(); - loadConfFile(); settings.lockCPU = false; _store = openStore(); } catch (Error & e) { diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 2ed310cba..6dd64c6c7 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -166,8 +166,6 @@ void initNix() if (sodium_init() == -1) throw Error("could not initialise libsodium"); - loadConfFile(); - startSignalHandlerThread(); /* Reset SIGCHLD to its default. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 823b4af74..b18525dd7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -291,6 +291,9 @@ void assertLibStoreInitialized() { } void initLibStore() { + + loadConfFile(); + initLibStoreDone = true; } From 969307671500cb6cb9c01ab91c1d815ebd6a644b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:09:35 +0100 Subject: [PATCH 35/61] Move initLibStore() immediately after initLibUtil() Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. The goal of this reordering is to make initLibStore self-sufficient in a following commit. --- src/libmain/shared.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 6dd64c6c7..5e19bddb7 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -162,6 +162,7 @@ void initNix() #endif initLibUtil(); + initLibStore(); if (sodium_init() == -1) throw Error("could not initialise libsodium"); @@ -223,7 +224,6 @@ void initNix() #endif preloadNSS(); - initLibStore(); } From a58be394769fb174ee4b6ff5ce16744cf5806485 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:12:11 +0100 Subject: [PATCH 36/61] Move sodium_init() to initLibStore() Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 5 ----- src/libstore/globals.cc | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 5e19bddb7..8e693fd8d 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -28,8 +28,6 @@ #include -#include - namespace nix { @@ -164,9 +162,6 @@ void initNix() initLibUtil(); initLibStore(); - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - startSignalHandlerThread(); /* Reset SIGCHLD to its default. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index b18525dd7..1e66838c5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -13,6 +13,8 @@ #include +#include + namespace nix { @@ -292,6 +294,9 @@ void assertLibStoreInitialized() { void initLibStore() { + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); + loadConfFile(); initLibStoreDone = true; From e706ffa007120249deace149dc4ba7cacf2c8beb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:24:14 +0100 Subject: [PATCH 37/61] Move preloadNSS() from initNix to initLibStore It is required for the sandbox, which is a libstore responsibility; not just libmain. Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 42 --------------------------------------- src/libstore/globals.cc | 44 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 8e693fd8d..cbd80756e 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -20,11 +19,6 @@ #ifdef __linux__ #include #endif -#ifdef __GLIBC__ -#include -#include -#include -#endif #include @@ -113,41 +107,6 @@ std::string getArg(const std::string & opt, return *i; } -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { -#ifdef __GLIBC__ - /* On linux, glibc will run every lookup through the nss layer. - * That means every lookup goes, by default, through nscd, which acts as a local - * cache. - * Because we run builds in a sandbox, we also remove access to nscd otherwise - * lookups would leak into the sandbox. - * - * But now we have a new problem, we need to make sure the nss_dns backend that - * does the dns lookups when nscd is not available is loaded or available. - * - * We can't make it available without leaking nix's environment, so instead we'll - * load the backend, and configure nss so it does not try to run dns lookups - * through nscd. - * - * This is technically only used for builtins:fetch* functions so we only care - * about dns. - * - * All other platforms are unaffected. - */ - if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) - warn("unable to load nss_dns backend"); - // FIXME: get hosts entry from nsswitch.conf. - __nss_configure_lookup("hosts", "files dns"); -#endif - }); -} - static void sigHandler(int signo) { } @@ -218,7 +177,6 @@ void initNix() unsetenv("TMPDIR"); #endif - preloadNSS(); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1e66838c5..6848991a2 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,11 @@ #include +#ifdef __GLIBC__ +#include +#include +#include +#endif namespace nix { @@ -283,6 +289,42 @@ void initPlugins() settings.pluginFiles.pluginsLoaded = true; } +static void preloadNSS() +{ + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + static std::once_flag dns_resolve_flag; + + std::call_once(dns_resolve_flag, []() { +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) + warn("unable to load nss_dns backend"); + // FIXME: get hosts entry from nsswitch.conf. + __nss_configure_lookup("hosts", "files dns"); +#endif + }); +} + static bool initLibStoreDone = false; void assertLibStoreInitialized() { @@ -299,6 +341,8 @@ void initLibStore() { loadConfFile(); + preloadNSS(); + initLibStoreDone = true; } From 52d6ce6515ff1e8462b67b2adb1942477ce122f8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:35:28 +0100 Subject: [PATCH 38/61] Move macOS TMPDIR hack from initNix to initLibStore This code is bad. We shouldn't unset variables in programs whose children may need them. Fixing one issue at a time, so postponing. See https://github.com/NixOS/nix/issues/7731 Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 8 -------- src/libstore/globals.cc | 8 ++++++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index cbd80756e..2a7e09e65 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -169,14 +169,6 @@ void initNix() gettimeofday(&tv, 0); srandom(tv.tv_usec); - /* On macOS, don't use the per-session TMPDIR (as set e.g. by - sshd). This breaks build users because they don't have access - to the TMPDIR, in particular in ‘nix-store --serve’. */ -#if __APPLE__ - if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) - unsetenv("TMPDIR"); -#endif - } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 6848991a2..5a8825be5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -343,6 +343,14 @@ void initLibStore() { preloadNSS(); + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + unsetenv("TMPDIR"); +#endif + initLibStoreDone = true; } From 1107ea363f600f37152e2b144d03c4071c2a6b6b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:41:24 +0100 Subject: [PATCH 39/61] libmain: Clarify the lack of initLibExpr() Quote Why not initLibExpr()? initGC() is essentially that, but detectStackOverflow is not an instance of the init function concept, as it may have to be invoked more than once per process. Furthermore, renaming initGC to initLibExpr is more trouble than it's worth at this time. --- src/libmain/shared.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 2a7e09e65..a25865aad 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -156,7 +156,10 @@ void initNix() if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); #endif - /* Register a SIGSEGV handler to detect stack overflows. */ + /* Register a SIGSEGV handler to detect stack overflows. + Why not initLibExpr()? initGC() is essentially that, but + detectStackOverflow is not an instance of the init function concept, as + it may have to be invoked more than once per process. */ detectStackOverflow(); /* There is no privacy in the Nix system ;-) At least not for From 781d3dceb303d9fceabe9a39eae0f7f986e1adcc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:43:14 +0100 Subject: [PATCH 40/61] Move initLibUtil() from initNix to initLibStore libutil is a dependency of libstore, so it should always be initialized as such. libutil is also a dependency of libmain. Being explicit about this dependency might be good, but not worth the slight code complexity until the library structure gets more advanced. Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 1 - src/libstore/globals.cc | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a25865aad..56f47a4ac 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -118,7 +118,6 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - initLibUtil(); initLibStore(); startSignalHandlerThread(); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5a8825be5..3f944f024 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -336,6 +336,8 @@ void assertLibStoreInitialized() { void initLibStore() { + initLibUtil(); + if (sodium_init() == -1) throw Error("could not initialise libsodium"); From 2196fd1146aa077419a113059ced924a648f9766 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 18:38:54 +0100 Subject: [PATCH 41/61] libutil: Provide alternatives to startSignalHandlerThread How signals should be handled depends on what kind of process Nix is integrated into. The signal handler thread used by the stand-alone Nix commands / processes may not work well in the context of other runtime systems, such as those of Python, Perl, or Haskell. --- src/libutil/util.cc | 44 ++++++++++++++++++++++++++++++++++++++++++-- src/libutil/util.hh | 19 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 0099f7ebc..5c19dc737 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1748,13 +1748,39 @@ void triggerInterrupt() } static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} void startSignalHandlerThread() { updateWindowSize(); - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); + saveSignalMask(); sigset_t set; sigemptyset(&set); @@ -1771,6 +1797,20 @@ void startSignalHandlerThread() static void restoreSignals() { + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) throw SysError("restoring signals"); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 783a4a601..08993e1cf 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -448,6 +448,8 @@ void setStackSize(size_t stackSize); /** * Restore the original inherited Unix process context (such as signal * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). */ void restoreProcessContext(bool restoreMounts = true); @@ -817,9 +819,26 @@ class Callback; /** * Start a thread that handles various signals. Also block those signals * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). */ void startSignalHandlerThread(); +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + struct InterruptCallback { virtual ~InterruptCallback() { }; From 2445afd92c99ec0901a0e1a00fadda12aad15220 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 3 Feb 2023 18:07:47 +0100 Subject: [PATCH 42/61] Require openssl >= 1.1.1 Versions older this are sufficiently old that we don't want to support them, and they require extra support code. --- configure.ac | 2 +- src/libutil/hash.cc | 23 ----------------------- src/libutil/util.cc | 1 - src/libutil/util.hh | 2 -- 4 files changed, 1 insertion(+), 27 deletions(-) diff --git a/configure.ac b/configure.ac index f1f45f868..ba5756169 100644 --- a/configure.ac +++ b/configure.ac @@ -184,7 +184,7 @@ fi # Look for OpenSSL, a required dependency. FIXME: this is only (maybe) # used by S3BinaryCacheStore. -PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) +PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) # Look for libarchive. diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 9df8bcfb4..02bddc8d9 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -17,29 +17,6 @@ namespace nix { -#if OPENSSL_VERSION_NUMBER < 0x10101000L -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} -#endif - -void initOpenSSL() { -#if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); -#endif -} - static size_t regularHashSize(HashType type) { switch (type) { case htMD5: return md5HashSize; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5c19dc737..21d1c8dcd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -48,7 +48,6 @@ extern char * * environ __attribute__((weak)); namespace nix { void initLibUtil() { - initOpenSSL(); } std::optional getEnv(const std::string & key) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 08993e1cf..6ff9d2524 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -34,8 +34,6 @@ struct Source; void initLibUtil(); -void initOpenSSL(); - /** * The system for which Nix is compiled. */ From 1c0b680ef9ca9604ff993a9d693355254ddc5bf4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 5 Feb 2023 14:16:27 +0100 Subject: [PATCH 43/61] libstore: Remove lockCPU dead code Left over from 9747ea84b, https://github.com/NixOS/nix/pull/5821 --- perl/lib/Nix/Store.xs | 1 - src/libstore/globals.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index b3f192810..10a0c4067 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -27,7 +27,6 @@ static ref store() if (!_store) { try { initLibStore(); - settings.lockCPU = false; _store = openStore(); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 3f944f024..1b38e32fb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -49,7 +49,6 @@ Settings::Settings() , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); From ddebeb934a20225eec518520c96768bf00f0810a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 5 Feb 2023 14:16:27 +0100 Subject: [PATCH 44/61] libstore: Remove lockCPU dead code Left over from 9747ea84b, https://github.com/NixOS/nix/pull/5821 --- src/libstore/globals.hh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 63c7389da..c29ad5f89 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -478,11 +478,6 @@ public: )", {"env-keep-derivations"}}; - /** - * Whether to lock the Nix client and worker to the same CPU. - */ - bool lockCPU; - Setting sandboxMode{ this, #if __linux__ From bfc558c972aa8d6f5ef15a3e720bed964925ae32 Mon Sep 17 00:00:00 2001 From: Archit Gupta Date: Fri, 14 Apr 2023 11:33:38 -0700 Subject: [PATCH 45/61] Whitelist commit-lockfile-summary in flake nixConfig --- src/libexpr/flake/config.cc | 2 +- src/nix/flake.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 89ddbde7e..e89014862 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -31,7 +31,7 @@ static void writeTrustedList(const TrustedList & trustedList) void ConfigFile::apply() { - std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry"}; + std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lockfile-summary"}; for (auto & [name, value] : settings) { diff --git a/src/nix/flake.md b/src/nix/flake.md index d70f34eeb..965f6eb48 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -382,9 +382,9 @@ The following attributes are supported in `flake.nix`: * `nixConfig`: a set of `nix.conf` options to be set when evaluating any part of a flake. In the interests of security, only a small set of whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`, - `bash-prompt-suffix`, and `flake-registry`) are allowed to be set without - confirmation so long as `accept-flake-config` is not set in the global - configuration. + `bash-prompt-suffix`, `flake-registry`, and `commit-lockfile-summary`) + are allowed to be set without confirmation so long as `accept-flake-config` + is not set in the global configuration. ## Flake inputs From 9df7f3f5379ba79e6b40fb73bb91604cc7116c85 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 3 Mar 2021 01:11:09 +0000 Subject: [PATCH 46/61] Introduce `Worker::makeGoal` This takes a `DerivedPath` so the caller doesn't need to care about which sort of goal does what. --- src/libstore/build/entry-points.cc | 24 ++++-------------------- src/libstore/build/worker.cc | 15 +++++++++++++++ src/libstore/build/worker.hh | 12 ++++++++++-- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 2925fe3ca..7d725b881 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -10,16 +10,8 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); - } + for (auto & br : reqs) + goals.insert(worker.makeGoal(br, buildMode)); worker.run(goals); @@ -55,16 +47,8 @@ std::vector Store::buildPathsWithResults( Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); - } + for (const auto & br : reqs) + goals.insert(worker.makeGoal(br, buildMode)); worker.run(goals); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index f775f8486..6ad4a0e2b 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -92,6 +92,7 @@ std::shared_ptr Worker::makePathSubstitutionGoal(const Sto return goal; } + std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) { std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; @@ -104,6 +105,20 @@ std::shared_ptr Worker::makeDrvOutputSubstitutionGoal return goal; } + +GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) -> GoalPtr { + return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + }, + [&](const DerivedPath::Opaque & bo) -> GoalPtr { + return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); + }, + }, req.raw()); +} + + template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 48a1a27fa..bb51d641d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -181,7 +181,7 @@ public: */ /** - * derivation goal + * @ref DerivationGoal "derivation goal" */ private: std::shared_ptr makeDerivationGoalCommon( @@ -196,11 +196,19 @@ public: const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); /** - * substitution goal + * @ref SubstitutionGoal "substitution goal" */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + /** + * Make a goal corresponding to the `DerivedPath`. + * + * It will be a `DerivationGoal` for a `DerivedPath::Built` or + * a `SubstitutionGoal` for a `DerivedPath::Opaque`. + */ + GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal); + /** * Remove a dead goal. */ From 37fca662b0acef3c104a159709a394832e297dda Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 01:26:07 +0000 Subject: [PATCH 47/61] Make `KeyedBuildResult`, `BuildResult` like before, and fix bug another way In https://github.com/NixOS/nix/pull/6311#discussion_r834863823, I realized since derivation goals' wanted outputs can "grow" due to overlapping dependencies (See `DerivationGoal::addWantedOutputs`, called by `Worker::makeDerivationGoalCommon`), the previous bug fix had an unfortunate side effect of causing more pointless rebuilds. In paticular, we have this situation: 1. Goal made from `DerivedPath::Built { foo, {a} }`. 2. Goal gives on on substituting, starts building. 3. Goal made from `DerivedPath::Built { foo, {b} }`, in fact is just modified original goal. 4. Though the goal had gotten as far as building, so all outputs were going to be produced, `addWantedOutputs` no longer knows that and so the goal is flagged to be restarted. This might sound far-fetched with input-addressed drvs, where we usually basically have all our goals "planned out" before we start doing anything, but with CA derivation goals and especially RFC 92, where *drv resolution* means goals are created after some building is completed, it is more likely to happen. So the first thing to do was restore the clearing of `wantedOutputs` we used to do, and then filter the outputs in `buildPathsWithResults` to only get the ones we care about. But fix also has its own side effect in that the `DerivedPath` in the `BuildResult` in `DerivationGoal` cannot be trusted; it is merely the *first* `DerivedPath` for which this goal was originally created. To remedy this, I made `BuildResult` be like it was before, and instead made `KeyedBuildResult` be a subclass wit the path. Only `buildPathsWithResults` returns `KeyedBuildResult`s, everything else just becomes like it was before, where the "key" is unambiguous from context. I think separating the "primary key" field(s) from the other fields is good practical in general anyways. (I would like to do the same thing for `ValidPathInfo`.) Among other things, it allows constructions like `std::map` where doesn't contain duplicate keys and just precludes the possibility of those duplicate keys being out of sync. We might leverage the above someday to overload `buildPathsWithResults` to take a *set* of return a *map* per the above. ----- Unfortunately, we need to avoid C++20 strictness on designated initializers. (BTW https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2287r1.html this offers some new syntax for this use-case. Hopefully this will be adopted and we can eventually use it.) No having that yet, maybe it would be better to not make `KeyedBuildResult` a subclass to just avoid this. Co-authored-by: Robert Hensing --- src/libstore/build-result.hh | 16 ++++--- src/libstore/build/derivation-goal.cc | 30 ++++++++++--- src/libstore/build/entry-points.cc | 29 +++++++----- src/libstore/build/goal.cc | 23 ++++++++++ src/libstore/build/goal.hh | 16 ++++++- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/legacy-ssh-store.cc | 9 +--- src/libstore/remote-store.cc | 50 +++++++++++++-------- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 3 +- src/libstore/worker-protocol.hh | 1 + 11 files changed, 130 insertions(+), 51 deletions(-) diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 27d1a1b6c..e07296eab 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -83,11 +83,6 @@ struct BuildResult */ bool isNonDeterministic = false; - /** - * The derivation we built or the store path we substituted. - */ - DerivedPath path; - /** * For derivations, a mapping from the names of the wanted outputs * to actual paths. @@ -116,4 +111,15 @@ struct BuildResult } }; +/** + * A `BuildResult` together with its "primary key". + */ +struct KeyedBuildResult : BuildResult +{ + /** + * The derivation we built or the store path we substituted. + */ + DerivedPath path; +}; + } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 26faf8c8e..606f9fd48 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -330,6 +330,10 @@ void DerivationGoal::outputsSubstitutionTried() produced using a substitute. So we have to build instead. */ void DerivationGoal::gaveUpOnSubstitution() { + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs = OutputsSpec::All { }; + /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); if (useDerivation) @@ -570,8 +574,6 @@ void DerivationGoal::inputsRealised() build hook. */ state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); - - buildResult = BuildResult { .path = buildResult.path }; } void DerivationGoal::started() @@ -1452,12 +1454,28 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) { Goal::waiteeDone(waitee, result); - if (waitee->buildResult.success()) - if (auto bfd = std::get_if(&waitee->buildResult.path)) - for (auto & [output, realisation] : waitee->buildResult.builtOutputs) + if (!useDerivation) return; + auto & fullDrv = *dynamic_cast(drv.get()); + + auto * dg = dynamic_cast(&*waitee); + if (!dg) return; + + auto outputs = fullDrv.inputDrvs.find(dg->drvPath); + if (outputs == fullDrv.inputDrvs.end()) return; + + for (auto & outputName : outputs->second) { + auto buildResult = dg->getBuildResult(DerivedPath::Built { + .drvPath = dg->drvPath, + .outputs = OutputsSpec::Names { outputName }, + }); + if (buildResult.success()) { + for (auto & [output, realisation] : buildResult.builtOutputs) { inputDrvOutputs.insert_or_assign( - { bfd->drvPath, output.outputName }, + { dg->drvPath, output.outputName }, realisation.outPath); + } + } + } } } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 7d725b881..74eae0692 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -39,7 +39,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } } -std::vector Store::buildPathsWithResults( +std::vector Store::buildPathsWithResults( const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) @@ -47,15 +47,23 @@ std::vector Store::buildPathsWithResults( Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) - goals.insert(worker.makeGoal(br, buildMode)); + std::vector> state; + + for (const auto & req : reqs) { + auto goal = worker.makeGoal(req, buildMode); + goals.insert(goal); + state.push_back({req, goal}); + } worker.run(goals); - std::vector results; + std::vector results; - for (auto & i : goals) - results.push_back(i->buildResult); + for (auto & [req, goalPtr] : state) + results.emplace_back(KeyedBuildResult { + goalPtr->getBuildResult(req), + /* .path = */ req, + }); return results; } @@ -68,15 +76,14 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); - return goal->buildResult; + return goal->getBuildResult(DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All {}, + }); } catch (Error & e) { return BuildResult { .status = BuildResult::MiscFailure, .errorMsg = e.msg(), - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = OutputsSpec::All { }, - }, }; }; } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index d59b94797..13b2e509a 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -11,6 +11,29 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } +BuildResult Goal::getBuildResult(const DerivedPath & req) { + BuildResult res { buildResult }; + + if (auto pbp = std::get_if(&req)) { + auto & bp = *pbp; + + /* Because goals are in general shared between derived paths + that share the same derivation, we need to filter their + results to get back just the results we care about. + */ + + for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { + if (bp.outputs.contains(it->first.outputName)) + ++it; + else + it = res.builtOutputs.erase(it); + } + } + + return res; +} + + void addToWeakGoals(WeakGoals & goals, GoalPtr p) { if (goals.find(p) != goals.end()) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index f4bf6f38b..c0e12a2ed 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -81,11 +81,26 @@ struct Goal : public std::enable_shared_from_this */ ExitCode exitCode = ecBusy; +protected: /** * Build result. */ BuildResult buildResult; +public: + + /** + * Project a `BuildResult` with just the information that pertains + * to the given request. + * + * In general, goals may be aliased between multiple requests, and + * the stored `BuildResult` has information for the union of all + * requests. We don't want to leak what the other request are for + * sake of both privacy and determinism, and this "safe accessor" + * ensures we don't. + */ + BuildResult getBuildResult(const DerivedPath &); + /** * Exception containing an error message, if any. */ @@ -93,7 +108,6 @@ struct Goal : public std::enable_shared_from_this Goal(Worker & worker, DerivedPath path) : worker(worker) - , buildResult { .path = std::move(path) } { } virtual ~Goal() diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 58d6901d3..af937f6b1 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1335,7 +1335,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo result.rethrow(); } - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr) override diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d2ddbbe5f..7b40b27e0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -287,12 +287,7 @@ public: conn->to.flush(); - BuildResult status { - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = OutputsSpec::All { }, - }, - }; + BuildResult status; status.status = (BuildResult::Status) readInt(conn->from); conn->from >> status.errorMsg; @@ -330,7 +325,7 @@ public: conn->to.flush(); - BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } }; + BuildResult result; result.status = (BuildResult::Status) readInt(conn->from); if (!result.success()) { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b862902d1..734e6f27f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -125,10 +125,26 @@ void write(const Store & store, Sink & out, const DrvOutput & drvOutput) } -BuildResult read(const Store & store, Source & from, Phantom _) +KeyedBuildResult read(const Store & store, Source & from, Phantom _) { auto path = worker_proto::read(store, from, Phantom {}); - BuildResult res { .path = path }; + auto br = worker_proto::read(store, from, Phantom {}); + return KeyedBuildResult { + std::move(br), + /* .path = */ std::move(path), + }; +} + +void write(const Store & store, Sink & to, const KeyedBuildResult & res) +{ + worker_proto::write(store, to, res.path); + worker_proto::write(store, to, static_cast(res)); +} + + +BuildResult read(const Store & store, Source & from, Phantom _) +{ + BuildResult res; res.status = (BuildResult::Status) readInt(from); from >> res.errorMsg @@ -142,7 +158,6 @@ BuildResult read(const Store & store, Source & from, Phantom _) void write(const Store & store, Sink & to, const BuildResult & res) { - worker_proto::write(store, to, res.path); to << res.status << res.errorMsg @@ -865,7 +880,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod readInt(conn->from); } -std::vector RemoteStore::buildPathsWithResults( +std::vector RemoteStore::buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) @@ -880,7 +895,7 @@ std::vector RemoteStore::buildPathsWithResults( writeDerivedPaths(*this, conn, paths); conn->to << buildMode; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom> {}); + return worker_proto::read(*this, conn->from, Phantom> {}); } else { // Avoid deadlock. conn_.reset(); @@ -889,21 +904,25 @@ std::vector RemoteStore::buildPathsWithResults( // fails, but meh. buildPaths(paths, buildMode, evalStore); - std::vector results; + std::vector results; for (auto & path : paths) { std::visit( overloaded { [&](const DerivedPath::Opaque & bo) { - results.push_back(BuildResult { - .status = BuildResult::Substituted, - .path = bo, + results.push_back(KeyedBuildResult { + { + .status = BuildResult::Substituted, + }, + /* .path = */ bo, }); }, [&](const DerivedPath::Built & bfd) { - BuildResult res { - .status = BuildResult::Built, - .path = bfd, + KeyedBuildResult res { + { + .status = BuildResult::Built + }, + /* .path = */ bfd, }; OutputPathMap outputs; @@ -952,12 +971,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res { - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = OutputsSpec::All { }, - }, - }; + BuildResult res; res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 1c45f543e..a30466647 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -114,7 +114,7 @@ public: void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5bee272bf..9d6880de9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -92,6 +92,7 @@ enum BuildMode { bmNormal, bmRepair, bmCheck }; enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; +struct KeyedBuildResult; typedef std::map> StorePathCAMap; @@ -575,7 +576,7 @@ public: * case of a build/substitution error, this function won't throw an * exception, but return a BuildResult containing an error message. */ - virtual std::vector buildPathsWithResults( + virtual std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c7a6f8688..34b2fc17b 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -103,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, BuildResult); +MAKE_WORKER_PROTO(, KeyedBuildResult); MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(template, std::vector); From 0f2b5146c79895ac10362b6da56b535fc3d963a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Feb 2023 13:25:55 -0500 Subject: [PATCH 48/61] Make restarting state machines explicit If my memory is correct, @edolstra objected to modifying `wantedOutputs` upon falling back to doing a build (as we did before), because we should only modify it in response to new requests --- *actual* wants --- and not because we are "incidentally" building all the outptus beyond what may have been requested. That's a fair point, and the alternative is to replace the boolean soup with proper enums: Instead of modifying `wantedOuputs` som more, we'll modify `needsRestart` to indicate we are passed the need. --- src/libstore/build/derivation-goal.cc | 49 +++++++++++++++++++------ src/libstore/build/derivation-goal.hh | 52 ++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 18 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 606f9fd48..5bb664bff 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -145,8 +145,20 @@ void DerivationGoal::work() void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { auto newWanted = wantedOutputs.union_(outputs); - if (!newWanted.isSubsetOf(wantedOutputs)) - needRestart = true; + switch (needRestart) { + case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed: + if (!newWanted.isSubsetOf(wantedOutputs)) + needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed; + break; + case NeedRestartForMoreOutputs::OutputsAddedDoNeed: + /* No need to check whether we added more outputs, because a + restart is already queued up. */ + break; + case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed: + /* We are already building all outputs, so it doesn't matter if + we now want more. */ + break; + }; wantedOutputs = newWanted; } @@ -297,12 +309,29 @@ void DerivationGoal::outputsSubstitutionTried() In particular, it may be the case that the hole in the closure is an output of the current derivation, which causes a loop if retried. */ - if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; + { + bool substitutionFailed = + nrIncompleteClosure > 0 && + nrIncompleteClosure == nrFailed; + switch (retrySubstitution) { + case RetrySubstitution::NoNeed: + if (substitutionFailed) + retrySubstitution = RetrySubstitution::YesNeed; + break; + case RetrySubstitution::YesNeed: + // Should not be able to reach this state from here. + assert(false); + break; + case RetrySubstitution::AlreadyRetried: + debug("substitution failed again, but we already retried once. Not retrying again."); + break; + } + } nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - if (needRestart) { - needRestart = false; + if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { + needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; haveDerivation(); return; } @@ -330,9 +359,9 @@ void DerivationGoal::outputsSubstitutionTried() produced using a substitute. So we have to build instead. */ void DerivationGoal::gaveUpOnSubstitution() { - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs = OutputsSpec::All { }; + /* At this point we are building all outputs, so if more are wanted there + is no need to restart. */ + needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed; /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); @@ -455,8 +484,8 @@ void DerivationGoal::inputsRealised() return; } - if (retrySubstitution && !retriedSubstitution) { - retriedSubstitution = true; + if (retrySubstitution == RetrySubstitution::YesNeed) { + retrySubstitution = RetrySubstitution::AlreadyRetried; haveDerivation(); return; } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 3a6f0c2d9..9b5bd1805 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -78,22 +78,58 @@ struct DerivationGoal : public Goal */ std::map, StorePath> inputDrvOutputs; + /** + * See `needRestart`; just for that field. + */ + enum struct NeedRestartForMoreOutputs { + /** + * The goal state machine is progressing based on the current value of + * `wantedOutputs. No actions are needed. + */ + OutputsUnmodifedDontNeed, + /** + * `wantedOutputs` has been extended, but the state machine is + * proceeding according to its old value, so we need to restart. + */ + OutputsAddedDoNeed, + /** + * The goal state machine has progressed to the point of doing a build, + * in which case all outputs will be produced, so extensions to + * `wantedOutputs` no longer require a restart. + */ + BuildInProgressWillNotNeed, + }; + /** * Whether additional wanted outputs have been added. */ - bool needRestart = false; + NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; + + /** + * See `retrySubstitution`; just for that field. + */ + enum RetrySubstitution { + /** + * No issues have yet arose, no need to restart. + */ + NoNeed, + /** + * Something failed and there is an incomplete closure. Let's retry + * substituting. + */ + YesNeed, + /** + * We are current or have already retried substitution, and whether or + * not something goes wrong we will not retry again. + */ + AlreadyRetried, + }; /** * Whether to retry substituting the outputs after building the * inputs. This is done in case of an incomplete closure. */ - bool retrySubstitution = false; - - /** - * Whether we've retried substitution, in which case we won't try - * again. - */ - bool retriedSubstitution = false; + RetrySubstitution retrySubstitution = RetrySubstitution::NoNeed; /** * The derivation stored at drvPath. From 24866b71c40f0fcb5a601d90d4f87366fe626090 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Apr 2023 18:18:32 -0400 Subject: [PATCH 49/61] Introduce `SingleDrvOutputs` In many cases we are dealing with a collection of realisations, they are all outputs of the same derivation. In that case, we don't need "derivation hashes modulos" to be part of our map key, because the output names alone will be unique. Those hashes are still part of the realisation proper, so we aren't loosing any information, we're just "normalizing our schema" by narrowing the "primary key". Besides making our data model a bit "tighter" this allows us to avoid a double `for` loop in `DerivationGoal::waiteeDone`. The inner `for` loop was previously just to select the output we cared about without knowing its hash. Now we can just select the output by name directly. Note that neither protocol is changed as part of this: we are still transferring `DrvOutputs` over the wire for `BuildResult`s. I would only consider revising this once #6223 is merged, and we can mention protocol versions inside factored-out serialization logic. Until then it is better not change anything because it would come a the cost of code reuse. --- src/build-remote/build-remote.cc | 5 ++-- src/libcmd/installables.cc | 4 +-- src/libstore/build-result.hh | 2 +- src/libstore/build/derivation-goal.cc | 26 ++++++++-------- src/libstore/build/derivation-goal.hh | 12 ++++---- src/libstore/build/goal.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 6 ++-- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/daemon.cc | 5 +++- src/libstore/legacy-ssh-store.cc | 6 +++- src/libstore/realisation.hh | 33 ++++++++++++++++++++- src/libstore/remote-store.cc | 20 +++++++++---- src/nix-store/nix-store.cc | 5 +++- 13 files changed, 90 insertions(+), 38 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index cfc4baaca..ce9c7f45a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -311,8 +311,9 @@ connected: auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); - assert(result.builtOutputs.count(thisOutputId)); - auto newRealisation = result.builtOutputs.at(thisOutputId); + auto i = result.builtOutputs.find(outputName); + assert(i != result.builtOutputs.end()); + auto & newRealisation = i->second; missingRealisations.insert(newRealisation); missingPaths.insert(newRealisation.outPath); } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 32ae46d9f..0a2fe0073 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -593,8 +593,8 @@ std::vector, BuiltPathWithResult>> Installable::build std::visit(overloaded { [&](const DerivedPath::Built & bfd) { std::map outputs; - for (auto & path : buildResult.builtOutputs) - outputs.emplace(path.first.outputName, path.second.outPath); + for (auto & [outputName, realisation] : buildResult.builtOutputs) + outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { .path = BuiltPath::Built { bfd.drvPath, outputs }, .info = aux.info, diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index e07296eab..b7a56e791 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -87,7 +87,7 @@ struct BuildResult * For derivations, a mapping from the names of the wanted outputs * to actual paths. */ - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; /** * The start/stop times of the build (or one of the rounds, if it diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5bb664bff..a4bb94b0e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1013,7 +1013,7 @@ void DerivationGoal::resolvedFinished() auto resolvedDrv = *resolvedDrvGoal->drv; auto & resolvedResult = resolvedDrvGoal->buildResult; - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; if (resolvedResult.success()) { auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); @@ -1039,7 +1039,7 @@ void DerivationGoal::resolvedFinished() worker.store.printStorePath(drvPath), wantedOutput); auto realisation = [&]{ - auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + auto take1 = get(resolvedResult.builtOutputs, wantedOutput); if (take1) return *take1; /* The above `get` should work. But sateful tracking of @@ -1064,7 +1064,7 @@ void DerivationGoal::resolvedFinished() worker.store.registerDrvOutput(newRealisation); } outputPaths.insert(realisation.outPath); - builtOutputs.emplace(realisation.id, realisation); + builtOutputs.emplace(wantedOutput, realisation); } runPostBuildHook( @@ -1189,7 +1189,7 @@ HookReply DerivationGoal::tryBuildHook() } -DrvOutputs DerivationGoal::registerOutputs() +SingleDrvOutputs DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1351,7 +1351,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() } -std::pair DerivationGoal::checkPathValidity() +std::pair DerivationGoal::checkPathValidity() { if (!drv->type().isPure()) return { false, {} }; @@ -1364,7 +1364,7 @@ std::pair DerivationGoal::checkPathValidity() return static_cast(names); }, }, wantedOutputs.raw()); - DrvOutputs validOutputs; + SingleDrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { auto initialOutput = get(initialOutputs, i.first); @@ -1407,7 +1407,7 @@ std::pair DerivationGoal::checkPathValidity() } } if (info.wanted && info.known && info.known->isValid()) - validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); + validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path }); } // If we requested all the outputs, we are always fine. @@ -1431,7 +1431,7 @@ std::pair DerivationGoal::checkPathValidity() } -DrvOutputs DerivationGoal::assertPathValidity() +SingleDrvOutputs DerivationGoal::assertPathValidity() { auto [allValid, validOutputs] = checkPathValidity(); if (!allValid) @@ -1442,7 +1442,7 @@ DrvOutputs DerivationGoal::assertPathValidity() void DerivationGoal::done( BuildResult::Status status, - DrvOutputs builtOutputs, + SingleDrvOutputs builtOutputs, std::optional ex) { buildResult.status = status; @@ -1498,11 +1498,11 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { - for (auto & [output, realisation] : buildResult.builtOutputs) { + auto i = buildResult.builtOutputs.find(outputName); + if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { dg->drvPath, output.outputName }, - realisation.outPath); - } + { dg->drvPath, outputName }, + i->second.outPath); } } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 9b5bd1805..7033b7a58 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -253,7 +253,7 @@ struct DerivationGoal : public Goal * Check that the derivation outputs all exist and register them * as valid. */ - virtual DrvOutputs registerOutputs(); + virtual SingleDrvOutputs registerOutputs(); /** * Open a log file and a pipe to it. @@ -306,17 +306,17 @@ struct DerivationGoal : public Goal * Update 'initialOutputs' to determine the current status of the * outputs of the derivation. Also returns a Boolean denoting * whether all outputs are valid and non-corrupt, and a - * 'DrvOutputs' structure containing the valid and wanted + * 'SingleDrvOutputs' structure containing the valid and wanted * outputs. */ - std::pair checkPathValidity(); + std::pair checkPathValidity(); /** * Aborts if any output is not valid or corrupt, and otherwise - * returns a 'DrvOutputs' structure containing the wanted + * returns a 'SingleDrvOutputs' structure containing the wanted * outputs. */ - DrvOutputs assertPathValidity(); + SingleDrvOutputs assertPathValidity(); /** * Forcibly kill the child process, if any. @@ -329,7 +329,7 @@ struct DerivationGoal : public Goal void done( BuildResult::Status status, - DrvOutputs builtOutputs = {}, + SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); void waiteeDone(GoalPtr waitee, ExitCode result) override; diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 13b2e509a..ca7097a68 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -23,7 +23,7 @@ BuildResult Goal::getBuildResult(const DerivedPath & req) { */ for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { - if (bp.outputs.contains(it->first.outputName)) + if (bp.outputs.contains(it->first)) ++it; else it = res.builtOutputs.erase(it); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index af937f6b1..6cb483a9c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2174,7 +2174,7 @@ void LocalDerivationGoal::runChild() } -DrvOutputs LocalDerivationGoal::registerOutputs() +SingleDrvOutputs LocalDerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -2691,7 +2691,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; for (auto & [outputName, newInfo] : infos) { auto oldinfo = get(initialOutputs, outputName); @@ -2710,7 +2710,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() worker.store.registerDrvOutput(thisRealisation); } if (wantedOutputs.contains(outputName)) - builtOutputs.emplace(thisRealisation.id, thisRealisation); + builtOutputs.emplace(outputName, thisRealisation); } return builtOutputs; diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 42d32a31a..9acd7593d 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -237,7 +237,7 @@ struct LocalDerivationGoal : public DerivationGoal * Check that the derivation outputs all exist and register them * as valid. */ - DrvOutputs registerOutputs() override; + SingleDrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 63898f8dc..621a59c0a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -637,7 +637,10 @@ static void performOp(TunnelLogger * logger, ref store, to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; } if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - worker_proto::write(*store, to, res.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + worker_proto::write(*store, to, builtOutputs); } break; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7b40b27e0..6e50fe6cd 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -294,7 +294,11 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - status.builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); } return status; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index a18cf2aa8..3922d1267 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -13,9 +13,25 @@ namespace nix { class Store; +/** + * A general `Realisation` key. + * + * This is similar to a `DerivedPath::Opaque`, but the derivation is + * identified by its "hash modulo" instead of by its store path. + */ struct DrvOutput { - // The hash modulo of the derivation + /** + * The hash modulo of the derivation. + * + * Computed from the derivation itself for most types of + * derivations, but computed from the (fixed) content address of the + * output for fixed-output derivations. + */ Hash drvHash; + + /** + * The name of the output. + */ std::string outputName; std::string to_string() const; @@ -60,6 +76,21 @@ struct Realisation { GENERATE_CMP(Realisation, me->id, me->outPath); }; +/** + * Collection type for a single derivation's outputs' `Realisation`s. + * + * Since these are the outputs of a single derivation, we know the + * output names are unique so we can use them as the map key. + */ +typedef std::map SingleDrvOutputs; + +/** + * Collection type for multiple derivations' outputs' `Realisation`s. + * + * `DrvOutput` is used because in general the derivations are not all + * the same, so we need to identify firstly which derivation, and + * secondly which output of that derivation. + */ typedef std::map DrvOutputs; struct OpaquePath { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 734e6f27f..69e809a0f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -152,7 +152,11 @@ BuildResult read(const Store & store, Source & from, Phantom _) >> res.isNonDeterministic >> res.startTime >> res.stopTime; - res.builtOutputs = worker_proto::read(store, from, Phantom {}); + auto builtOutputs = worker_proto::read(store, from, Phantom {}); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); return res; } @@ -165,7 +169,10 @@ void write(const Store & store, Sink & to, const BuildResult & res) << res.isNonDeterministic << res.startTime << res.stopTime; - worker_proto::write(store, to, res.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + worker_proto::write(store, to, builtOutputs); } @@ -941,10 +948,10 @@ std::vector RemoteStore::buildPathsWithResults( queryRealisation(outputId); if (!realisation) throw MissingRealisation(outputId); - res.builtOutputs.emplace(realisation->id, *realisation); + res.builtOutputs.emplace(output, *realisation); } else { res.builtOutputs.emplace( - outputId, + output, Realisation { .id = outputId, .outPath = outputPath, @@ -979,7 +986,10 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); - res.builtOutputs = builtOutputs; + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); } return res; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7035e6a7b..74f255bee 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -935,7 +935,10 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { - worker_proto::write(*store, out, status.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + worker_proto::write(*store, out, builtOutputs); } break; From d0cf615cbbd1f4b15e04cb3696af109066096aef Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Apr 2023 14:27:26 +0200 Subject: [PATCH 50/61] add link to `nix-conf` setting Co-authored-by: John Ericson --- doc/manual/src/command-ref/opt-common.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index c94b6aef8..ad4e99e21 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -203,7 +203,7 @@ Most Nix commands accept the following command-line options: instead. - [`-I`](#opt-I) *path*\ - Add a path to the Nix expression search path. + Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). This option may be given multiple times. Paths added through `-I` take precedence over [`NIX_PATH`](./env-common.md#env-NIX_PATH). From f4119a67ccb12ab8988ccb5d0fd19660a20d61d0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Apr 2023 14:54:30 +0200 Subject: [PATCH 51/61] use @docroot@ link --- doc/manual/src/command-ref/opt-common.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index ad4e99e21..7a012250d 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -205,7 +205,7 @@ Most Nix commands accept the following command-line options: - [`-I`](#opt-I) *path*\ Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](./env-common.md#env-NIX_PATH). + Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value*\ Set the Nix configuration option *name* to *value*. This overrides From 537e8719f2ca8e18312bd8dcc37124fb1b25d4d3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 09:15:11 -0400 Subject: [PATCH 52/61] Explain various `.self = false,` Co-authored-by: Robert Hensing --- src/libstore/binary-cache-store.cc | 2 ++ src/libstore/local-store.cc | 1 + src/nix/profile.cc | 1 + 3 files changed, 4 insertions(+) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 628e9b9db..fcd763a9d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -315,6 +315,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, @@ -433,6 +434,7 @@ StorePath BinaryCacheStore::addToStore( }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0072a16dc..7fb312c37 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1328,6 +1328,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 5aa87a313..fd63b3519 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -209,6 +209,7 @@ struct ProfileManifest }, .references = { .others = std::move(references), + // profiles never refer to themselves .self = false, }, }, From 2c8475600d16e463a9c63aa76aee9f6152128f14 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Apr 2023 22:39:04 -0400 Subject: [PATCH 53/61] Fix some issues with experimental config settings Issues: 1. Features gated on disabled experimental settings should warn and be ignored, not silently succeed. 2. Experimental settings in the same config "batch" (file or env var) as the enabling of the experimental feature should work. 3. For (2), the order should not matter. These are analogous to the issues @roberth caught with my changes for arg handling, but they are instead for config handling. Co-authored-by: Robert Hensing --- src/libstore/globals.cc | 27 +++++----- src/libstore/globals.hh | 4 +- src/libutil/config-impl.hh | 71 +++++++++++++++++++++++++ src/libutil/config.cc | 94 ++++++++++++++++++++-------------- src/libutil/config.hh | 46 +++++++++++++++-- src/libutil/tests/config.cc | 2 + tests/experimental-features.sh | 60 +++++++++++++++++++--- 7 files changed, 238 insertions(+), 66 deletions(-) create mode 100644 src/libutil/config-impl.hh diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1b38e32fb..4c66d08ee 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -22,6 +22,9 @@ #include #endif +#include "config-impl.hh" + + namespace nix { @@ -192,18 +195,18 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smDisabled, false}, }); -template<> void BaseSetting::set(const std::string & str, bool append) +template<> SandboxMode BaseSetting::parse(const std::string & str) const { - if (str == "true") value = smEnabled; - else if (str == "relaxed") value = smRelaxed; - else if (str == "false") value = smDisabled; + if (str == "true") return smEnabled; + else if (str == "relaxed") return smRelaxed; + else if (str == "false") return smDisabled; else throw UsageError("option '%s' has invalid value '%s'", name, str); } -template<> bool BaseSetting::isAppendable() +template<> struct BaseSetting::trait { - return false; -} + static constexpr bool appendable = false; +}; template<> std::string BaseSetting::to_string() const { @@ -235,23 +238,23 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s }); } -void MaxBuildJobsSetting::set(const std::string & str, bool append) +unsigned int MaxBuildJobsSetting::parse(const std::string & str) const { - if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); + if (str == "auto") return std::max(1U, std::thread::hardware_concurrency()); else { if (auto n = string2Int(str)) - value = *n; + return *n; else throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); } } -void PluginFilesSetting::set(const std::string & str, bool append) +Paths PluginFilesSetting::parse(const std::string & str) const { if (pluginsLoaded) throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - BaseSetting::set(str, append); + return BaseSetting::parse(str); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d6c5d437a..609cf53b8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -26,7 +26,7 @@ struct MaxBuildJobsSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + unsigned int parse(const std::string & str) const override; }; struct PluginFilesSetting : public BaseSetting @@ -43,7 +43,7 @@ struct PluginFilesSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + Paths parse(const std::string & str) const override; }; const uint32_t maxIdsPerBuild = diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh new file mode 100644 index 000000000..b6cae5ec3 --- /dev/null +++ b/src/libutil/config-impl.hh @@ -0,0 +1,71 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * One only needs to include this when one is declaring a + * `BaseClass` setting, or as derived class of such an + * instantiation. + */ + +#include "config.hh" + +namespace nix { + +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting>::trait +{ + static constexpr bool appendable = true; +}; + +template +struct BaseSetting::trait +{ + static constexpr bool appendable = false; +}; + +template +bool BaseSetting::isAppendable() +{ + return trait::appendable; +} + +template<> void BaseSetting::appendOrSet(Strings && newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append); + +template +void BaseSetting::appendOrSet(T && newValue, bool append) +{ + static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type"); + assert(!append); + value = std::move(newValue); +} + +template +void BaseSetting::set(const std::string & str, bool append) +{ + if (experimentalFeatureSettings.isEnabled(experimentalFeature)) + appendOrSet(parse(str), append); + else { + assert(experimentalFeature); + warn("Ignoring setting '%s' because experimental feature '%s' is not enabled", + name, + showExperimentalFeature(*experimentalFeature)); + } +} + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index a42f3a849..085a884dc 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -3,6 +3,8 @@ #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "config-impl.hh" + #include namespace nix { @@ -80,6 +82,8 @@ void Config::getSettings(std::map & res, bool overridd void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { unsigned int pos = 0; + std::vector> parsedContents; + while (pos < contents.size()) { std::string line; while (pos < contents.size() && contents[pos] != '\n') @@ -125,8 +129,21 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string auto i = tokens.begin(); advance(i, 2); - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + parsedContents.push_back({ + name, + concatStringsSep(" ", Strings(i, tokens.end())), + }); }; + + // First apply experimental-feature related settings + for (auto & [name, value] : parsedContents) + if (name == "experimental-features" || name == "extra-experimental-features") + set(name, value); + + // Then apply other settings + for (auto & [name, value] : parsedContents) + if (name != "experimental-features" && name != "extra-experimental-features") + set(name, value); } void AbstractConfig::applyConfigFile(const Path & path) @@ -202,12 +219,6 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } -template -bool BaseSetting::isAppendable() -{ - return false; -} - template void BaseSetting::convertToArg(Args & args, const std::string & category) { @@ -231,9 +242,9 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) }); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> std::string BaseSetting::parse(const std::string & str) const { - value = str; + return str; } template<> std::string BaseSetting::to_string() const @@ -242,11 +253,11 @@ template<> std::string BaseSetting::to_string() const } template -void BaseSetting::set(const std::string & str, bool append) +T BaseSetting::parse(const std::string & str) const { static_assert(std::is_integral::value, "Integer required."); if (auto n = string2Int(str)) - value = *n; + return *n; else throw UsageError("setting '%s' has invalid value '%s'", name, str); } @@ -258,12 +269,12 @@ std::string BaseSetting::to_string() const return std::to_string(value); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> bool BaseSetting::parse(const std::string & str) const { if (str == "true" || str == "yes" || str == "1") - value = true; + return true; else if (str == "false" || str == "no" || str == "0") - value = false; + return false; else throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); } @@ -291,16 +302,15 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & }); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> Strings BaseSetting::parse(const std::string & str) const { - auto ss = tokenizeString(str); - if (!append) value.clear(); - for (auto & s : ss) value.push_back(std::move(s)); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(Strings && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) value.push_back(std::move(s)); } template<> std::string BaseSetting::to_string() const @@ -308,16 +318,16 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringSet BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); - for (auto & s : tokenizeString(str)) - value.insert(s); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) + value.insert(s); } template<> std::string BaseSetting::to_string() const @@ -325,21 +335,24 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting>::set(const std::string & str, bool append) +template<> std::set BaseSetting>::parse(const std::string & str) const { - if (!append) value.clear(); + std::set res; for (auto & s : tokenizeString(str)) { auto thisXpFeature = parseExperimentalFeature(s); if (thisXpFeature) - value.insert(thisXpFeature.value()); + res.insert(thisXpFeature.value()); else warn("unknown experimental feature '%s'", s); } + return res; } -template<> bool BaseSetting>::isAppendable() +template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) + value.insert(s); } template<> std::string BaseSetting>::to_string() const @@ -350,20 +363,23 @@ template<> std::string BaseSetting>::to_string() c return concatStringsSep(" ", stringifiedXpFeatures); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringMap BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); + StringMap res; for (auto & s : tokenizeString(str)) { auto eq = s.find_first_of('='); if (std::string::npos != eq) - value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); + res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } + return res; } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && [k, v] : std::move(newValue)) + value.emplace(std::move(k), std::move(v)); } template<> std::string BaseSetting::to_string() const @@ -387,15 +403,15 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting>; -void PathSetting::set(const std::string & str, bool append) +Path PathSetting::parse(const std::string & str) const { if (str == "") { if (allowEmpty) - value = ""; + return ""; else throw UsageError("setting '%s' cannot be empty", name); } else - value = canonPath(str); + return canonPath(str); } bool GlobalConfig::set(const std::string & name, const std::string & value) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 162626791..2675baed7 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -215,8 +215,11 @@ protected: virtual void set(const std::string & value, bool append = false) = 0; - virtual bool isAppendable() - { return false; } + /** + * Whether the type is appendable; i.e. whether the `append` + * parameter to `set()` is allowed to be `true`. + */ + virtual bool isAppendable() = 0; virtual std::string to_string() const = 0; @@ -241,6 +244,23 @@ protected: const T defaultValue; const bool documentDefault; + /** + * Parse the string into a `T`. + * + * Used by `set()`. + */ + virtual T parse(const std::string & str) const; + + /** + * Append or overwrite `value` with `newValue`. + * + * Some types to do not support appending in which case `append` + * should never be passed. The default handles this case. + * + * @param append Whether to append or overwrite. + */ + virtual void appendOrSet(T && newValue, bool append); + public: BaseSetting(const T & def, @@ -268,9 +288,25 @@ public: template void setDefault(const U & v) { if (!overridden) value = v; } - void set(const std::string & str, bool append = false) override; + /** + * Require any experimental feature the setting depends on + * + * Uses `parse()` to get the value from `str`, and `appendOrSet()` + * to set it. + */ + void set(const std::string & str, bool append = false) override final; - bool isAppendable() override; + /** + * C++ trick; This is template-specialized to compile-time indicate whether + * the type is appendable. + */ + struct trait; + + /** + * Always defined based on the C++ magic + * with `trait` above. + */ + bool isAppendable() override final; virtual void override(const T & v) { @@ -336,7 +372,7 @@ public: options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + Path parse(const std::string & str) const override; Path operator +(const char * p) const { return value + p; } diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index f250e934e..886e70da5 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -82,6 +82,7 @@ namespace nix { TestSetting() : AbstractSetting("test", "test", {}) {} void set(const std::string & value, bool append) override {} std::string to_string() const override { return {}; } + bool isAppendable() override { return false; } }; Config config; @@ -90,6 +91,7 @@ namespace nix { ASSERT_FALSE(config.set("test", "value")); config.addSetting(&setting); ASSERT_TRUE(config.set("test", "value")); + ASSERT_FALSE(config.set("extra-test", "value")); } TEST(Config, withInitialValue) { diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh index 73554da8c..607bf0a8e 100644 --- a/tests/experimental-features.sh +++ b/tests/experimental-features.sh @@ -23,20 +23,64 @@ source common.sh # # Medium case, the configuration effects --help # grep_both_ways store gc --help -expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' -nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' +# Test settings that are gated on experimental features; the setting is ignored +# with a warning if the experimental feature is not enabled. The order of the +# `setting = value` lines in the configuration should not matter. + +# 'flakes' experimental-feature is disabled before, ignore and warn +NIX_CONFIG=' + experimental-features = nix-command + accept-flake-config = true +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "false" $TEST_ROOT/stdout +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is disabled after, ignore and warn +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "false" $TEST_ROOT/stdout +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is enabled before, process +NIX_CONFIG=' + experimental-features = nix-command flakes + accept-flake-config = true +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "true" $TEST_ROOT/stdout +grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is enabled after, process +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command flakes +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "true" $TEST_ROOT/stdout +grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr + +function exit_code_both_ways { + expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null + nix --experimental-features 'nix-command flakes' "$@" 1>/dev/null + + # Also, the order should not matter + expect 1 nix "$@" --experimental-features 'nix-command' 1>/dev/null + nix "$@" --experimental-features 'nix-command flakes' 1>/dev/null +} + +exit_code_both_ways show-config --flake-registry 'https://no' # Double check these are stable -nix --experimental-features '' --help -nix --experimental-features '' doctor --help -nix --experimental-features '' repl --help -nix --experimental-features '' upgrade-nix --help +nix --experimental-features '' --help 1>/dev/null +nix --experimental-features '' doctor --help 1>/dev/null +nix --experimental-features '' repl --help 1>/dev/null +nix --experimental-features '' upgrade-nix --help 1>/dev/null # These 3 arguments are currently given to all commands, which is wrong (as not # all care). To deal with fixing later, we simply make them require the # nix-command experimental features --- it so happens that the commands we wish # stabilizing to do not need them anyways. for arg in '--print-build-logs' '--offline' '--refresh'; do - nix --experimental-features 'nix-command' "$arg" --help - ! nix --experimental-features '' "$arg" --help + nix --experimental-features 'nix-command' "$arg" --help 1>/dev/null + expect 1 nix --experimental-features '' "$arg" --help 1>/dev/null done From d41e1bed5e1e1f87927ca1e8e6e1c1ad18b1ea7f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 09:41:39 -0400 Subject: [PATCH 54/61] Experimentally allow forcing `nix-daemon` trust; use this to test We finally test the status quo of remote build trust in a number of ways. We create a new experimental feature on `nix-daemon` to do so. PR #3921, which improves the situation with trustless remote building, will build upon these changes. This code / tests was pull out of there to make this, so everything is easier to review, and in particular we test before and after so the new behavior in that PR is readily apparent from the testsuite diff alone. --- src/libstore/daemon.cc | 2 + src/libutil/experimental-features.cc | 12 +++- src/libutil/experimental-features.hh | 1 + src/nix/daemon.cc | 63 ++++++++++++++----- tests/build-remote-trustless-after.sh | 2 + tests/build-remote-trustless-should-fail-0.sh | 29 +++++++++ tests/build-remote-trustless-should-pass-0.sh | 9 +++ tests/build-remote-trustless-should-pass-1.sh | 9 +++ tests/build-remote-trustless-should-pass-3.sh | 14 +++++ tests/build-remote-trustless.sh | 14 +++++ tests/local.mk | 4 ++ tests/nix-daemon-untrusting.sh | 3 + 12 files changed, 145 insertions(+), 17 deletions(-) create mode 100644 tests/build-remote-trustless-after.sh create mode 100644 tests/build-remote-trustless-should-fail-0.sh create mode 100644 tests/build-remote-trustless-should-pass-0.sh create mode 100644 tests/build-remote-trustless-should-pass-1.sh create mode 100644 tests/build-remote-trustless-should-pass-3.sh create mode 100644 tests/build-remote-trustless.sh create mode 100755 tests/nix-daemon-untrusting.sh diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0d7ec2af0..af9a76f1e 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1067,6 +1067,8 @@ void processConnection( opCount++; + debug("performing daemon worker op: %d", op); + try { performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); } catch (Error & e) { diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 5b4418714..bd1899662 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -189,6 +189,16 @@ constexpr std::array xpFeatureDetails = {{ runtime dependencies. )", }, + { + .tag = Xp::DaemonTrustOverride, + .name = "daemon-trust-override", + .description = R"( + Allow forcing trusting or not trusting clients with + `nix-daemon`. This is useful for testing, but possibly also + useful for various experiments with `nix-daemon --stdio` + networking. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 8ef66263a..3c00bc4e5 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -28,6 +28,7 @@ enum struct ExperimentalFeature AutoAllocateUids, Cgroups, DiscardReferences, + DaemonTrustOverride, }; /** diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 7ae7b4ea6..c1a91c63d 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -273,8 +273,12 @@ static std::pair authPeer(const PeerInfo & peer) /** * Run a server. The loop opens a socket and accepts new connections from that * socket. + * + * @param forceTrustClientOpt If present, force trusting or not trusted + * the client. Otherwise, decide based on the authentication settings + * and user credentials (from the unix domain socket). */ -static void daemonLoop() +static void daemonLoop(std::optional forceTrustClientOpt) { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -317,9 +321,18 @@ static void daemonLoop() closeOnExec(remote.get()); - PeerInfo peer = getPeerInfo(remote.get()); - auto [_trusted, user] = authPeer(peer); - auto trusted = _trusted; + PeerInfo peer { .pidKnown = false }; + TrustedFlag trusted; + std::string user; + + if (forceTrustClientOpt) + trusted = *forceTrustClientOpt; + else { + peer = getPeerInfo(remote.get()); + auto [_trusted, _user] = authPeer(peer); + trusted = _trusted; + user = _user; + }; printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), peer.pidKnown ? std::to_string(peer.pid) : "", @@ -410,38 +423,47 @@ static void forwardStdioConnection(RemoteStore & store) { * Unlike `forwardStdioConnection()` we do process commands ourselves in * this case, not delegating to another daemon. * - * @note `Trusted` is unconditionally passed because in this mode we - * blindly trust the standard streams. Limiting access to those is - * explicitly not `nix-daemon`'s responsibility. + * @param trustClient Whether to trust the client. Forwarded directly to + * `processConnection()`. */ -static void processStdioConnection(ref store) +static void processStdioConnection(ref store, TrustedFlag trustClient) { FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); - processConnection(store, from, to, Trusted, NotRecursive); + processConnection(store, from, to, trustClient, NotRecursive); } /** * Entry point shared between the new CLI `nix daemon` and old CLI * `nix-daemon`. + * + * @param forceTrustClientOpt See `daemonLoop()` and the parameter with + * the same name over there for details. */ -static void runDaemon(bool stdio) +static void runDaemon(bool stdio, std::optional forceTrustClientOpt) { if (stdio) { auto store = openUncachedStore(); - if (auto remoteStore = store.dynamic_pointer_cast()) + // If --force-untrusted is passed, we cannot forward the connection and + // must process it ourselves (before delegating to the next store) to + // force untrusting the client. + if (auto remoteStore = store.dynamic_pointer_cast(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted)) forwardStdioConnection(*remoteStore); else - processStdioConnection(store); + // `Trusted` is passed in the auto (no override case) because we + // cannot see who is on the other side of a plain pipe. Limiting + // access to those is explicitly not `nix-daemon`'s responsibility. + processStdioConnection(store, forceTrustClientOpt.value_or(Trusted)); } else - daemonLoop(); + daemonLoop(forceTrustClientOpt); } static int main_nix_daemon(int argc, char * * argv) { { auto stdio = false; + std::optional isTrustedOpt = std::nullopt; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") @@ -452,11 +474,20 @@ static int main_nix_daemon(int argc, char * * argv) printVersion("nix-daemon"); else if (*arg == "--stdio") stdio = true; - else return false; + else if (*arg == "--force-trusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = Trusted; + } else if (*arg == "--force-untrusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = NotTrusted; + } else if (*arg == "--default-trust") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = std::nullopt; + } else return false; return true; }); - runDaemon(stdio); + runDaemon(stdio, isTrustedOpt); return 0; } @@ -482,7 +513,7 @@ struct CmdDaemon : StoreCommand void run(ref store) override { - runDaemon(false); + runDaemon(false, std::nullopt); } }; diff --git a/tests/build-remote-trustless-after.sh b/tests/build-remote-trustless-after.sh new file mode 100644 index 000000000..19f59e6ae --- /dev/null +++ b/tests/build-remote-trustless-after.sh @@ -0,0 +1,2 @@ +outPath=$(readlink -f $TEST_ROOT/result) +grep 'FOO BAR BAZ' ${remoteDir}/${outPath} diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh new file mode 100644 index 000000000..5e3d5ae07 --- /dev/null +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -0,0 +1,29 @@ +source common.sh + +enableFeatures "daemon-trust-override" + +restartDaemon + +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +# We first build a dependency of the derivation we eventually want to +# build. +nix-build build-hook.nix -A passthru.input2 \ + -o "$TEST_ROOT/input2" \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/local" \ + --option system-features bar + +# Now when we go to build that downstream derivation, Nix will fail +# because we cannot trustlessly build input-addressed derivations with +# `inputDrv` dependencies. + +file=build-hook.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +expectStderr 1 source build-remote-trustless.sh \ + | grepQuiet "you are not privileged to build input-addressed derivations" diff --git a/tests/build-remote-trustless-should-pass-0.sh b/tests/build-remote-trustless-should-pass-0.sh new file mode 100644 index 000000000..2a7ebd8c6 --- /dev/null +++ b/tests/build-remote-trustless-should-pass-0.sh @@ -0,0 +1,9 @@ +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-store +proto=ssh + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-pass-1.sh b/tests/build-remote-trustless-should-pass-1.sh new file mode 100644 index 000000000..516bdf092 --- /dev/null +++ b/tests/build-remote-trustless-should-pass-1.sh @@ -0,0 +1,9 @@ +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-daemon +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-pass-3.sh b/tests/build-remote-trustless-should-pass-3.sh new file mode 100644 index 000000000..40f81da5a --- /dev/null +++ b/tests/build-remote-trustless-should-pass-3.sh @@ -0,0 +1,14 @@ +source common.sh + +enableFeatures "daemon-trust-override" + +restartDaemon + +# Remote doesn't trusts us, but this is fine because we are only +# building (fixed) CA derivations. +file=build-hook-ca-fixed.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless.sh b/tests/build-remote-trustless.sh new file mode 100644 index 000000000..9df44e0c5 --- /dev/null +++ b/tests/build-remote-trustless.sh @@ -0,0 +1,14 @@ +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +remoteDir=$TEST_ROOT/remote + +# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for +# more details. +nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ + --arg busybox $busybox \ + --store $TEST_ROOT/local \ + --builders "$proto://localhost?remote-program=$prog&remote-store=${remoteDir}%3Fsystem-features=foo%20bar%20baz - - 1 1 foo,bar,baz" diff --git a/tests/local.mk b/tests/local.mk index 6cb466e8e..7c3b42599 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -70,6 +70,10 @@ nix_tests = \ check-reqs.sh \ build-remote-content-addressed-fixed.sh \ build-remote-content-addressed-floating.sh \ + build-remote-trustless-should-pass-0.sh \ + build-remote-trustless-should-pass-1.sh \ + build-remote-trustless-should-pass-3.sh \ + build-remote-trustless-should-fail-0.sh \ nar-access.sh \ pure-eval.sh \ eval.sh \ diff --git a/tests/nix-daemon-untrusting.sh b/tests/nix-daemon-untrusting.sh new file mode 100755 index 000000000..bcdb70989 --- /dev/null +++ b/tests/nix-daemon-untrusting.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec nix-daemon --force-untrusted "$@" From aa74c7b0bcd31a6c0f75f5fa09f417bcbef4ad14 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 11:22:31 -0400 Subject: [PATCH 55/61] Gate experimental features in `DerivationOutput::fromJSON` This is an entry point for outside data, so we need to check enabled experimental features here. --- src/libstore/derivations.cc | 5 +++- src/libstore/derivations.hh | 6 ++++- src/libstore/tests/derivation.cc | 42 +++++++++++++++++++++++++------- tests/ca/derivation-json.sh | 3 +++ tests/impure-derivations.sh | 9 +++++++ 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 0de36504b..15f3908ed 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -989,7 +989,8 @@ nlohmann::json DerivationOutput::toJSON( DerivationOutput DerivationOutput::fromJSON( const Store & store, std::string_view drvName, std::string_view outputName, - const nlohmann::json & _json) + const nlohmann::json & _json, + const ExperimentalFeatureSettings & xpSettings) { std::set keys; auto json = (std::map) _json; @@ -1028,6 +1029,7 @@ DerivationOutput DerivationOutput::fromJSON( } else if (keys == (std::set { "hashAlgo" })) { + xpSettings.require(Xp::CaDerivations); auto [method, hashType] = methodAlgo(); return DerivationOutput::CAFloating { .method = method, @@ -1040,6 +1042,7 @@ DerivationOutput DerivationOutput::fromJSON( } else if (keys == (std::set { "hashAlgo", "impure" })) { + xpSettings.require(Xp::ImpureDerivations); auto [method, hashType] = methodAlgo(); return DerivationOutput::Impure { .method = method, diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index ccdde36ca..d00b23b6d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -136,11 +136,15 @@ struct DerivationOutput : _DerivationOutputRaw const Store & store, std::string_view drvName, std::string_view outputName) const; + /** + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ static DerivationOutput fromJSON( const Store & store, std::string_view drvName, std::string_view outputName, - const nlohmann::json & json); + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; typedef std::map DerivationOutputs; diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 80ee52fd0..6f94904dd 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -1,6 +1,7 @@ #include #include +#include "experimental-features.hh" #include "derivations.hh" #include "tests/libstore.hh" @@ -9,10 +10,32 @@ namespace nix { class DerivationTest : public LibStoreTest { +public: + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; }; -#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \ +class CaDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "ca-derivations"); + } +}; + +class ImpureDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "impure-derivations"); + } +}; + +#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ STR ## _json, \ @@ -22,7 +45,7 @@ class DerivationTest : public LibStoreTest OUTPUT_NAME)); \ } \ \ - TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ DerivationOutput { VAL }, \ @@ -30,10 +53,11 @@ class DerivationTest : public LibStoreTest *store, \ DRV_NAME, \ OUTPUT_NAME, \ - STR ## _json)); \ + STR ## _json, \ + mockXpSettings)); \ } -TEST_JSON(inputAddressed, +TEST_JSON(DerivationTest, inputAddressed, R"({ "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", @@ -42,7 +66,7 @@ TEST_JSON(inputAddressed, }), "drv-name", "output-name") -TEST_JSON(caFixed, +TEST_JSON(DerivationTest, caFixed, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -56,7 +80,7 @@ TEST_JSON(caFixed, }), "drv-name", "output-name") -TEST_JSON(caFloating, +TEST_JSON(CaDerivationTest, caFloating, R"({ "hashAlgo": "r:sha256" })", @@ -66,12 +90,12 @@ TEST_JSON(caFloating, }), "drv-name", "output-name") -TEST_JSON(deferred, +TEST_JSON(DerivationTest, deferred, R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") -TEST_JSON(impure, +TEST_JSON(ImpureDerivationTest, impure, R"({ "hashAlgo": "r:sha256", "impure": true diff --git a/tests/ca/derivation-json.sh b/tests/ca/derivation-json.sh index 3615177e9..c1480fd17 100644 --- a/tests/ca/derivation-json.sh +++ b/tests/ca/derivation-json.sh @@ -16,6 +16,9 @@ drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) # With --dry-run nothing is actually written [[ ! -e "$drvPath3" ]] +# But the JSON is rejected without the experimental feature +expectStderr 1 nix derivation add < $TEST_HOME/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" + # Without --dry-run it is actually written drvPath4=$(nix derivation add < $TEST_HOME/foo.json) [[ "$drvPath4" = "$drvPath3" ]] diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh index c7dadf397..39d053a04 100644 --- a/tests/impure-derivations.sh +++ b/tests/impure-derivations.sh @@ -10,6 +10,15 @@ clearStore # Basic test of impure derivations: building one a second time should not use the previous result. printf 0 > $TEST_ROOT/counter +# `nix derivation add` with impure derivations work +drvPath=$(nix-instantiate ./impure-derivations.nix -A impure) +nix derivation show $drvPath | jq .[] > $TEST_HOME/impure-drv.json +drvPath2=$(nix derivation add < $TEST_HOME/impure-drv.json) +[[ "$drvPath" = "$drvPath2" ]] + +# But only with the experimental feature! +expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'impure-derivations' is disabled" + nix build --dry-run --json --file ./impure-derivations.nix impure.all json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all) path1=$(echo $json | jq -r .[].outputs.out) From 324ed0c36732fdc6c8230271da3f1a211b7ee8d4 Mon Sep 17 00:00:00 2001 From: Noah Snelson Date: Mon, 17 Apr 2023 20:15:08 -0700 Subject: [PATCH 56/61] Documentation: fix typo for `Nix database` link in manual Fixes broken link for `Nix database` anchor in the Glossary page of the Nix manual. --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a9782be5c..eeb19ad50 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -127,7 +127,7 @@ builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. - - Nix database{#gloss-nix-database}\ + - [Nix database]{#gloss-nix-database}\ An SQlite database to track [reference]s between [store object]s. This is an implementation detail of the [local store]. From 40fcb22313e65d1a57d0f6052ec046971ca07b8c Mon Sep 17 00:00:00 2001 From: Michael Utz Date: Tue, 18 Apr 2023 13:18:30 +0300 Subject: [PATCH 57/61] Update installing-binary.md --- doc/manual/src/installation/installing-binary.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index e3fd962bd..525654d35 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -136,7 +136,7 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing +1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash @@ -153,6 +153,7 @@ which you may remove. ```console sudo mv /etc/zshrc.backup-before-nix /etc/zshrc sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc ``` This will stop shells from sourcing the file and bringing everything you From d30d2dc861ddb20f035a0ae549e57fc439217b62 Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Mon, 17 Apr 2023 19:34:09 +0200 Subject: [PATCH 58/61] Make "NAR info file is corrupt" messages more informative Recently, I encountered the "NAR info file 'xxxx' is corrupt" error with my binary cache. The message is not helpful in determining, which kind of corruption happened. The file, fetched with curl, looked reasonably. This commit adds more information to the error message, which should allow debugging and hopefully fixing the problem. --- src/libstore/nar-info.cc | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 274cd861c..d17253741 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,15 +7,18 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { - auto corrupt = [&]() { - return Error("NAR info file '%1%' is corrupt", whence); + unsigned line = 1; + + auto corrupt = [&](const char * reason) { + return Error("NAR info file '%1%' is corrupt: %2%", whence, + std::string(reason) + (line > 0 ? " at line " + std::to_string(line) : "")); }; auto parseHashField = [&](const std::string & s) { try { return Hash::parseAnyPrefixed(s); } catch (BadHash &) { - throw corrupt(); + throw corrupt("bad hash"); } }; @@ -26,12 +29,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt(); + if (colon == std::string::npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt(); + if (eol == std::string::npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); @@ -47,7 +50,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & fileHash = parseHashField(value); else if (name == "FileSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid FileSize"); fileSize = *n; } else if (name == "NarHash") { @@ -56,12 +59,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & } else if (name == "NarSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid NarSize"); narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); - if (!references.empty()) throw corrupt(); + if (!references.empty()) throw corrupt("extra References"); for (auto & r : refs) references.insert(StorePath(r)); } @@ -72,17 +75,26 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (ca) throw corrupt(); + if (ca) throw corrupt("extra CA"); // FIXME: allow blank ca or require skipping field? ca = ContentAddress::parseOpt(value); } pos = eol + 1; + line += 1; } if (compression == "") compression = "bzip2"; - if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt(); + if (!havePath || !haveNarHash || url.empty() || narSize == 0) { + line = 0; // don't include line information in the error + throw corrupt( + !havePath ? "StorePath missing" : + !haveNarHash ? "NarHash missing" : + url.empty() ? "URL missing" : + narSize == 0 ? "NarSize missing or zero" + : "?"); + } } std::string NarInfo::to_string(const Store & store) const From 5cd9890e8a96e2f2ab205738c1e2e4a6b615f443 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Apr 2023 16:06:58 +0200 Subject: [PATCH 59/61] src/nix/flake.md: Itemize safe nixConfigs --- src/nix/flake.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 965f6eb48..456fd0ea1 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -381,10 +381,12 @@ The following attributes are supported in `flake.nix`: * `nixConfig`: a set of `nix.conf` options to be set when evaluating any part of a flake. In the interests of security, only a small set of - whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`, - `bash-prompt-suffix`, `flake-registry`, and `commit-lockfile-summary`) - are allowed to be set without confirmation so long as `accept-flake-config` - is not set in the global configuration. + set of options is allowed to be set without confirmation so long as [`accept-flake-config`](@docroot@/command-ref/conf-file.md#conf-accept-flake-config) is not enabled in the global configuration: + - [`bash-prompt`](@docroot@/command-ref/conf-file.md#conf-bash-prompt) + - [`bash-prompt-prefix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-prefix) + - [`bash-prompt-suffix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-suffix) + - [`flake-registry`](@docroot@/command-ref/conf-file.md#conf-flake-registry) + - [`commit-lockfile-summary`](@docroot@/command-ref/conf-file.md#conf-commit-lockfile-summary) ## Flake inputs From 3eb343754e42228b113bdcb7aec24ae18384a5fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Apr 2023 19:36:05 -0400 Subject: [PATCH 60/61] Move `test/recursive.sh` nix expr to file I found it hard to read as a big string literal. --- tests/recursive.nix | 56 +++++++++++++++++++++++++++++++++++++++++++ tests/recursive.sh | 58 +-------------------------------------------- 2 files changed, 57 insertions(+), 57 deletions(-) create mode 100644 tests/recursive.nix diff --git a/tests/recursive.nix b/tests/recursive.nix new file mode 100644 index 000000000..fa8cc04db --- /dev/null +++ b/tests/recursive.nix @@ -0,0 +1,56 @@ +with import ./config.nix; + +mkDerivation rec { + name = "recursive"; + dummy = builtins.toFile "dummy" "bla bla"; + SHELL = shell; + + # Note: this is a string without context. + unreachable = builtins.getEnv "unreachable"; + + NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; + + requiredSystemFeatures = [ "recursive-nix" ]; + + buildCommand = '' + mkdir $out + opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" + + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + # Check that we can query/build paths in our input closure. + nix $opts path-info $dummy + nix $opts build $dummy + + # Make sure we cannot query/build paths not in out input closure. + [[ -e $unreachable ]] + (! nix $opts path-info $unreachable) + (! nix $opts build $unreachable) + + # Add something to the store. + echo foobar > foobar + foobar=$(nix $opts store add-path ./foobar) + + nix $opts path-info $foobar + nix $opts build $foobar + + # Add it to our closure. + ln -s $foobar $out/foobar + + [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] + + # Build a derivation. + nix $opts build -L --impure --expr ' + with import ${./config.nix}; + mkDerivation { + name = "inner1"; + buildCommand = "echo $fnord blaat > $out"; + fnord = builtins.toFile "fnord" "fnord"; + } + ' + + [[ $(nix $opts path-info --json ./result) =~ fnord ]] + + ln -s $(nix $opts path-info ./result) $out/inner1 + ''; +} diff --git a/tests/recursive.sh b/tests/recursive.sh index 6335d44a5..638f06f85 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -12,63 +12,7 @@ rm -f $TEST_ROOT/result export unreachable=$(nix store add-path ./recursive.sh) -NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' - with import ./config.nix; - mkDerivation rec { - name = "recursive"; - dummy = builtins.toFile "dummy" "bla bla"; - SHELL = shell; - - # Note: this is a string without context. - unreachable = builtins.getEnv "unreachable"; - - NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; - - requiredSystemFeatures = [ "recursive-nix" ]; - - buildCommand = '\'\'' - mkdir $out - opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" - - PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH - - # Check that we can query/build paths in our input closure. - nix $opts path-info $dummy - nix $opts build $dummy - - # Make sure we cannot query/build paths not in out input closure. - [[ -e $unreachable ]] - (! nix $opts path-info $unreachable) - (! nix $opts build $unreachable) - - # Add something to the store. - echo foobar > foobar - foobar=$(nix $opts store add-path ./foobar) - - nix $opts path-info $foobar - nix $opts build $foobar - - # Add it to our closure. - ln -s $foobar $out/foobar - - [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] - - # Build a derivation. - nix $opts build -L --impure --expr '\'' - with import ${./config.nix}; - mkDerivation { - name = "inner1"; - buildCommand = "echo $fnord blaat > $out"; - fnord = builtins.toFile "fnord" "fnord"; - } - '\'' - - [[ $(nix $opts path-info --json ./result) =~ fnord ]] - - ln -s $(nix $opts path-info ./result) $out/inner1 - '\'\''; - } -' +NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --file ./recursive.nix [[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] From 85f0cdc370021299142be9454483403e9bac2602 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 28 Jan 2023 20:31:10 -0500 Subject: [PATCH 61/61] Use `std::set` not `PathSet` for string contexts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std`. ----- In support of this change, `NixStringContext` is now defined as `std::std` not `std:vector`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libcmd/installable-flake.cc | 6 +- src/libcmd/repl.cc | 4 +- src/libexpr/eval-cache.cc | 12 ++-- src/libexpr/eval.cc | 58 ++++++++-------- src/libexpr/eval.hh | 24 ++++--- src/libexpr/flake/flake.cc | 2 +- src/libexpr/get-drvs.cc | 6 +- src/libexpr/primops.cc | 95 +++++++++++++++----------- src/libexpr/primops/context.cc | 54 ++++++++------- src/libexpr/primops/fetchClosure.cc | 7 +- src/libexpr/primops/fetchMercurial.cc | 5 +- src/libexpr/primops/fetchTree.cc | 5 +- src/libexpr/tests/json.cc | 2 +- src/libexpr/tests/value/context.cc | 61 ++++++++--------- src/libexpr/value-to-json.cc | 6 +- src/libexpr/value-to-json.hh | 4 +- src/libexpr/value-to-xml.cc | 10 +-- src/libexpr/value-to-xml.hh | 2 +- src/libexpr/value.hh | 12 ++-- src/libexpr/value/context.cc | 17 +++-- src/libexpr/value/context.hh | 15 ++-- src/nix-env/nix-env.cc | 2 +- src/nix-env/user-env.cc | 6 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix/bundle.cc | 2 +- src/nix/eval.cc | 2 +- src/nix/flake.cc | 2 +- 27 files changed, 219 insertions(+), 204 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index a3352af76..7e2a975f7 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -96,7 +96,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto v = attr->forceValue(); if (v.type() == nPath) { - PathSet context; + NixStringContext context; auto storePath = state->copyPathToStore(context, Path(v.path)); return {{ .path = DerivedPath::Opaque { @@ -107,10 +107,10 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() } else if (v.type() == nString) { - PathSet context; + NixStringContext context; auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath)); auto storePath = state->store->maybeParseStorePath(s); - if (storePath && context.count(std::string(s))) { + if (storePath && context.count(NixStringContextElem::Opaque { .path = *storePath })) { return {{ .path = DerivedPath::Opaque { .path = std::move(*storePath), diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 80c08bf1c..8d404b04e 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -596,7 +596,7 @@ bool NixRepl::processLine(std::string line) const auto [path, line] = [&] () -> std::pair { if (v.type() == nPath || v.type() == nString) { - PathSet context; + NixStringContext context; auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { @@ -940,7 +940,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m if (isDrv) { str << "«derivation "; Bindings::iterator i = v.attrs->find(state->sDrvPath); - PathSet context; + NixStringContext context; if (i != v.attrs->end()) str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); else diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 1219b2471..ba364f656 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -47,7 +47,7 @@ struct AttrDb { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v4"; + Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; @@ -300,7 +300,7 @@ struct AttrDb NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) - context.push_back(NixStringContextElem::parse(cfg, s)); + context.insert(NixStringContextElem::parse(s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: @@ -619,9 +619,11 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); - if (v.type() == nString) - return {v.string.s, v.getContext(*root->state.store)}; - else if (v.type() == nPath) + if (v.type() == nString) { + NixStringContext context; + copyContext(v, context); + return {v.string.s, std::move(context)}; + } else if (v.type() == nPath) return {v.path, {}}; else root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6668add8c..56ff5908b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -609,8 +609,7 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & { allowPath(storePath); - auto path = store->printStorePath(storePath); - v.mkString(path, PathSet({path})); + mkStorePathString(storePath, v); } Path EvalState::checkSourcePath(const Path & path_) @@ -692,7 +691,7 @@ void EvalState::checkURI(const std::string & uri) } -Path EvalState::toRealPath(const Path & path, const PathSet & context) +Path EvalState::toRealPath(const Path & path, const NixStringContext & context) { // FIXME: check whether 'path' is in 'context'. return @@ -944,25 +943,25 @@ void Value::mkString(std::string_view s) } -static void copyContextToValue(Value & v, const PathSet & context) +static void copyContextToValue(Value & v, const NixStringContext & context) { if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); for (auto & i : context) - v.string.context[n++] = dupString(i.c_str()); + v.string.context[n++] = dupString(i.to_string().c_str()); v.string.context[n] = 0; } } -void Value::mkString(std::string_view s, const PathSet & context) +void Value::mkString(std::string_view s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); } -void Value::mkStringMove(const char * s, const PathSet & context) +void Value::mkStringMove(const char * s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); @@ -1038,6 +1037,16 @@ void EvalState::mkPos(Value & v, PosIdx p) } +void EvalState::mkStorePathString(const StorePath & p, Value & v) +{ + v.mkString( + store->printStorePath(p), + NixStringContext { + NixStringContextElem::Opaque { .path = p }, + }); +} + + /* Create a thunk for the delayed computation of the given expression in the given environment. But if the expression is a variable, then look it up right away. This significantly reduces the number @@ -1900,7 +1909,7 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) { - PathSet context; + NixStringContext context; std::vector s; size_t sSize = 0; NixInt n = 0; @@ -2109,26 +2118,15 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string } -void copyContext(const Value & v, PathSet & context) +void copyContext(const Value & v, NixStringContext & context) { if (v.string.context) for (const char * * p = v.string.context; *p; ++p) - context.insert(*p); + context.insert(NixStringContextElem::parse(*p)); } -NixStringContext Value::getContext(const Store & store) -{ - NixStringContext res; - assert(internalType == tString); - if (string.context) - for (const char * * p = string.context; *p; ++p) - res.push_back(NixStringContextElem::parse(store, *p)); - return res; -} - - -std::string_view EvalState::forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx) +std::string_view EvalState::forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx) { auto s = forceString(v, pos, errorCtx); copyContext(v, context); @@ -2158,7 +2156,7 @@ bool EvalState::isDerivation(Value & v) std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore, bool copyToStore) + NixStringContext & context, bool coerceMore, bool copyToStore) { auto i = v.attrs->find(sToString); if (i != v.attrs->end()) { @@ -2172,7 +2170,7 @@ std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & return {}; } -BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet &context, +BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, NixStringContext &context, std::string_view errorCtx, bool coerceMore, bool copyToStore, bool canonicalizePath) { forceValue(v, pos); @@ -2249,7 +2247,7 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet & } -StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(NixStringContext & context, const Path & path) { if (nix::isDerivation(path)) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); @@ -2268,12 +2266,14 @@ StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) return dstPath; }(); - context.insert(store->printStorePath(dstPath)); + context.insert(NixStringContextElem::Opaque { + .path = dstPath + }); return dstPath; } -Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) +Path EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') @@ -2282,7 +2282,7 @@ Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std } -StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) +StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) @@ -2489,7 +2489,7 @@ void EvalState::printStats() } -std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const +std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { throw TypeError({ .msg = hintfmt("cannot coerce %1% to a string", showType()) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index b3b985683..74a7162ff 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -56,7 +56,7 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env); -void copyContext(const Value & v, PathSet & context); +void copyContext(const Value & v, NixStringContext & context); /** @@ -327,7 +327,7 @@ public: * intended to distinguish between import-from-derivation and * sources stored in the actual /nix/store. */ - Path toRealPath(const Path & path, const PathSet & context); + Path toRealPath(const Path & path, const NixStringContext & context); /** * Parse a Nix expression from the specified file. @@ -423,7 +423,7 @@ public: */ void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); - std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); + std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); [[gnu::noinline]] @@ -439,7 +439,7 @@ public: bool isDerivation(Value & v); std::optional tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore = false, bool copyToStore = true); + NixStringContext & context, bool coerceMore = false, bool copyToStore = true); /** * String coercion. @@ -449,12 +449,12 @@ public: * booleans and lists to a string. If `copyToStore` is set, * referenced paths are copied to the Nix store as a side effect. */ - BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, + BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - StorePath copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(NixStringContext & context, const Path & path); /** * Path coercion. @@ -463,12 +463,12 @@ public: * path. The result is guaranteed to be a canonicalised, absolute * path. Nothing is copied to the store. */ - Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); + Path coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** * Like coerceToPath, but the result must be a store path. */ - StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); + StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); public: @@ -573,6 +573,12 @@ public: void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, PosIdx pos); + /* Create a string representing a store path. + + The string is the printed store path with a context containing a single + `Opaque` element of that store path. */ + void mkStorePathString(const StorePath & storePath, Value & v); + void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /** @@ -584,7 +590,7 @@ public: * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path */ - [[nodiscard]] StringMap realiseContext(const PathSet & context); + [[nodiscard]] StringMap realiseContext(const NixStringContext & context); private: diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ac396236f..3fb1f3536 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -265,7 +265,7 @@ static Flake getFlake( state.symbols[setting.name], std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - PathSet emptyContext = {}; + NixStringContext emptyContext = {}; flake.config.settings.emplace( state.symbols[setting.name], state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned()); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1602fbffb..506a63677 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -71,7 +71,7 @@ std::optional DrvInfo::queryDrvPath() const { if (!drvPath && attrs) { Bindings::iterator i = attrs->find(state->sDrvPath); - PathSet context; + NixStringContext context; if (i == attrs->end()) drvPath = {std::nullopt}; else @@ -93,7 +93,7 @@ StorePath DrvInfo::queryOutPath() const { if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); - PathSet context; + NixStringContext context; if (i != attrs->end()) outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation"); } @@ -124,7 +124,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall /* And evaluate its ‘outPath’ attribute. */ Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; + NixStringContext context; outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation")); } else outputs.emplace(output, std::nullopt); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 510f674eb..d11b56f8b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -38,17 +38,16 @@ namespace nix { InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} -StringMap EvalState::realiseContext(const PathSet & context) +StringMap EvalState::realiseContext(const NixStringContext & context) { std::vector drvs; StringMap res; - for (auto & c_ : context) { + for (auto & c : context) { auto ensureValid = [&](const StorePath & p) { if (!store->isValidPath(p)) debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); }; - auto c = NixStringContextElem::parse(*store, c_); std::visit(overloaded { [&](const NixStringContextElem::Built & b) { drvs.push_back(DerivedPath::Built { @@ -112,7 +111,7 @@ struct RealisePathFlags { static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) { - PathSet context; + NixStringContext context; auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); @@ -158,7 +157,12 @@ static void mkOutputString( /* FIXME: we need to depend on the basic derivation, not derivation */ : downstreamPlaceholder(*state.store, drvPath, o.first), - {"!" + o.first + "!" + state.store->printStorePath(drvPath)}); + NixStringContext { + NixStringContextElem::Built { + .drvPath = drvPath, + .output = o.first, + } + }); } /* Load and evaluate an expression from path specified by the @@ -181,7 +185,9 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v auto storePath = *optStorePath; Derivation drv = state.store->readDerivation(storePath); auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(path, {"=" + path}); + attrs.alloc(state.sDrvPath).mkString(path, { + NixStringContextElem::DrvDeep { .drvPath = storePath }, + }); attrs.alloc(state.sName).mkString(drv.env["name"]); auto & outputsVal = attrs.alloc(state.sOutputs); state.mkList(outputsVal, drv.outputs.size()); @@ -358,7 +364,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto count = args[0]->listSize(); if (count == 0) state.error("at least one argument to 'exec' required").atPos(pos).debugThrow(); - PathSet context; + NixStringContext context; auto program = state.coerceToString(pos, *elems[0], context, "while evaluating the first element of the argument passed to builtins.exec", false, false).toOwned(); @@ -768,7 +774,7 @@ static RegisterPrimOp primop_abort({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.abort").toOwned(); state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s)); @@ -787,7 +793,7 @@ static RegisterPrimOp primop_throw({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtin.throw").toOwned(); state.debugThrowLastTrace(ThrownError(s)); @@ -800,7 +806,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { - PathSet context; + NixStringContext context; auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); @@ -1086,7 +1092,7 @@ drvName, Bindings * attrs, Value & v) Derivation drv; drv.name = drvName; - PathSet context; + NixStringContext context; bool contentAddressed = false; bool isImpure = false; @@ -1232,8 +1238,7 @@ drvName, Bindings * attrs, Value & v) /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - for (auto & c_ : context) { - auto c = NixStringContextElem::parse(*state.store, c_); + for (auto & c : context) { std::visit(overloaded { /* Since this allows the builder to gain access to every path in the dependency graph of the derivation (including @@ -1392,7 +1397,9 @@ drvName, Bindings * attrs, Value & v) } auto result = state.buildBindings(1 + drv.outputs.size()); - result.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); + result.alloc(state.sDrvPath).mkString(drvPathS, { + NixStringContextElem::DrvDeep { .drvPath = drvPath }, + }); for (auto & i : drv.outputs) mkOutputString(state, result, drvPath, drv, i); @@ -1437,7 +1444,7 @@ static RegisterPrimOp primop_placeholder({ /* Convert the argument to a path. !!! obsolete? */ static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; Path path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); v.mkString(canonPath(path), context); } @@ -1468,7 +1475,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, .errPos = state.positions[pos] })); - PathSet context; + NixStringContext context; Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")); /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so @@ -1482,7 +1489,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, auto path2 = state.store->toStorePath(path).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); - context.insert(state.store->printStorePath(path2)); + context.insert(NixStringContextElem::Opaque { .path = path2 }); v.mkString(path, context); } @@ -1538,7 +1545,7 @@ static RegisterPrimOp primop_pathExists({ following the last slash. */ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.baseNameOf", false, false)), context); @@ -1560,7 +1567,7 @@ static RegisterPrimOp primop_baseNameOf({ of the argument. */ static void prim_dirOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto path = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.dirOf", false, false); @@ -1597,7 +1604,12 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V refsSink << s; refs = refsSink.getResultPaths(); } - auto context = state.store->printStorePathSet(refs); + NixStringContext context; + for (auto && p : std::move(refs)) { + context.insert(NixStringContextElem::Opaque { + .path = std::move((StorePath &&)p), + }); + } v.mkString(s, context); } @@ -1628,7 +1640,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V i = getAttr(state, state.sPath, v2->attrs, "in an element of the __nixPath"); - PathSet context; + NixStringContext context; auto path = state.coerceToString(pos, *i->value, context, "while evaluating the `path` attribute of an element of the list passed to builtins.findFile", false, false).toOwned(); @@ -1787,7 +1799,7 @@ static RegisterPrimOp primop_readDir({ static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsXML(state, true, false, *args[0], out, context, pos); v.mkString(out.str(), context); } @@ -1895,7 +1907,7 @@ static RegisterPrimOp primop_toXML({ static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsJSON(state, true, *args[0], pos, out, context); v.mkString(out.str(), context); } @@ -1945,22 +1957,23 @@ static RegisterPrimOp primop_fromJSON({ as an input by derivations. */ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile")); std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; - for (auto path : context) { - if (path.at(0) != '/') + for (auto c : context) { + if (auto p = std::get_if(&c)) + refs.insert(p->path); + else state.debugThrowLastTrace(EvalError({ .msg = hintfmt( "in 'toFile': the file named '%1%' must not contain a reference " "to a derivation but contains (%2%)", - name, path), + name, c.to_string()), .errPos = state.positions[pos] })); - refs.insert(state.store->parseStorePath(path)); } auto storePath = settings.readOnlyMode @@ -2061,7 +2074,7 @@ static void addPath( FileIngestionMethod method, const std::optional expectedHash, Value & v, - const PathSet & context) + const NixStringContext & context) { try { // FIXME: handle CA derivation outputs (where path needs to @@ -2135,7 +2148,7 @@ static void addPath( static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; Path path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); @@ -2204,7 +2217,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional expectedHash; - PathSet context; + NixStringContext context; for (auto & attr : *args[0]->attrs) { auto n = state.symbols[attr.name]; @@ -3538,7 +3551,7 @@ static RegisterPrimOp primop_lessThan({ `"/nix/store/whatever..."'. */ static void prim_toString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.toString", true, false); @@ -3577,7 +3590,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, { int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring"); if (start < 0) @@ -3611,7 +3624,7 @@ static RegisterPrimOp primop_substring({ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); v.mkInt(s->size()); } @@ -3637,7 +3650,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, .errPos = state.positions[pos] })); - PathSet context; // discarded + NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); v.mkString(hashString(*ht, s).to_string(Base16, false)); @@ -3683,7 +3696,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto regex = state.regexCache->get(re); - PathSet context; + NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); std::cmatch match; @@ -3763,7 +3776,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto regex = state.regexCache->get(re); - PathSet context; + NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); auto begin = std::cregex_iterator(str.begin(), str.end(), regex); @@ -3860,7 +3873,7 @@ static RegisterPrimOp primop_split({ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto sep = state.forceString(*args[0], context, pos, "while evaluating the first argument (the separator string) passed to builtins.concatStringsSep"); state.forceList(*args[1], pos, "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep"); @@ -3900,15 +3913,15 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a for (auto elem : args[0]->listItems()) from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::vector> to; + std::vector> to; to.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { - PathSet ctx; + NixStringContext ctx; auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); to.emplace_back(s, std::move(ctx)); } - PathSet context; + NixStringContext context; auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); std::string res; diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index db43e5771..07bf400cf 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -7,7 +7,7 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); v.mkString(*s); } @@ -17,7 +17,7 @@ static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringCo static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); v.mkBool(!context.empty()); } @@ -33,17 +33,18 @@ static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); - PathSet context2; - for (auto && p : context) { - auto c = NixStringContextElem::parse(*state.store, p); + NixStringContext context2; + for (auto && c : context) { if (auto * ptr = std::get_if(&c)) { - context2.emplace(state.store->printStorePath(ptr->drvPath)); + context2.emplace(NixStringContextElem::Opaque { + .path = ptr->drvPath + }); } else { /* Can reuse original item */ - context2.emplace(std::move(p)); + context2.emplace(std::move(c)); } } @@ -79,22 +80,21 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, bool allOutputs = false; Strings outputs; }; - PathSet context; + NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); auto contextInfos = std::map(); - for (const auto & p : context) { - NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p); + for (auto && i : context) { std::visit(overloaded { - [&](NixStringContextElem::DrvDeep & d) { - contextInfos[d.drvPath].allOutputs = true; + [&](NixStringContextElem::DrvDeep && d) { + contextInfos[std::move(d.drvPath)].allOutputs = true; }, - [&](NixStringContextElem::Built & b) { - contextInfos[b.drvPath].outputs.emplace_back(std::move(b.output)); + [&](NixStringContextElem::Built && b) { + contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); }, - [&](NixStringContextElem::Opaque & o) { - contextInfos[o.path].path = true; + [&](NixStringContextElem::Opaque && o) { + contextInfos[std::move(o.path)].path = true; }, - }, ctx.raw()); + }, ((NixStringContextElem &&) i).raw()); } auto attrs = state.buildBindings(contextInfos.size()); @@ -129,7 +129,7 @@ static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); */ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto orig = state.forceString(*args[0], context, noPos, "while evaluating the first argument passed to builtins.appendContext"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.appendContext"); @@ -143,13 +143,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .msg = hintfmt("context key '%s' is not a store path", name), .errPos = state.positions[i.pos] }); + auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(name)); + state.store->ensurePath(namePath); state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context")) - context.emplace(name); + context.emplace(NixStringContextElem::Opaque { + .path = namePath, + }); } iter = i.value->attrs->find(sAllOutputs); @@ -161,7 +164,9 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .errPos = state.positions[i.pos] }); } - context.insert(concatStrings("=", name)); + context.emplace(NixStringContextElem::DrvDeep { + .drvPath = namePath, + }); } } @@ -176,7 +181,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar } for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); - context.insert(concatStrings("!", outputName, "!", name)); + context.emplace(NixStringContextElem::Built { + .drvPath = namePath, + .output = std::string { outputName }, + }); } } } diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 0dfa97fa3..4cf1f1e0b 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -18,7 +18,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg const auto & attrName = state.symbols[attr.name]; if (attrName == "fromPath") { - PathSet context; + NixStringContext context; fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); } @@ -27,7 +27,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg state.forceValue(*attr.value, attr.pos); toCA = true; if (attr.value->type() != nString || attr.value->string.s != std::string("")) { - PathSet context; + NixStringContext context; toPath = state.coerceToStorePath(attr.pos, *attr.value, context, "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); } @@ -114,8 +114,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg }); } - auto toPathS = state.store->printStorePath(*toPath); - v.mkString(toPathS, {toPathS}); + state.mkStorePathString(*toPath, v); } static RegisterPrimOp primop_fetchClosure({ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index c41bd60b6..2c0d98e74 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -13,7 +13,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a std::optional rev; std::optional ref; std::string_view name = "source"; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); @@ -73,8 +73,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto [tree, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs2.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 2e150c9d0..cd7039025 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -24,9 +24,8 @@ void emitTreeAttrs( auto attrs = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -107,7 +106,7 @@ static void fetchTree( const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index 411bc0ac3..7586bdd9b 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -8,7 +8,7 @@ namespace nix { protected: std::string getJSONValue(Value& value) { std::stringstream ss; - PathSet ps; + NixStringContext ps; printValueAsJSON(state, true, value, noPos, ss, ps); return ss.str(); } diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 083359b7a..27d6920b0 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,69 +8,62 @@ namespace nix { -// Testing of trivial expressions -struct NixStringContextElemTest : public LibExprTest { - const Store & store() const { - return *LibExprTest::store; - } -}; - -TEST_F(NixStringContextElemTest, empty_invalid) { +TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), ""), + NixStringContextElem::parse(""), BadNixStringContextElem); } -TEST_F(NixStringContextElemTest, single_bang_invalid) { +TEST(NixStringContextElemTest, single_bang_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "!"), + NixStringContextElem::parse("!"), BadNixStringContextElem); } -TEST_F(NixStringContextElemTest, double_bang_invalid) { +TEST(NixStringContextElemTest, double_bang_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "!!/"), + NixStringContextElem::parse("!!/"), BadStorePath); } -TEST_F(NixStringContextElemTest, eq_slash_invalid) { +TEST(NixStringContextElemTest, eq_slash_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "=/"), + NixStringContextElem::parse("=/"), BadStorePath); } -TEST_F(NixStringContextElemTest, slash_invalid) { +TEST(NixStringContextElemTest, slash_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "/"), + NixStringContextElem::parse("/"), BadStorePath); } -TEST_F(NixStringContextElemTest, opaque) { - std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; - auto elem = NixStringContextElem::parse(store(), opaque); +TEST(NixStringContextElemTest, opaque) { + std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = NixStringContextElem::parse(opaque); auto * p = std::get_if(&elem); ASSERT_TRUE(p); - ASSERT_EQ(p->path, store().parseStorePath(opaque)); - ASSERT_EQ(elem.to_string(store()), opaque); + ASSERT_EQ(p->path, StorePath { opaque }); + ASSERT_EQ(elem.to_string(), opaque); } -TEST_F(NixStringContextElemTest, drvDeep) { - std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; - auto elem = NixStringContextElem::parse(store(), drvDeep); +TEST(NixStringContextElemTest, drvDeep) { + std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(drvDeep); auto * p = std::get_if(&elem); ASSERT_TRUE(p); - ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1))); - ASSERT_EQ(elem.to_string(store()), drvDeep); + ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) }); + ASSERT_EQ(elem.to_string(), drvDeep); } -TEST_F(NixStringContextElemTest, built) { - std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; - auto elem = NixStringContextElem::parse(store(), built); +TEST(NixStringContextElemTest, built) { + std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built); auto * p = std::get_if(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5))); - ASSERT_EQ(elem.to_string(store()), built); + ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(elem.to_string(), built); } } @@ -116,12 +109,12 @@ Gen Arbitrary::arbitrary() namespace nix { -RC_GTEST_FIXTURE_PROP( +RC_GTEST_PROP( NixStringContextElemTest, prop_round_rip, (const NixStringContextElem & o)) { - RC_ASSERT(o == NixStringContextElem::parse(store(), o.to_string(store()))); + RC_ASSERT(o == NixStringContextElem::parse(o.to_string())); } } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index c35c876e3..67550b6b1 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -11,7 +11,7 @@ namespace nix { using json = nlohmann::json; json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { checkInterrupt(); @@ -94,13 +94,13 @@ json printValueAsJSON(EvalState & state, bool strict, } void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore) { str << printValueAsJSON(state, strict, v, pos, context, copyToStore); } json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore) const + NixStringContext & context, bool copyToStore) const { state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 713356c7f..47ac90313 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { nlohmann::json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 341c8922f..fe652fd49 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -18,7 +18,7 @@ static XMLAttrs singletonAttrs(const std::string & name, const std::string & val static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos); @@ -32,7 +32,7 @@ static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Bindings & attrs, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen) { StringSet names; @@ -54,7 +54,7 @@ static void showAttrs(EvalState & state, bool strict, bool location, static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) { checkInterrupt(); @@ -166,7 +166,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + bool location, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const { doc.writeEmptyElement("unevaluated"); @@ -174,7 +174,7 @@ void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos) + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index ace7ead0f..6d702c0f2 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,6 +10,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos); + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 7739f99df..2bb791841 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -100,7 +100,7 @@ class ExternalValueBase * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ - virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; + virtual std::string coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; /** * Compare to another value of the same type. Defaults to uncomparable, @@ -112,13 +112,13 @@ class ExternalValueBase * Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore = true) const; + NixStringContext & context, bool copyToStore = true) const; /** * Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const; virtual ~ExternalValueBase() @@ -268,9 +268,9 @@ public: void mkString(std::string_view s); - void mkString(std::string_view s, const PathSet & context); + void mkString(std::string_view s, const NixStringContext & context); - void mkStringMove(const char * s, const PathSet & context); + void mkStringMove(const char * s, const NixStringContext & context); inline void mkPath(const char * s) { @@ -394,8 +394,6 @@ public: */ bool isTrivial() const; - NixStringContext getContext(const Store &); - auto listItems() { struct ListIterable diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 61d9c53df..f76fc76e4 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,11 +1,10 @@ #include "value/context.hh" -#include "store-api.hh" #include namespace nix { -NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0) +NixStringContextElem NixStringContextElem::parse(std::string_view s0) { std::string_view s = s0; @@ -25,41 +24,41 @@ NixStringContextElem NixStringContextElem::parse(const Store & store, std::strin "String content element beginning with '!' should have a second '!'"); } return NixStringContextElem::Built { - .drvPath = store.parseStorePath(s.substr(index + 1)), + .drvPath = StorePath { s.substr(index + 1) }, .output = std::string(s.substr(0, index)), }; } case '=': { return NixStringContextElem::DrvDeep { - .drvPath = store.parseStorePath(s.substr(1)), + .drvPath = StorePath { s.substr(1) }, }; } default: { return NixStringContextElem::Opaque { - .path = store.parseStorePath(s), + .path = StorePath { s }, }; } } } -std::string NixStringContextElem::to_string(const Store & store) const { +std::string NixStringContextElem::to_string() const { return std::visit(overloaded { [&](const NixStringContextElem::Built & b) { std::string res; res += '!'; res += b.output; res += '!'; - res += store.printStorePath(b.drvPath); + res += b.drvPath.to_string(); return res; }, [&](const NixStringContextElem::DrvDeep & d) { std::string res; res += '='; - res += store.printStorePath(d.drvPath); + res += d.drvPath.to_string(); return res; }, [&](const NixStringContextElem::Opaque & o) { - return store.printStorePath(o.path); + return std::string { o.path.to_string() }; }, }, raw()); } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 8719602d8..287ae08a9 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -26,8 +26,6 @@ public: } }; -class Store; - /** * Plain opaque path to some store object. * @@ -80,12 +78,15 @@ struct NixStringContextElem : _NixStringContextElem_Raw { using DrvDeep = NixStringContextElem_DrvDeep; using Built = NixStringContextElem_Built; - inline const Raw & raw() const { + inline const Raw & raw() const & { return static_cast(*this); } - inline Raw & raw() { + inline Raw & raw() & { return static_cast(*this); } + inline Raw && raw() && { + return static_cast(*this); + } /** * Decode a context string, one of: @@ -93,10 +94,10 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘=’ * - ‘!!’ */ - static NixStringContextElem parse(const Store & store, std::string_view s); - std::string to_string(const Store & store) const; + static NixStringContextElem parse(std::string_view s); + std::string to_string() const; }; -typedef std::vector NixStringContext; +typedef std::set NixStringContext; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index f076ffdb0..06f9ff6c4 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -960,7 +960,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); metaObj[j] = nullptr; } else { - PathSet context; + NixStringContext context; metaObj[j] = printValueAsJSON(*globals.state, true, *v, noPos, context); } } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 745e9e174..3f9030827 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -119,9 +119,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Construct a Nix expression that calls the user environment builder with the manifest as argument. */ auto attrs = state.buildBindings(3); - attrs.alloc("manifest").mkString( - state.store->printStorePath(manifestFile), - {state.store->printStorePath(manifestFile)}); + state.mkStorePathString(manifestFile, attrs.alloc("manifest")); attrs.insert(state.symbols.create("derivations"), &manifest); Value args; args.mkAttrs(attrs); @@ -132,7 +130,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Evaluate it. */ debug("evaluating user environment builder"); state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); }); - PathSet context; + NixStringContext context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 6b5ba595d..4f1d12129 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -43,7 +43,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); state.forceValue(v, [&]() { return v.determinePos(noPos); }); - PathSet context; + NixStringContext context; if (evalOnly) { Value vRes; if (autoArgs.empty()) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 57c355f0c..bcc00d490 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -98,7 +98,7 @@ struct CmdBundle : InstallableValueCommand if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - PathSet context2; + NixStringContext context2; auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2, ""); auto attr2 = vRes->attrs->get(evalState->sOutPath); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 43db5150c..ec0112e13 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -62,7 +62,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption auto state = getEvalState(); auto [v, pos] = installable->toValue(*state); - PathSet context; + NixStringContext context; if (apply) { auto vApply = state->allocValue(); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index cd4ee5921..40aabfdb3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -438,7 +438,7 @@ struct CmdFlakeCheck : FlakeCommand if (auto attr = v.attrs->get(state->symbols.create("path"))) { if (attr->name == state->symbols.create("path")) { - PathSet context; + NixStringContext context; auto path = state->coerceToPath(attr->pos, *attr->value, context, ""); if (!store->isInStore(path)) throw Error("template '%s' has a bad 'path' attribute");