From b2748c6e99239ff6803ba0da76c362790c8be192 Mon Sep 17 00:00:00 2001 From: Lucas Franceschino Date: Mon, 25 May 2020 19:07:38 +0200 Subject: [PATCH 001/123] Make `functionArgs` primitive accept primops --- src/libexpr/primops.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0a4236da4..5875457ac 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1346,6 +1346,10 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); + if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) { + state.mkAttrs(v, 0); + return; + } if (args[0]->type != tLambda) throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); From 2f2ae993dc6d35e9c0e66e893e5d615116d42917 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 19:02:05 +0000 Subject: [PATCH 002/123] WIP systematize more of the worker protocol This refactor should *not* change the wire protocol. --- src/libstore/build.cc | 4 +- src/libstore/daemon.cc | 34 ++++----- src/libstore/derivations.cc | 4 +- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 14 ++-- src/libstore/remote-store.cc | 116 +++++++++++++------------------ src/libstore/store-api.cc | 16 ++--- src/libstore/worker-protocol.hh | 59 ++++++++++------ src/nix-store/nix-store.cc | 16 ++--- 9 files changed, 132 insertions(+), 135 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1f40dc42a..dc636c33f 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1864,11 +1864,11 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - writeStorePaths(worker.store, hook->sink, inputPaths); + write(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - writeStorePaths(worker.store, hook->sink, missingPaths); + write(worker.store, hook->sink, missingPaths); hook->sink = FdSink(); hook->toHook.writeSide = -1; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 9f7be6e1a..f92d384e5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -256,11 +256,11 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryValidPaths: { - auto paths = readStorePaths(*store, from); + auto paths = read(*store, from, Proxy {}); logger->startWork(); auto res = store->queryValidPaths(paths); logger->stopWork(); - writeStorePaths(*store, to, res); + write(*store, to, res); break; } @@ -276,11 +276,11 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQuerySubstitutablePaths: { - auto paths = readStorePaths(*store, from); + auto paths = read(*store, from, Proxy {}); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - writeStorePaths(*store, to, res); + write(*store, to, res); break; } @@ -309,7 +309,7 @@ static void performOp(TunnelLogger * logger, ref store, paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - writeStorePaths(*store, to, paths); + write(*store, to, paths); break; } @@ -397,7 +397,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - auto refs = readStorePaths(*store, from); + auto refs = read(*store, from, Proxy {}); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -518,7 +518,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths(*store, from); + options.pathsToDelete = read(*store, from, Proxy {}); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -587,7 +587,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - writeStorePaths(*store, to, i->second.references); + write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -598,11 +598,11 @@ static void performOp(TunnelLogger * logger, ref store, SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = readStorePaths(*store, from); + auto paths = read(*store, from, Proxy {}); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = readStorePathCAMap(*store, from); + pathsMap = read(*store, from, Proxy {}); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); @@ -610,7 +610,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - writeStorePaths(*store, to, i.second.references); + write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - writeStorePaths(*store, to, paths); + write(*store, to, paths); break; } @@ -639,7 +639,7 @@ static void performOp(TunnelLogger * logger, ref store, to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") << info->narHash->to_string(Base16, false); - writeStorePaths(*store, to, info->references); + write(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate @@ -699,7 +699,7 @@ static void performOp(TunnelLogger * logger, ref store, if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths(*store, from); + info.references = read(*store, from, Proxy {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); @@ -799,9 +799,9 @@ static void performOp(TunnelLogger * logger, ref store, uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - writeStorePaths(*store, to, willBuild); - writeStorePaths(*store, to, willSubstitute); - writeStorePaths(*store, to, unknown); + write(*store, to, willBuild); + write(*store, to, willSubstitute); + write(*store, to, unknown); to << downloadSize << narSize; break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7d0a5abeb..5972b5ad2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -477,7 +477,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = readStorePaths(store, in); + drv.inputSrcs = read(store, in, Proxy {}); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -505,7 +505,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr out << "" << ""; } } - writeStorePaths(store, out, drv.inputSrcs); + write(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index a0fc22264..c20c56156 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - writeStorePaths(*this, teeSink, info->references); + write(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths(*this, source); + info.references = read(*this, source, Proxy {}); auto deriver = readString(source); if (deriver != "") diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5d7566121..412e1950b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -107,7 +107,7 @@ struct LegacySSHStore : public Store auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = readStorePaths(*this, conn->from); + info->references = read(*this, conn->from, Proxy {}); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash->to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -168,7 +168,7 @@ struct LegacySSHStore : public Store conn->to << exportMagic << printStorePath(info.path); - writeStorePaths(*this, conn->to, info.references); + write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -251,10 +251,10 @@ struct LegacySSHStore : public Store conn->to << cmdQueryClosure << includeOutputs; - writeStorePaths(*this, conn->to, paths); + write(*this, conn->to, paths); conn->to.flush(); - for (auto & i : readStorePaths(*this, conn->from)) + for (auto & i : read(*this, conn->from, Proxy {})) out.insert(i); } @@ -267,10 +267,10 @@ struct LegacySSHStore : public Store << cmdQueryValidPaths << false // lock << maybeSubstitute; - writeStorePaths(*this, conn->to, paths); + write(*this, conn->to, paths); conn->to.flush(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } void connect() override diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1200ab200..de50b3e2e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -23,66 +23,44 @@ namespace nix { -template<> StorePathSet readStorePaths(const Store & store, Source & from) -{ - StorePathSet paths; - for (auto & i : readStrings(from)) - paths.insert(store.parseStorePath(i)); - return paths; -} - - -void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths) +void write(const Store & store, Sink & out, const StorePathSet & paths) { out << paths.size(); for (auto & i : paths) out << store.printStorePath(i); } + +std::string read(const Store & store, Source & from, Proxy _) +{ + return readString(from); +} + +void write(const Store & store, Sink & out, const std::string & str) +{ + out << str; +} + + StorePath read(const Store & store, Source & from, Proxy _) { - auto path = readString(from); - return store.parseStorePath(path); + return store.parseStorePath(readString(from)); } -StorePathCAMap readStorePathCAMap(const Store & store, Source & from) -{ - StorePathCAMap paths; - auto count = readNum(from); - while (count--) - paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); - return paths; -} - -void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths) -{ - out << paths.size(); - for (auto & i : paths) { - out << store.printStorePath(i.first); - out << renderContentAddress(i.second); - } -} - -std::map readOutputPathMap(const Store & store, Source & from) -{ - std::map pathMap; - auto rawInput = readStrings(from); - if (rawInput.size() % 2) - throw Error("got an odd number of elements from the daemon when trying to read a output path map"); - auto curInput = rawInput.begin(); - while (curInput != rawInput.end()) { - auto thisKey = *curInput++; - auto thisValue = *curInput++; - pathMap.emplace(thisKey, store.parseStorePath(thisValue)); - } - return pathMap; -} - - void write(const Store & store, Sink & out, const StorePath & storePath) { - auto path = store.printStorePath(storePath); - out << path; + out << store.printStorePath(storePath); +} + + +ContentAddress read(const Store & store, Source & from, Proxy _) +{ + return parseContentAddress(readString(from)); +} + +void write(const Store & store, Sink & out, const ContentAddress & ca) +{ + out << renderContentAddress(ca); } @@ -319,9 +297,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - writeStorePaths(*this, conn->to, paths); + write(*this, conn->to, paths); conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } } @@ -331,7 +309,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } @@ -348,9 +326,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - writeStorePaths(*this, conn->to, paths); + write(*this, conn->to, paths); conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } } @@ -372,7 +350,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.references = read(*this, conn->from, Proxy {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -385,9 +363,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - writeStorePaths(*this, conn->to, paths); + write(*this, conn->to, paths); } else - writeStorePathCAMap(*this, conn->to, pathsMap); + write(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -395,7 +373,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.references = read(*this, conn->from, Proxy {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -428,7 +406,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths(*this, conn->from); + info->references = read(*this, conn->from, Proxy {}); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -447,7 +425,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : readStorePaths(*this, conn->from)) + for (auto & i : read(*this, conn->from, Proxy {})) referrers.insert(i); } @@ -457,7 +435,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } @@ -469,7 +447,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } @@ -508,7 +486,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - writeStorePaths(*this, sink, info.references); + write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -518,7 +496,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = readStorePaths(*this, conn->from); + auto importedPaths = read(*this, conn->from, Proxy {}); assert(importedPaths.size() <= 1); } @@ -527,7 +505,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash->to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -660,7 +638,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, auto conn(getConnection()); conn->to << wopAddTextToStore << name << s; - writeStorePaths(*this, conn->to, references); + write(*this, conn->to, references); conn.processStderr(); return parseStorePath(readString(conn->from)); @@ -762,7 +740,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << wopCollectGarbage << options.action; - writeStorePaths(*this, conn->to, options.pathsToDelete); + write(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -824,9 +802,9 @@ void RemoteStore::queryMissing(const std::vector & targets ss.push_back(p.to_string(*this)); conn->to << ss; conn.processStderr(); - willBuild = readStorePaths(*this, conn->from); - willSubstitute = readStorePaths(*this, conn->from); - unknown = readStorePaths(*this, conn->from); + willBuild = read(*this, conn->from, Proxy {}); + willSubstitute = read(*this, conn->from, Proxy {}); + unknown = read(*this, conn->from, Proxy {}); conn->from >> downloadSize >> narSize; return; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4c68709ef..e894d2b85 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -362,14 +362,14 @@ bool Store::PathInfoCacheValue::isKnownNow() } OutputPathMap Store::queryDerivationOutputMapAssumeTotal(const StorePath & path) { - auto resp = queryDerivationOutputMap(path); - OutputPathMap result; - for (auto & [outName, optOutPath] : resp) { - if (!optOutPath) - throw Error("output '%s' has no store path mapped to it", outName); - result.insert_or_assign(outName, *optOutPath); - } - return result; + auto resp = queryDerivationOutputMap(path); + OutputPathMap result; + for (auto & [outName, optOutPath] : resp) { + if (!optOutPath) + throw Error("output '%s' has no store path mapped to it", outName); + result.insert_or_assign(outName, *optOutPath); + } + return result; } StorePathSet Store::queryDerivationOutputs(const StorePath & path) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ad5854c85..08eec9b48 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -66,33 +66,50 @@ typedef enum { class Store; struct Source; -template T readStorePaths(const Store & store, Source & from); - -void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); - /* To guide overloading */ template struct Proxy {}; template -std::map read(const Store & store, Source & from, Proxy> _) +std::set read(const Store & store, Source & from, Proxy> _) { - std::map resMap; - auto size = (size_t)readInt(from); + std::set resSet; + auto size = readNum(from); while (size--) { - auto thisKey = readString(from); - resMap.insert_or_assign(std::move(thisKey), read(store, from, Proxy {})); + resSet.insert(read(store, from, Proxy {})); + } + return resSet; +} + +template +void write(const Store & store, Sink & out, const std::set & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + write(store, out, key); + } +} + +template +std::map read(const Store & store, Source & from, Proxy> _) +{ + std::map resMap; + auto size = readNum(from); + while (size--) { + resMap.insert_or_assign( + read(store, from, Proxy {}), + read(store, from, Proxy {})); } return resMap; } -template -void write(const Store & store, Sink & out, const std::map & resMap) +template +void write(const Store & store, Sink & out, const std::map & resMap) { out << resMap.size(); - for (auto & i : resMap) { - out << i.first; - write(store, out, i.second); + for (auto & [key, value] : resMap) { + write(store, out, key); + write(store, out, value); } } @@ -106,26 +123,28 @@ std::optional read(const Store & store, Source & from, Proxy case 1: return read(store, from, Proxy {}); default: - throw Error("got an invalid tag bit for std::optional: %#04x", tag); + throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag); } } template void write(const Store & store, Sink & out, const std::optional & optVal) { - out << (optVal ? 1 : 0); + out << (uint64_t) (optVal ? 1 : 0); if (optVal) write(store, out, *optVal); } +std::string read(const Store & store, Source & from, Proxy _); + +void write(const Store & store, Sink & out, const std::string & str); + StorePath read(const Store & store, Source & from, Proxy _); void write(const Store & store, Sink & out, const StorePath & storePath); -StorePathCAMap readStorePathCAMap(const Store & store, Source & from); +ContentAddress read(const Store & store, Source & from, Proxy _); -void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths); - -void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); +void write(const Store & store, Sink & out, const ContentAddress & ca); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7b26970ef..a1fb921ef 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -815,7 +815,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = readStorePaths(*store, in); + auto paths = read(*store, in, Proxy {}); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -845,19 +845,19 @@ static void opServe(Strings opFlags, Strings opArgs) } } - writeStorePaths(*store, out, store->queryValidPaths(paths)); + write(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - auto paths = readStorePaths(*store, in); + auto paths = read(*store, in, Proxy {}); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - writeStorePaths(*store, out, info->references); + write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -885,7 +885,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdExportPaths: { readInt(in); // obsolete - store->exportPaths(readStorePaths(*store, in), out); + store->exportPaths(read(*store, in, Proxy {}), out); break; } @@ -934,9 +934,9 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(readStorePaths(*store, in), + store->computeFSClosure(read(*store, in, Proxy {}), closure, false, includeOutputs); - writeStorePaths(*store, out, closure); + write(*store, out, closure); break; } @@ -949,7 +949,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(in), htSHA256); - info.references = readStorePaths(*store, in); + info.references = read(*store, in, Proxy {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); From 1bab8a321f246c1c202268851b0e706ff5030d2e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 21:56:42 +0000 Subject: [PATCH 003/123] Remove unneeded definition Template instantiations will cover this case fine. --- src/libstore/remote-store.cc | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index de50b3e2e..5de8f95a7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -23,14 +23,6 @@ namespace nix { -void write(const Store & store, Sink & out, const StorePathSet & paths) -{ - out << paths.size(); - for (auto & i : paths) - out << store.printStorePath(i); -} - - std::string read(const Store & store, Source & from, Proxy _) { return readString(from); From 3d8240c32eedd0809450be52e4ac7625ffdad9aa Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 6 Aug 2020 16:04:18 -0400 Subject: [PATCH 004/123] Remove leftover commented code --- src/libstore/worker-protocol.hh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b4c260e26..3bb27ab22 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -136,7 +136,6 @@ void write(const Store & store, Sink & out, const std::map & resMap) { out << resMap.size(); for (auto & i : resMap) { - // out << i.first; write(store, out, i.first); write(store, out, i.second); } @@ -150,7 +149,6 @@ std::optional read(const Store & store, Source & from, Phantom {}); return read(store, from, Phantom {}); default: throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag); From 9ab07e99f527d1fa3adfa02839da477a1528d64b Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 6 Aug 2020 18:04:13 -0400 Subject: [PATCH 005/123] Use template structs instead of phantoms --- src/libstore/build.cc | 4 +- src/libstore/daemon.cc | 36 +++---- src/libstore/derivations.cc | 4 +- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 14 +-- src/libstore/remote-store.cc | 61 ++++++------ src/libstore/worker-protocol.hh | 165 +++++++++++++++---------------- src/nix-store/nix-store.cc | 16 +-- 8 files changed, 148 insertions(+), 156 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8340828e7..e1c360338 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1864,11 +1864,11 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - nix::worker_proto::write(worker.store, hook->sink, inputPaths); + WorkerProto::write(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - nix::worker_proto::write(worker.store, hook->sink, missingPaths); + WorkerProto::write(worker.store, hook->sink, missingPaths); hook->sink = FdSink(); hook->toHook.writeSide = -1; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6d734abdc..148bd5cc9 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -256,11 +256,11 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryValidPaths: { - auto paths = nix::worker_proto::read(*store, from, Phantom {}); + auto paths = WorkerProto::read(*store, from); logger->startWork(); auto res = store->queryValidPaths(paths); logger->stopWork(); - nix::worker_proto::write(*store, to, res); + WorkerProto::write(*store, to, res); break; } @@ -276,11 +276,11 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQuerySubstitutablePaths: { - auto paths = nix::worker_proto::read(*store, from, Phantom {}); + auto paths = WorkerProto::read(*store, from); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - nix::worker_proto::write(*store, to, res); + WorkerProto::write(*store, to, res); break; } @@ -309,7 +309,7 @@ static void performOp(TunnelLogger * logger, ref store, paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - nix::worker_proto::write(*store, to, paths); + WorkerProto::write(*store, to, paths); break; } @@ -327,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto outputs = store->queryDerivationOutputMap(path); logger->stopWork(); - nix::worker_proto::write(*store, to, outputs); + WorkerProto>>::write(*store, to, outputs); break; } @@ -397,7 +397,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - auto refs = nix::worker_proto::read(*store, from, Phantom {}); + auto refs = WorkerProto::read(*store, from); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -518,7 +518,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = nix::worker_proto::read(*store, from, Phantom {}); + options.pathsToDelete = WorkerProto::read(*store, from); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -587,7 +587,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - nix::worker_proto::write(*store, to, i->second.references); + WorkerProto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -598,11 +598,11 @@ static void performOp(TunnelLogger * logger, ref store, SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = nix::worker_proto::read(*store, from, Phantom {}); + auto paths = WorkerProto::read(*store, from); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = nix::worker_proto::read(*store, from, Phantom {}); + pathsMap = WorkerProto::read(*store, from); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); @@ -610,7 +610,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - nix::worker_proto::write(*store, to, i.second.references); + WorkerProto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - nix::worker_proto::write(*store, to, paths); + WorkerProto::write(*store, to, paths); break; } @@ -639,7 +639,7 @@ static void performOp(TunnelLogger * logger, ref store, to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") << info->narHash->to_string(Base16, false); - nix::worker_proto::write(*store, to, info->references); + WorkerProto::write(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate @@ -699,7 +699,7 @@ static void performOp(TunnelLogger * logger, ref store, if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash::parseAny(readString(from), htSHA256); - info.references = nix::worker_proto::read(*store, from, Phantom {}); + info.references = WorkerProto::read(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); @@ -799,9 +799,9 @@ static void performOp(TunnelLogger * logger, ref store, uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - nix::worker_proto::write(*store, to, willBuild); - nix::worker_proto::write(*store, to, willSubstitute); - nix::worker_proto::write(*store, to, unknown); + WorkerProto::write(*store, to, willBuild); + WorkerProto::write(*store, to, willSubstitute); + WorkerProto::write(*store, to, unknown); to << downloadSize << narSize; break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b7aef9d65..bf2758ae5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -605,7 +605,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = nix::worker_proto::read(store, in, Phantom {}); + drv.inputSrcs = WorkerProto::read(store, in); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -640,7 +640,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.output); } - nix::worker_proto::write(store, out, drv.inputSrcs); + WorkerProto::write(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 7551294f7..e2e52b0af 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - nix::worker_proto::write(*this, teeSink, info->references); + WorkerProto::write(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = nix::worker_proto::read(*this, source, Phantom {}); + info.references = WorkerProto::read(*this, source); auto deriver = readString(source); if (deriver != "") diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 0951610d3..ee07b7156 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -107,7 +107,7 @@ struct LegacySSHStore : public Store auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = nix::worker_proto::read(*this, conn->from, Phantom {}); + info->references = WorkerProto::read(*this, conn->from); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash->to_string(Base16, false); - nix::worker_proto::write(*this, conn->to, info.references); + WorkerProto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -168,7 +168,7 @@ struct LegacySSHStore : public Store conn->to << exportMagic << printStorePath(info.path); - nix::worker_proto::write(*this, conn->to, info.references); + WorkerProto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -251,10 +251,10 @@ struct LegacySSHStore : public Store conn->to << cmdQueryClosure << includeOutputs; - nix::worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); conn->to.flush(); - for (auto & i : nix::worker_proto::read(*this, conn->from, Phantom {})) + for (auto & i : WorkerProto::read(*this, conn->from)) out.insert(i); } @@ -267,10 +267,10 @@ struct LegacySSHStore : public Store << cmdQueryValidPaths << false // lock << maybeSubstitute; - nix::worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); conn->to.flush(); - return nix::worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } void connect() override diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ac776b95a..48450eb67 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -22,40 +22,36 @@ namespace nix { -namespace worker_proto { - -std::string read(const Store & store, Source & from, Phantom _) +std::string WorkerProto::read(const Store & store, Source & from) { return readString(from); } -void write(const Store & store, Sink & out, const std::string & str) +void WorkerProto::write(const Store & store, Sink & out, const std::string & str) { out << str; } -StorePath read(const Store & store, Source & from, Phantom _) +StorePath WorkerProto::read(const Store & store, Source & from) { return store.parseStorePath(readString(from)); } -void write(const Store & store, Sink & out, const StorePath & storePath) +void WorkerProto::write(const Store & store, Sink & out, const StorePath & storePath) { out << store.printStorePath(storePath); } -ContentAddress read(const Store & store, Source & from, Phantom _) +ContentAddress WorkerProto::read(const Store & store, Source & from) { return parseContentAddress(readString(from)); } -void write(const Store & store, Sink & out, const ContentAddress & ca) +void WorkerProto::write(const Store & store, Sink & out, const ContentAddress & ca) { out << renderContentAddress(ca); } -} - /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -290,9 +286,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - nix::worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } } @@ -302,7 +298,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return nix::worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -319,9 +315,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - nix::worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } } @@ -343,7 +339,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -356,9 +352,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); } else - worker_proto::write(*this, conn->to, pathsMap); + WorkerProto::write(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -366,7 +362,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -399,7 +395,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash::parseAny(readString(conn->from), htSHA256); - info->references = worker_proto::read(*this, conn->from, Phantom {}); + info->references = WorkerProto::read(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -418,7 +414,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) + for (auto & i : WorkerProto::read(*this, conn->from)) referrers.insert(i); } @@ -428,7 +424,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -440,7 +436,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -449,8 +445,7 @@ std::map> RemoteStore::queryDerivationOutp auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom>> {}); - + return WorkerProto>>::read(*this, conn->from); } std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) @@ -479,7 +474,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + WorkerProto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -489,7 +484,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); + auto importedPaths = WorkerProto::read(*this, conn->from); assert(importedPaths.size() <= 1); } @@ -498,7 +493,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash->to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + WorkerProto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -631,7 +626,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, auto conn(getConnection()); conn->to << wopAddTextToStore << name << s; - worker_proto::write(*this, conn->to, references); + WorkerProto::write(*this, conn->to, references); conn.processStderr(); return parseStorePath(readString(conn->from)); @@ -733,7 +728,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << wopCollectGarbage << options.action; - worker_proto::write(*this, conn->to, options.pathsToDelete); + WorkerProto::write(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -795,9 +790,9 @@ void RemoteStore::queryMissing(const std::vector & targets ss.push_back(p.to_string(*this)); conn->to << ss; conn.processStderr(); - willBuild = worker_proto::read(*this, conn->from, Phantom {}); - willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); - unknown = worker_proto::read(*this, conn->from, Phantom {}); + willBuild = WorkerProto::read(*this, conn->from); + willSubstitute = WorkerProto::read(*this, conn->from); + unknown = WorkerProto::read(*this, conn->from); conn->from >> downloadSize >> narSize; return; } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 3bb27ab22..60543d626 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -66,105 +66,102 @@ typedef enum { class Store; struct Source; -/* To guide overloading */ template -struct Phantom {}; +struct WorkerProto { + static T read(const Store & store, Source & from); + static void write(const Store & store, Sink & out, const T & t); +}; +template<> +struct WorkerProto { + static std::string read(const Store & store, Source & from); + static void write(const Store & store, Sink & out, const std::string & t); +}; -namespace worker_proto { -/* FIXME maybe move more stuff inside here */ +template<> +struct WorkerProto { + static StorePath read(const Store & store, Source & from); + static void write(const Store & store, Sink & out, const StorePath & t); +}; -std::string read(const Store & store, Source & from, Phantom _); -void write(const Store & store, Sink & out, const std::string & str); - -StorePath read(const Store & store, Source & from, Phantom _); -void write(const Store & store, Sink & out, const StorePath & storePath); - -ContentAddress read(const Store & store, Source & from, Phantom _); -void write(const Store & store, Sink & out, const ContentAddress & ca); +template<> +struct WorkerProto { + static ContentAddress read(const Store & store, Source & from); + static void write(const Store & store, Sink & out, const ContentAddress & t); +}; template -std::set read(const Store & store, Source & from, Phantom> _); -template -void write(const Store & store, Sink & out, const std::set & resSet); +struct WorkerProto> { + + static std::set read(const Store & store, Source & from) + { + std::set resSet; + auto size = readNum(from); + while (size--) { + resSet.insert(WorkerProto::read(store, from)); + } + return resSet; + } + + static void write(const Store & store, Sink & out, const std::set & resSet) + { + out << resSet.size(); + for (auto & key : resSet) { + WorkerProto::write(store, out, key); + } + } + +}; template -std::map read(const Store & store, Source & from, Phantom> _); -template -void write(const Store & store, Sink & out, const std::map & resMap); +struct WorkerProto> { -template -std::optional read(const Store & store, Source & from, Phantom> _); -template -void write(const Store & store, Sink & out, const std::optional & optVal); - -template -std::set read(const Store & store, Source & from, Phantom> _) -{ - std::set resSet; - auto size = readNum(from); - while (size--) { - resSet.insert(read(store, from, Phantom {})); + static std::map read(const Store & store, Source & from) + { + std::map resMap; + auto size = readNum(from); + while (size--) { + resMap.insert_or_assign( + WorkerProto::read(store, from), + WorkerProto::read(store, from)); + } + return resMap; } - return resSet; -} + + static void write(const Store & store, Sink & out, const std::map & resMap) + { + out << resMap.size(); + for (auto & i : resMap) { + WorkerProto::write(store, out, i.first); + WorkerProto::write(store, out, i.second); + } + } + +}; template -void write(const Store & store, Sink & out, const std::set & resSet) -{ - out << resSet.size(); - for (auto & key : resSet) { - write(store, out, key); +struct WorkerProto> { + + static std::optional read(const Store & store, Source & from) + { + auto tag = readNum(from); + switch (tag) { + case 0: + return std::nullopt; + case 1: + return WorkerProto::read(store, from); + default: + throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag); + } } -} -template -std::map read(const Store & store, Source & from, Phantom> _) -{ - std::map resMap; - auto size = readNum(from); - while (size--) { - resMap.insert_or_assign( - read(store, from, Phantom {}), - read(store, from, Phantom {})); + static void write(const Store & store, Sink & out, const std::optional & optVal) + { + out << (uint64_t) (optVal ? 1 : 0); + if (optVal) + WorkerProto::write(store, out, *optVal); } - return resMap; -} - -template -void write(const Store & store, Sink & out, const std::map & resMap) -{ - out << resMap.size(); - for (auto & i : resMap) { - write(store, out, i.first); - write(store, out, i.second); - } -} - -template -std::optional read(const Store & store, Source & from, Phantom> _) -{ - auto tag = readNum(from); - switch (tag) { - case 0: - return std::nullopt; - case 1: - return read(store, from, Phantom {}); - default: - throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag); - } -} - -template -void write(const Store & store, Sink & out, const std::optional & optVal) -{ - out << (uint64_t) (optVal ? 1 : 0); - if (optVal) - nix::worker_proto::write(store, out, *optVal); -} - - -} +}; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index a0007f1c4..e59009d11 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -815,7 +815,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = nix::worker_proto::read(*store, in, Phantom {}); + auto paths = WorkerProto::read(*store, in); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -845,19 +845,19 @@ static void opServe(Strings opFlags, Strings opArgs) } } - nix::worker_proto::write(*store, out, store->queryValidPaths(paths)); + WorkerProto::write(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - auto paths = nix::worker_proto::read(*store, in, Phantom {}); + auto paths = WorkerProto::read(*store, in); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - nix::worker_proto::write(*store, out, info->references); + WorkerProto::write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -885,7 +885,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdExportPaths: { readInt(in); // obsolete - store->exportPaths(nix::worker_proto::read(*store, in, Phantom {}), out); + store->exportPaths(WorkerProto::read(*store, in), out); break; } @@ -934,9 +934,9 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(nix::worker_proto::read(*store, in, Phantom {}), + store->computeFSClosure(WorkerProto::read(*store, in), closure, false, includeOutputs); - nix::worker_proto::write(*store, out, closure); + WorkerProto::write(*store, out, closure); break; } @@ -949,7 +949,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash::parseAny(readString(in), htSHA256); - info.references = nix::worker_proto::read(*store, in, Phantom {}); + info.references = WorkerProto::read(*store, in); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); From 46f9dd56da7d2af82148c47e40108f3c11ffe4d7 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 6 Aug 2020 19:30:05 -0400 Subject: [PATCH 006/123] Fix bug due to non-deterministic arg eval order --- src/libstore/worker-protocol.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 60543d626..1e8fd027c 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -121,9 +121,9 @@ struct WorkerProto> { std::map resMap; auto size = readNum(from); while (size--) { - resMap.insert_or_assign( - WorkerProto::read(store, from), - WorkerProto::read(store, from)); + auto k = WorkerProto::read(store, from); + auto v = WorkerProto::read(store, from); + resMap.insert_or_assign(std::move(k), std::move(v)); } return resMap; } From 59979e705352abb1624d3427c2c7145ed43b1b84 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Aug 2020 18:10:58 +0000 Subject: [PATCH 007/123] Fix bad debug format string --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ba28e78c8..6baaa31d9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3850,7 +3850,7 @@ void DerivationGoal::registerOutputs() something like that. */ canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - debug("scanning for references for output %1 in temp location '%1%'", outputName, actualPath); + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); /* Pass blank Sink as we are not ready to hash data at this stage. */ NullSink blank; From e0b0e18905835fdc8ccbbf1c0f5d016d9f466187 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 23 Aug 2020 15:27:30 +0000 Subject: [PATCH 008/123] Add constructor for BasicDerivation -> Derivation --- src/libstore/derivations.hh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b09480e1e..2ea4178c0 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -100,7 +100,7 @@ struct BasicDerivation StringPairs env; std::string name; - BasicDerivation() { } + BasicDerivation() = default; virtual ~BasicDerivation() { }; bool isBuiltin() const; @@ -127,7 +127,8 @@ struct Derivation : BasicDerivation std::string unparse(const Store & store, bool maskOutputs, std::map * actualInputs = nullptr) const; - Derivation() { } + Derivation() = default; + Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } }; From 8eb73a87245acf9d93dc401831b629981864fa58 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 22 Aug 2020 20:44:47 +0000 Subject: [PATCH 009/123] CA derivations that depend on other CA derivations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt --- src/libstore/build.cc | 51 +++++++++++++++++++++++++++++++---- src/libstore/derivations.cc | 53 +++++++++++++++++++++++++++++++++++++ src/libstore/derivations.hh | 18 +++++++++++++ src/libstore/local-store.cc | 27 +++++++++++++++++-- tests/content-addressed.nix | 48 ++++++++++++++++++++++++--------- tests/content-addressed.sh | 20 +++++++------- 6 files changed, 189 insertions(+), 28 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6baaa31d9..f5256bf87 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -984,6 +984,8 @@ private: void tryLocalBuild(); void buildDone(); + void resolvedFinished(); + /* Is the build hook willing to perform the build? */ HookReply tryBuildHook(); @@ -1451,8 +1453,39 @@ void DerivationGoal::inputsRealised() /* Determine the full set of input paths. */ /* First, the input derivations. */ - if (useDerivation) - for (auto & [depDrvPath, wantedDepOutputs] : dynamic_cast(drv.get())->inputDrvs) { + if (useDerivation) { + auto & fullDrv = *dynamic_cast(drv.get()); + + if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { + /* We are be able to resolve this derivation based on the + now-known results of dependencies. If so, we become a stub goal + aliasing that resolved derivation goal */ + Derivation drvResolved { fullDrv.resolve(worker.store) }; + + auto pathResolved = writeDerivation(worker.store, drvResolved); + /* Add to memotable to speed up downstream goal's queries with the + original derivation. */ + drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); + + auto msg = fmt("Resolved derivation: '%s' -> '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved)); + act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, + Logger::Fields { + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved), + }); + + auto resolvedGoal = worker.makeDerivationGoal( + pathResolved, wantedOutputs, + buildMode == bmRepair ? bmRepair : bmNormal); + addWaitee(resolvedGoal); + + state = &DerivationGoal::resolvedFinished; + return; + } + + for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ @@ -1472,6 +1505,7 @@ void DerivationGoal::inputsRealised() worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); } } + } /* Second, the input sources. */ worker.store.computeFSClosure(drv->inputSrcs, inputPaths); @@ -1893,6 +1927,9 @@ void DerivationGoal::buildDone() done(BuildResult::Built); } +void DerivationGoal::resolvedFinished() { + done(BuildResult::Built); +} HookReply DerivationGoal::tryBuildHook() { @@ -2065,7 +2102,7 @@ void linkOrCopy(const Path & from, const Path & to) file (e.g. 32000 of ext3), which is quite possible after a 'nix-store --optimise'. FIXME: actually, why don't we just bind-mount in this case? - + It can also fail with EPERM in BeegFS v7 and earlier versions which don't allow hard-links to other directories */ if (errno != EMLINK && errno != EPERM) @@ -4248,10 +4285,14 @@ void DerivationGoal::registerOutputs() { ValidPathInfos infos2; for (auto & [outputName, newInfo] : infos) { - /* FIXME: we will want to track this mapping in the DB whether or - not we have a drv file. */ if (useDerivation) worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path); + else { + /* Once a floating CA derivations reaches this point, it must + already be resolved, drvPath the basic derivation path, and + a file existsing at that path for sake of the DB's foreign key. */ + assert(drv->type() != DerivationType::CAFloating); + } infos2.push_back(newInfo); } worker.store.registerValidPaths(infos2); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 34541227b..d96d4083d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -672,4 +672,57 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath return "/" + hashString(htSHA256, clearText).to_string(Base32, false); } + +// N.B. Outputs are left unchanged +static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { + + debug("Rewriting the derivation"); + + for (auto &rewrite: rewrites) { + debug("rewriting %s as %s", rewrite.first, rewrite.second); + } + + drv.builder = rewriteStrings(drv.builder, rewrites); + for (auto & arg: drv.args) { + arg = rewriteStrings(arg, rewrites); + } + + StringPairs newEnv; + for (auto & envVar: drv.env) { + auto envName = rewriteStrings(envVar.first, rewrites); + auto envValue = rewriteStrings(envVar.second, rewrites); + newEnv.emplace(envName, envValue); + } + drv.env = newEnv; +} + + +Sync drvPathResolutions; + +BasicDerivation Derivation::resolve(Store & store) { + BasicDerivation resolved { *this }; + + // Input paths that we'll want to rewrite in the derivation + StringMap inputRewrites; + + for (auto & input : inputDrvs) { + auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first); + StringSet newOutputNames; + for (auto & outputName : input.second) { + auto actualPathOpt = inputDrvOutputs.at(outputName); + if (!actualPathOpt) + throw Error("input drv '%s' wasn't yet built", store.printStorePath(input.first)); + auto actualPath = *actualPathOpt; + inputRewrites.emplace( + downstreamPlaceholder(store, input.first, outputName), + store.printStorePath(actualPath)); + resolved.inputSrcs.insert(std::move(actualPath)); + } + } + + rewriteDerivation(store, resolved, inputRewrites); + + return resolved; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 2ea4178c0..0bb565e8a 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "hash.hh" #include "content-address.hh" +#include "sync.hh" #include #include @@ -127,6 +128,13 @@ struct Derivation : BasicDerivation std::string unparse(const Store & store, bool maskOutputs, std::map * actualInputs = nullptr) const; + /* Return the underlying basic derivation but with + + 1. input drv outputs moved to input sources. + + 2. placeholders replaced with realized input store paths. */ + BasicDerivation resolve(Store & store); + Derivation() = default; Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } }; @@ -187,6 +195,16 @@ typedef std::map DrvHashes; extern DrvHashes drvHashes; // FIXME: global, not thread-safe +/* Memoisation of `readDerivation(..).resove()`. */ +typedef std::map< + StorePath, + std::optional +> DrvPathResolutions; + +// FIXME: global, though at least thread-safe. +// FIXME: arguably overlaps with hashDerivationModulo memo table. +extern Sync drvPathResolutions; + bool wantOutput(const string & output, const std::set & wanted); struct Source; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0086bb13e..e51d127b3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -803,13 +803,36 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) } -std::map> LocalStore::queryPartialDerivationOutputMap(const StorePath & path) +std::map> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) { + auto path = path_; std::map> outputs; - BasicDerivation drv = readDerivation(path); + Derivation drv = readDerivation(path); for (auto & [outName, _] : drv.outputs) { outputs.insert_or_assign(outName, std::nullopt); } + bool haveCached = false; + { + auto resolutions = drvPathResolutions.lock(); + auto resolvedPathOptIter = resolutions->find(path); + if (resolvedPathOptIter != resolutions->end()) { + auto & [_, resolvedPathOpt] = *resolvedPathOptIter; + if (resolvedPathOpt) + path = *resolvedPathOpt; + haveCached = true; + } + } + /* can't just use else-if instead of `!haveCached` because we need to unlock + `drvPathResolutions` before it is locked in `Derivation::resolve`. */ + if (!haveCached && drv.type() == DerivationType::CAFloating) { + /* Resolve drv and use that path instead. */ + auto pathResolved = writeDerivation(*this, drv.resolve(*this)); + /* Store in memo table. */ + /* FIXME: memo logic should not be local-store specific, should have + wrapper-method instead. */ + drvPathResolutions.lock()->insert_or_assign(path, pathResolved); + path = std::move(pathResolved); + } return retrySQLite>>([&]() { auto state(_state.lock()); diff --git a/tests/content-addressed.nix b/tests/content-addressed.nix index 586e4cba6..a46c21164 100644 --- a/tests/content-addressed.nix +++ b/tests/content-addressed.nix @@ -4,16 +4,40 @@ with import ./config.nix; # A simple content-addressed derivation. # The derivation can be arbitrarily modified by passing a different `seed`, # but the output will always be the same -mkDerivation { - name = "simple-content-addressed"; - buildCommand = '' - set -x - echo "Building a CA derivation" - echo "The seed is ${toString seed}" - mkdir -p $out - echo "Hello World" > $out/hello - ''; - __contentAddressed = true; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; +rec { + root = mkDerivation { + name = "simple-content-addressed"; + buildCommand = '' + set -x + echo "Building a CA derivation" + echo "The seed is ${toString seed}" + mkdir -p $out + echo "Hello World" > $out/hello + ''; + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + }; + dependent = mkDerivation { + name = "dependent"; + buildCommand = '' + echo "building a dependent derivation" + mkdir -p $out + echo ${root}/hello > $out/dep + ''; + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + }; + transitivelyDependent = mkDerivation { + name = "transitively-dependent"; + buildCommand = '' + echo "building transitively-dependent" + cat ${dependent}/dep + echo ${dependent} > $out + ''; + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + }; } diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 2968f3a8c..522310585 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -2,15 +2,17 @@ source common.sh -clearStore -clearCache - -export REMOTE_STORE=file://$cacheDir - -drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix --arg seed 1) +drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A root --arg seed 1) nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1 -out1=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 1 --no-out-link) -out2=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 2 --no-out-link) +testDerivation () { + local derivationPath=$1 + local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link") + local out1=$(nix-build "${commonArgs[@]}" --arg seed 1) + local out2=$(nix-build "${commonArgs[@]}" --arg seed 2) + test $out1 == $out2 +} -test $out1 == $out2 +testDerivation root +testDerivation dependent +testDerivation transitivelyDependent From 421ed527c70201c722cbefc10576ae77e383ba8e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 28 Aug 2020 17:22:57 -0400 Subject: [PATCH 010/123] Update src/libstore/build.cc Thanks for catching, @regnat. --- src/libstore/build.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f5256bf87..1249668c4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1477,8 +1477,7 @@ void DerivationGoal::inputsRealised() }); auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, - buildMode == bmRepair ? bmRepair : bmNormal); + pathResolved, wantedOutputs, buildMode); addWaitee(resolvedGoal); state = &DerivationGoal::resolvedFinished; From 4db0010a9374e357de3db3c0cf1cb1b490a14727 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 28 Aug 2020 22:03:54 +0000 Subject: [PATCH 011/123] Test CA derivation input caching --- tests/content-addressed.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index 522310585..5997a432f 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -9,10 +9,13 @@ testDerivation () { local derivationPath=$1 local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link") local out1=$(nix-build "${commonArgs[@]}" --arg seed 1) - local out2=$(nix-build "${commonArgs[@]}" --arg seed 2) + local out2=$(nix-build "${commonArgs[@]}" --arg seed 2 "${extraArgs[@]}") test $out1 == $out2 } testDerivation root +# The seed only changes the root derivation, and not it's output, so the +# dependent derivations should only need to be built once. +extaArgs=(-j0) testDerivation dependent testDerivation transitivelyDependent From aad4abcc9c27d5c1a2349e40f51f076387e0f844 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 4 Sep 2020 01:17:38 +0000 Subject: [PATCH 012/123] Fix floating CA tests We will sometimes try to query the outputs of derivations we can't resolve. That's fine; it just means we don't know what those outputs are yet. --- src/libstore/build.cc | 4 +++- src/libstore/derivations.cc | 4 ++-- src/libstore/derivations.hh | 2 +- src/libstore/local-store.cc | 9 +++++++-- tests/content-addressed.sh | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1249668c4..1f77b8ea8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1460,7 +1460,9 @@ void DerivationGoal::inputsRealised() /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ - Derivation drvResolved { fullDrv.resolve(worker.store) }; + std::optional attempt = fullDrv.tryResolve(worker.store); + assert(attempt); + Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); /* Add to memotable to speed up downstream goal's queries with the diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ce57a5bb0..695265860 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -671,7 +671,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String Sync drvPathResolutions; -BasicDerivation Derivation::resolve(Store & store) { +std::optional Derivation::tryResolve(Store & store) { BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation @@ -683,7 +683,7 @@ BasicDerivation Derivation::resolve(Store & store) { for (auto & outputName : input.second) { auto actualPathOpt = inputDrvOutputs.at(outputName); if (!actualPathOpt) - throw Error("input drv '%s' wasn't yet built", store.printStorePath(input.first)); + return std::nullopt; auto actualPath = *actualPathOpt; inputRewrites.emplace( downstreamPlaceholder(store, input.first, outputName), diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index e4d85aa05..eaffbf452 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -133,7 +133,7 @@ struct Derivation : BasicDerivation 1. input drv outputs moved to input sources. 2. placeholders replaced with realized input store paths. */ - BasicDerivation resolve(Store & store); + std::optional tryResolve(Store & store); Derivation() = default; Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e51d127b3..f490188ce 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -825,8 +825,13 @@ std::map> LocalStore::queryPartialDerivati /* can't just use else-if instead of `!haveCached` because we need to unlock `drvPathResolutions` before it is locked in `Derivation::resolve`. */ if (!haveCached && drv.type() == DerivationType::CAFloating) { - /* Resolve drv and use that path instead. */ - auto pathResolved = writeDerivation(*this, drv.resolve(*this)); + /* Try resolve drv and use that path instead. */ + auto attempt = drv.tryResolve(*this); + if (!attempt) + /* If we cannot resolve the derivation, we cannot have any path + assigned so we return the map of all std::nullopts. */ + return outputs; + auto pathResolved = writeDerivation(*this, *std::move(attempt)); /* Store in memo table. */ /* FIXME: memo logic should not be local-store specific, should have wrapper-method instead. */ diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index b2e94fe1e..34334b22d 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -9,7 +9,7 @@ testDerivation () { local derivationPath=$1 local commonArgs=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" "--no-out-link") local out1 out2 - out1=$(set -e; nix-build "${commonArgs[@]}" --arg seed 1) + out1=$(nix-build "${commonArgs[@]}" --arg seed 1) out2=$(nix-build "${commonArgs[@]}" --arg seed 2 "${secondSeedArgs[@]}") test "$out1" == "$out2" } From 98dfd7531d6df6abc925a446f390c4a5bbb9a51d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 4 Sep 2020 18:33:58 +0000 Subject: [PATCH 013/123] Fix querying outputs for CA derivations some more If we resolve using the known path of a derivation whose output we didn't have, we previously blew up. Now we just fail gracefully, returning the map of all outputs unknown. --- src/libstore/derivations.cc | 4 ++-- src/libstore/derivations.hh | 4 +++- src/libstore/local-store.cc | 20 ++++++++++++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 695265860..afac00fc4 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -69,7 +69,7 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(Store & store, - const Derivation & drv, RepairFlag repair) + const Derivation & drv, RepairFlag repair, bool readOnly) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs) @@ -79,7 +79,7 @@ StorePath writeDerivation(Store & store, held during a garbage collection). */ auto suffix = std::string(drv.name) + drvExtension; auto contents = drv.unparse(store, false); - return settings.readOnlyMode + return readOnly || settings.readOnlyMode ? store.computeStorePathForText(suffix, contents, references) : store.addTextToStore(suffix, contents, references, repair); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 8aa496143..716862127 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -146,7 +146,9 @@ enum RepairFlag : bool { NoRepair = false, Repair = true }; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(Store & store, - const Derivation & drv, RepairFlag repair = NoRepair); + const Derivation & drv, + RepairFlag repair = NoRepair, + bool readOnly = false); /* Read a derivation from a file. */ Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f490188ce..0755cfa91 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -728,7 +728,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); if (!use.next()) - throw Error("path '%s' is not valid", printStorePath(path)); + throw InvalidPath("path '%s' is not valid", printStorePath(path)); return use.getInt(0); } @@ -831,7 +831,8 @@ std::map> LocalStore::queryPartialDerivati /* If we cannot resolve the derivation, we cannot have any path assigned so we return the map of all std::nullopts. */ return outputs; - auto pathResolved = writeDerivation(*this, *std::move(attempt)); + /* Just compute store path */ + auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true); /* Store in memo table. */ /* FIXME: memo logic should not be local-store specific, should have wrapper-method instead. */ @@ -841,8 +842,19 @@ std::map> LocalStore::queryPartialDerivati return retrySQLite>>([&]() { auto state(_state.lock()); - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); + uint64_t drvId; + try { + drvId = queryValidPathId(*state, path); + } catch (InvalidPath &) { + /* FIXME? if the derivation doesn't exist, we cannot have a mapping + for it. */ + return outputs; + } + + auto useQueryDerivationOutputs { + state->stmtQueryDerivationOutputs.use() + (drvId) + }; while (useQueryDerivationOutputs.next()) outputs.insert_or_assign( From a303c0b6dc71b1e0d6a57986c3f7a9b61361cd92 Mon Sep 17 00:00:00 2001 From: Greg Hale Date: Wed, 17 Jun 2020 15:08:59 -0400 Subject: [PATCH 014/123] Fetch commits from github/gitlab using Auth header `nix flake info` calls the github 'commits' API, which requires authorization when the repository is private. Currently this request fails with a 404. This commit adds an authorization header when calling the 'commits' API. It also changes the way that the 'tarball' API authenticates, moving the user's token from a query parameter into the Authorization header. The query parameter method is recently deprecated and will be disallowed in November 2020. Using them today triggers a warning email. --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops/fetchTree.cc | 4 +- src/libfetchers/fetchers.hh | 2 + src/libfetchers/github.cc | 76 +++++++++++++++++++++++--------- src/libfetchers/registry.cc | 2 +- src/libfetchers/tarball.cc | 9 ++-- src/libstore/filetransfer.cc | 3 ++ src/libstore/filetransfer.hh | 4 ++ src/libstore/globals.hh | 3 ++ src/libutil/types.hh | 2 + src/nix-channel/nix-channel.cc | 6 +-- 12 files changed, 84 insertions(+), 31 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 10c1a6975..d71aa22f1 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -76,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s) if (isUri(s)) { return state.store->toRealPath( fetchers::downloadTarball( - state.store, resolveUri(s), "source", false).first.storePath); + state.store, resolveUri(s), Headers {}, "source", false).first.storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 24b21f7da..28e31f46b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -719,7 +719,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), "source", false).first.storePath) }; + store, resolveUri(elem.second), Headers {}, "source", false).first.storePath) }; } catch (FileTransferError & e) { logWarning({ .name = "Entry download", diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 06e8304b8..3001957b4 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -201,8 +201,8 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath - : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + ? fetchers::downloadTarball(state.store, *url, Headers {}, name, (bool) expectedHash).first.storePath + : fetchers::downloadFile(state.store, *url, Headers{}, name, (bool) expectedHash).storePath; auto path = state.store->toRealPath(storePath); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 89b1e6e7d..62807e53b 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -118,12 +118,14 @@ struct DownloadFileResult DownloadFileResult downloadFile( ref store, const std::string & url, + const Headers & headers, const std::string & name, bool immutable); std::pair downloadTarball( ref store, const std::string & url, + const Headers & headers, const std::string & name, bool immutable); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1cc0c5e2e..d8d0351b9 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -3,11 +3,24 @@ #include "fetchers.hh" #include "globals.hh" #include "store-api.hh" +#include "types.hh" #include namespace nix::fetchers { +struct DownloadUrl +{ + std::string url; + std::optional> access_token_header; + + DownloadUrl(const std::string & url) + : url(url) { } + + DownloadUrl(const std::string & url, const std::pair & access_token_header) + : url(url), access_token_header(access_token_header) { } +}; + // A github or gitlab url const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check std::regex urlRegex(urlRegexS, std::regex::ECMAScript); @@ -16,6 +29,8 @@ struct GitArchiveInputScheme : InputScheme { virtual std::string type() = 0; + virtual std::pair accessHeaderFromToken(const std::string & token) const = 0; + std::optional inputFromURL(const ParsedURL & url) override { if (url.scheme != type()) return {}; @@ -131,7 +146,7 @@ struct GitArchiveInputScheme : InputScheme virtual Hash getRevFromRef(nix::ref store, const Input & input) const = 0; - virtual std::string getDownloadUrl(const Input & input) const = 0; + virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; std::pair fetch(ref store, const Input & _input) override { @@ -160,7 +175,12 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto [tree, lastModified] = downloadTarball(store, url, "source", true); + Headers headers; + if (url.access_token_header) { + headers.push_back(*url.access_token_header); + } + + auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true); input.attrs.insert_or_assign("lastModified", lastModified); @@ -182,11 +202,8 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string type() override { return "github"; } - void addAccessToken(std::string & url) const - { - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken; + std::pair accessHeaderFromToken(const std::string & token) const { + return std::pair("Authorization", fmt("token %s", token)); } Hash getRevFromRef(nix::ref store, const Input & input) const override @@ -195,18 +212,21 @@ struct GitHubInputScheme : GitArchiveInputScheme auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - addAccessToken(url); + Headers headers; + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + headers.push_back(accessHeaderFromToken(accessToken)); auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", false).storePath))); + downloadFile(store, url, headers, "source", false).storePath))); auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } - std::string getDownloadUrl(const Input & input) const override + DownloadUrl getDownloadUrl(const Input & input) const override { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. @@ -215,9 +235,13 @@ struct GitHubInputScheme : GitArchiveInputScheme host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - addAccessToken(url); - - return url; + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") { + auto auth_header = accessHeaderFromToken(accessToken); + return DownloadUrl(url, auth_header); + } else { + return DownloadUrl(url); + } } void clone(const Input & input, const Path & destDir) override @@ -234,21 +258,31 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string type() override { return "gitlab"; } + std::pair accessHeaderFromToken(const std::string & token) const { + return std::pair("Authorization", fmt("Bearer %s", token)); + } + Hash getRevFromRef(nix::ref store, const Input & input) const override { auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + + Headers headers; + std::string accessToken = settings.gitlabAccessToken.get(); + if (accessToken != "") + headers.push_back(accessHeaderFromToken(accessToken)); + auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", false).storePath))); + downloadFile(store, url, headers, "source", false).storePath))); auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } - std::string getDownloadUrl(const Input & input) const override + DownloadUrl getDownloadUrl(const Input & input) const override { // FIXME: This endpoint has a rate limit threshold of 5 requests per minute auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); @@ -256,12 +290,14 @@ struct GitLabInputScheme : GitArchiveInputScheme host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: "`) - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken;*/ + std::string accessToken = settings.gitlabAccessToken.get(); + if (accessToken != "") { + auto auth_header = accessHeaderFromToken(accessToken); + return DownloadUrl(url, auth_header); + } else { + return DownloadUrl(url); + } - return url; } void clone(const Input & input, const Path & destDir) override diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 4367ee810..551e7684a 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -145,7 +145,7 @@ static std::shared_ptr getGlobalRegistry(ref store) auto path = settings.flakeRegistry.get(); if (!hasPrefix(path, "/")) { - auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + auto storePath = downloadFile(store, path, Headers {}, "flake-registry.json", false).storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json"); path = store->toRealPath(storePath); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index a2d16365e..cf6d6e3d2 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -5,12 +5,14 @@ #include "store-api.hh" #include "archive.hh" #include "tarfile.hh" +#include "types.hh" namespace nix::fetchers { DownloadFileResult downloadFile( ref store, const std::string & url, + const Headers & headers, const std::string & name, bool immutable) { @@ -36,7 +38,7 @@ DownloadFileResult downloadFile( if (cached && !cached->expired) return useCached(); - FileTransferRequest request(url); + FileTransferRequest request(url, headers); if (cached) request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); FileTransferResult res; @@ -110,6 +112,7 @@ DownloadFileResult downloadFile( std::pair downloadTarball( ref store, const std::string & url, + const Headers & headers, const std::string & name, bool immutable) { @@ -127,7 +130,7 @@ std::pair downloadTarball( getIntAttr(cached->infoAttrs, "lastModified") }; - auto res = downloadFile(store, url, name, immutable); + auto res = downloadFile(store, url, headers, name, immutable); std::optional unpackedStorePath; time_t lastModified; @@ -222,7 +225,7 @@ struct TarballInputScheme : InputScheme std::pair fetch(ref store, const Input & input) override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), Headers {}, "source", false).first; return {std::move(tree), input}; } }; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 4149f8155..13ed429fa 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -112,6 +112,9 @@ struct curlFileTransfer : public FileTransfer requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); + for (auto it = request.headers.begin(); it != request.headers.end(); ++it){ + requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str()); + } } ~TransferItem() diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 0d608c8d8..7e302ff39 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -51,6 +51,7 @@ extern FileTransferSettings fileTransferSettings; struct FileTransferRequest { std::string uri; + Headers headers; std::string expectedETag; bool verifyTLS = true; bool head = false; @@ -65,6 +66,9 @@ struct FileTransferRequest FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } + FileTransferRequest(const std::string & uri, Headers headers) + : uri(uri), headers(headers) { } + std::string verb() { return data ? "upload" : "download"; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 02721285a..b2e7610ee 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -863,6 +863,9 @@ public: Setting githubAccessToken{this, "", "github-access-token", "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."}; + Setting gitlabAccessToken{this, "", "gitlab-access-token", + "GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."}; + Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 3af485fa0..55d02bcf9 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -24,6 +24,8 @@ typedef string Path; typedef list Paths; typedef set PathSet; +typedef vector> Headers; + /* Helper class to run code at startup. */ template struct OnStartup diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 3ccf620c9..94d33a75c 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -87,7 +87,7 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto result = fetchers::downloadFile(store, url, Headers {}, std::string(baseNameOf(url)), false); auto filename = store->toRealPath(result.storePath); url = result.effectiveUrl; @@ -112,9 +112,9 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", Headers {}, "nixexprs.tar.xz", false).storePath); } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", Headers {}, "nixexprs.tar.bz2", false).storePath); } } From 7fdbb377ba800728a47095008cec11be7d970330 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 9 Sep 2020 14:55:43 -0400 Subject: [PATCH 015/123] Start to fix floating CA + remote building --- src/libstore/build.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ee12f8e67..32980f264 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4613,7 +4613,7 @@ void DerivationGoal::flushLine() std::map> DerivationGoal::queryPartialDerivationOutputMap() { - if (drv->type() != DerivationType::CAFloating) { + if (!useDerivation || drv->type() != DerivationType::CAFloating) { std::map> res; for (auto & [name, output] : drv->outputs) res.insert_or_assign(name, output.path(worker.store, drv->name, name)); @@ -4625,7 +4625,7 @@ std::map> DerivationGoal::queryPartialDeri OutputPathMap DerivationGoal::queryDerivationOutputMap() { - if (drv->type() != DerivationType::CAFloating) { + if (!useDerivation || drv->type() != DerivationType::CAFloating) { OutputPathMap res; for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) res.insert_or_assign(name, *output.second); From 2741fffa350ec59d29ade24dd93007d535a61bde Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 9 Sep 2020 19:13:21 +0000 Subject: [PATCH 016/123] Ensure resolved CA derivations are written so we can link outputs to deriver and thus properly cache. --- src/libstore/build.cc | 33 ++++++++++++++++++++------------- src/libstore/derivations.hh | 1 + 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 32980f264..87c50f0e6 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4298,11 +4298,13 @@ void DerivationGoal::registerOutputs() /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); + { + ValidPathInfos infos2; + for (auto & [outputName, newInfo] : infos) { + infos2.push_back(newInfo); + } + worker.store.registerValidPaths(infos2); } - worker.store.registerValidPaths(infos2); /* In case of a fixed-output derivation hash mismatch, throw an exception now that we have registered the output as valid. */ @@ -4314,16 +4316,21 @@ void DerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - for (auto & [outputName, newInfo] : infos) { - if (useDerivation) - worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path); - else { - /* Once a floating CA derivations reaches this point, it must - already be resolved, drvPath the basic derivation path, and - a file existsing at that path for sake of the DB's foreign key. */ - assert(drv->type() != DerivationType::CAFloating); - } + bool isCaFloating = drv->type() == DerivationType::CAFloating; + + auto drvPath2 = drvPath; + if (!useDerivation && isCaFloating) { + /* Once a floating CA derivations reaches this point, it + must already be resolved, so we don't bother trying to + downcast drv to get would would just be an empty + inputDrvs field. */ + Derivation drv2 { *drv }; + drvPath2 = writeDerivation(worker.store, drv2); } + + if (useDerivation || isCaFloating) + for (auto & [outputName, newInfo] : infos) + worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index adbf8c094..74601134e 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -136,6 +136,7 @@ struct Derivation : BasicDerivation std::optional tryResolve(Store & store); Derivation() = default; + Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } }; From 3f93bc0d39661af52466a316a3daf2e49165755e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 17:38:25 +0200 Subject: [PATCH 017/123] Typo --- src/libutil/split.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/split.hh b/src/libutil/split.hh index d19d7d8ed..87a23b13e 100644 --- a/src/libutil/split.hh +++ b/src/libutil/split.hh @@ -9,7 +9,7 @@ namespace nix { // If `separator` is found, we return the portion of the string before the // separator, and modify the string argument to contain only the part after the -// separator. Otherwise, wer return `std::nullopt`, and we leave the argument +// separator. Otherwise, we return `std::nullopt`, and we leave the argument // string alone. static inline std::optional splitPrefixTo(std::string_view & string, char separator) { auto sepInstance = string.find(separator); From 9ee3122ec71ae43f5cd8bf0a9282777ba17342c5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 13 Sep 2020 14:38:05 +0200 Subject: [PATCH 018/123] Remove redundant import --- src/libstore/daemon.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8adabf549..2e068992b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -2,7 +2,6 @@ #include "monitor-fd.hh" #include "worker-protocol.hh" #include "store-api.hh" -#include "local-store.hh" #include "finally.hh" #include "affinity.hh" #include "archive.hh" From 29c82ccc77714a74e8948ce8c531de5a8d870176 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 13 Sep 2020 14:39:11 +0200 Subject: [PATCH 019/123] Add Source.drainInto(Sink) --- src/libutil/serialise.cc | 13 ++++++++++--- src/libutil/serialise.hh | 2 ++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 00c945113..a469a1e73 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -93,7 +93,7 @@ void Source::operator () (unsigned char * data, size_t len) } -std::string Source::drain() +void Source::drainInto(Sink & sink) { std::string s; std::vector buf(8192); @@ -101,12 +101,19 @@ std::string Source::drain() size_t n; try { n = read(buf.data(), buf.size()); - s.append((char *) buf.data(), n); + sink(buf.data(), n); } catch (EndOfFile &) { break; } } - return s; +} + + +std::string Source::drain() +{ + StringSink s; + drainInto(s); + return *s.s; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6f4f4c855..2fd06bb4d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -69,6 +69,8 @@ struct Source virtual bool good() { return true; } + void drainInto(Sink & sink); + std::string drain(); }; From dfa547c6a81d6fb7ef3d3f69a98ebe969df42828 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 17:15:05 +0200 Subject: [PATCH 020/123] Add ContentAddressMethod and parse/render it --- src/libstore/content-address.cc | 62 ++++++++++++++++++++++++++------- src/libstore/content-address.hh | 19 ++++++++++ 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 0885c3d0e..bf56d3d77 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -37,48 +37,86 @@ std::string renderContentAddress(ContentAddress ca) { }, ca); } -ContentAddress parseContentAddress(std::string_view rawCa) { - auto rest = rawCa; +std::string renderContentAddressMethod(ContentAddressMethod cam) { + return std::visit(overloaded { + [](TextHashMethod &th) { + return std::string{"text:"} + printHashType(htSHA256); + }, + [](FixedOutputHashMethod &fshm) { + return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); + } + }, cam); +} + +/* + Parses content address strings up to the hash. + */ +static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) { + std::string wholeInput(rest); std::string_view prefix; { auto optPrefix = splitPrefixTo(rest, ':'); if (!optPrefix) - throw UsageError("not a content address because it is not in the form ':': %s", rawCa); + throw UsageError("not a content address because it is not in the form ':': %s", wholeInput); prefix = *optPrefix; } auto parseHashType_ = [&](){ auto hashTypeRaw = splitPrefixTo(rest, ':'); if (!hashTypeRaw) - throw UsageError("content address hash must be in form ':', but found: %s", rawCa); + throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); HashType hashType = parseHashType(*hashTypeRaw); return std::move(hashType); }; // Switch on prefix if (prefix == "text") { - // No parsing of the method, "text" only support flat. + // No parsing of the ingestion method, "text" only support flat. HashType hashType = parseHashType_(); if (hashType != htSHA256) throw Error("text content address hash should use %s, but instead uses %s", printHashType(htSHA256), printHashType(hashType)); - return TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), - }; + return TextHashMethod {}; } else if (prefix == "fixed") { // Parse method auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; HashType hashType = parseHashType_(); - return FixedOutputHash { - .method = method, - .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + return FixedOutputHashMethod { + .fileIngestionMethod = method, + .hashType = std::move(hashType), }; } else throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); -}; +} + +ContentAddress parseContentAddress(std::string_view rawCa) { + auto rest = rawCa; + + ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); + + return std::visit( + overloaded { + [&](TextHashMethod thm) { + return ContentAddress(TextHash { + .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) + }); + }, + [&](FixedOutputHashMethod fohMethod) { + return ContentAddress(FixedOutputHash { + .method = fohMethod.fileIngestionMethod, + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), + }); + }, + }, caMethod); +} + +ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) { + std::string_view asPrefix {std::string{caMethod} + ":"}; + return parseContentAddressMethodPrefix(asPrefix); +} std::optional parseContentAddressOpt(std::string_view rawCaOpt) { return rawCaOpt == "" ? std::optional {} : parseContentAddress(rawCaOpt); diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 22a039242..f6a6f5140 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -55,4 +55,23 @@ std::optional parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); +/* + We only have one way to hash text with references, so this is single-value + type is only useful in std::variant. +*/ +struct TextHashMethod { }; +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; + +typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > ContentAddressMethod; + +ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + +std::string renderContentAddressMethod(ContentAddressMethod caMethod); + } From 14b30b3f3d5af75c210a15cb128e67c0eff66149 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 17:36:16 +0200 Subject: [PATCH 021/123] Move FramedSource and FramedSink, extract withFramedSink --- src/libstore/daemon.cc | 47 ----------- src/libstore/remote-store.cc | 151 +++++++++++++++++++---------------- src/libutil/serialise.hh | 47 +++++++++++ 3 files changed, 127 insertions(+), 118 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2e068992b..010d3bc2d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -748,59 +748,12 @@ static void performOp(TunnelLogger * logger, ref store, info.ultimate = false; if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { - - struct FramedSource : Source - { - Source & from; - bool eof = false; - std::vector pending; - size_t pos = 0; - - FramedSource(Source & from) : from(from) - { } - - ~FramedSource() - { - if (!eof) { - while (true) { - auto n = readInt(from); - if (!n) break; - std::vector data(n); - from(data.data(), n); - } - } - } - - size_t read(unsigned char * data, size_t len) override - { - if (eof) throw EndOfFile("reached end of FramedSource"); - - if (pos >= pending.size()) { - size_t len = readInt(from); - if (!len) { - eof = true; - return 0; - } - pending = std::vector(len); - pos = 0; - from(pending.data(), len); - } - - auto n = std::min(len, pending.size() - pos); - memcpy(data, pending.data() + pos, n); - pos += n; - return n; - } - }; - logger->startWork(); - { FramedSource source(from); store->addToStore(info, source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs); } - logger->stopWork(); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e92b94975..993b05dc0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -306,6 +306,8 @@ struct ConnectionHandle std::rethrow_exception(ex); } } + + void withFramedSink(std::function fun); }; @@ -564,78 +566,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << repair << !checkSigs; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { - - conn->to.flush(); - - std::exception_ptr ex; - - struct FramedSink : BufferedSink - { - ConnectionHandle & conn; - std::exception_ptr & ex; - - FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) - { } - - ~FramedSink() - { - try { - conn->to << 0; - conn->to.flush(); - } catch (...) { - ignoreException(); - } - } - - void write(const unsigned char * data, size_t len) override - { - /* Don't send more data if the remote has - encountered an error. */ - if (ex) { - auto ex2 = ex; - ex = nullptr; - std::rethrow_exception(ex2); - } - conn->to << len; - conn->to(data, len); - }; - }; - - /* Handle log messages / exceptions from the remote on a - separate thread. */ - std::thread stderrThread([&]() - { - try { - conn.processStderr(nullptr, nullptr, false); - } catch (...) { - ex = std::current_exception(); - } - }); - - Finally joinStderrThread([&]() - { - if (stderrThread.joinable()) { - stderrThread.join(); - if (ex) { - try { - std::rethrow_exception(ex); - } catch (...) { - ignoreException(); - } - } - } - }); - - { - FramedSink sink(conn, ex); + conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); - sink.flush(); - } - - stderrThread.join(); - if (ex) - std::rethrow_exception(ex); - + }); } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { conn.processStderr(0, &source); } else { @@ -992,6 +925,82 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } + +struct FramedSink : nix::BufferedSink +{ + ConnectionHandle & conn; + std::exception_ptr & ex; + + FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) + { } + + ~FramedSink() + { + try { + conn->to << 0; + conn->to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + conn->to << len; + conn->to(data, len); + }; +}; + +void +ConnectionHandle::withFramedSink(std::function fun) { + (*this)->to.flush(); + + std::exception_ptr ex; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + processStderr(nullptr, nullptr, false); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink(*this, ex); + fun(sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + +} + static RegisterStoreImplementation regStore; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 2fd06bb4d..6027d5961 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -406,4 +406,51 @@ struct StreamToSourceAdapter : Source }; +/* Like SizedSource, but guarantees that the whole frame is consumed after + destruction. This ensures that the original stream is in a known state. */ +struct FramedSource : Source +{ + Source & from; + bool eof = false; + std::vector pending; + size_t pos = 0; + + FramedSource(Source & from) : from(from) + { } + + ~FramedSource() + { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } + } + } + + size_t read(unsigned char * data, size_t len) override + { + if (eof) throw EndOfFile("reached end of FramedSource"); + + if (pos >= pending.size()) { + size_t len = readInt(from); + if (!len) { + eof = true; + return 0; + } + pending = std::vector(len); + pos = 0; + from(pending.data(), len); + } + + auto n = std::min(len, pending.size() - pos); + memcpy(data, pending.data() + pos, n); + pos += n; + return n; + } +}; + + } From c00e07834327a8ef626cf4f1ecb216ee1b6a0877 Mon Sep 17 00:00:00 2001 From: Marwan Aljubeh Date: Fri, 18 Sep 2020 17:10:39 +0100 Subject: [PATCH 022/123] Add a nix.conf option for allowing a symlinked store --- src/libstore/globals.cc | 1 + src/libstore/globals.hh | 13 +++++++++++++ src/libstore/local-store.cc | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 491c664db..ca2e75603 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -41,6 +41,7 @@ Settings::Settings() { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; + ignoreSymlinkStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); if (caFile == "") { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 02721285a..129cef6b4 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -881,6 +881,19 @@ public: Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", "Path or URI of the global flake registry."}; + + Setting ignoreSymlinkStore{ + this, false, "ignore-symlink-store", + R"( + If set to `true`, Nix will stop complaining if the store directory + (typically /nix/store) contains symlink components. + + This risks making some builds "impure" because builders sometimes + "canonicalise" paths by resolving all symlink components. Problems + occur if those builds are then deployed to machines where /nix/store + resolves to a different location from that of the build machine. You + can enable this setting if you are sure you're not going to do that. + )"}; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c618203f0..24b9ea7bd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -109,7 +109,7 @@ LocalStore::LocalStore(const Params & params) } /* Ensure that the store and its parents are not symlinks. */ - if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { + if (!settings.ignoreSymlinkStore) { Path path = realStoreDir; struct stat st; while (path != "/") { From e40772cd35adcd158d30727f7f294b823df8010a Mon Sep 17 00:00:00 2001 From: Marwan Aljubeh Date: Fri, 18 Sep 2020 17:18:45 +0100 Subject: [PATCH 023/123] Lint issue: replacing tabs with spaces --- src/libstore/globals.hh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 129cef6b4..ddc13898d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -885,11 +885,11 @@ public: Setting ignoreSymlinkStore{ this, false, "ignore-symlink-store", R"( - If set to `true`, Nix will stop complaining if the store directory - (typically /nix/store) contains symlink components. + If set to `true`, Nix will stop complaining if the store directory + (typically /nix/store) contains symlink components. - This risks making some builds "impure" because builders sometimes - "canonicalise" paths by resolving all symlink components. Problems + This risks making some builds "impure" because builders sometimes + "canonicalise" paths by resolving all symlink components. Problems occur if those builds are then deployed to machines where /nix/store resolves to a different location from that of the build machine. You can enable this setting if you are sure you're not going to do that. From e34fe47d0ce19fc7657970fb0e610bffbc3e43f0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 19:27:11 +0200 Subject: [PATCH 024/123] Overhaul wopAddToStore --- src/libstore/daemon.cc | 105 ++++++++++++++++++++------------ src/libstore/remote-store.cc | 103 +++++++++++++++++++------------ src/libstore/remote-store.hh | 7 +-- src/libstore/worker-protocol.hh | 2 +- 4 files changed, 133 insertions(+), 84 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 010d3bc2d..0a08a2e12 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -349,47 +349,74 @@ static void performOp(TunnelLogger * logger, ref store, } case wopAddToStore: { - HashType hashAlgo; - std::string baseName; - FileIngestionMethod method; - { - bool fixed; - uint8_t recursive; - std::string hashAlgoRaw; - from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; - if (recursive > (uint8_t) FileIngestionMethod::Recursive) - throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); - method = FileIngestionMethod { recursive }; - /* Compatibility hack. */ - if (!fixed) { - hashAlgoRaw = "sha256"; - method = FileIngestionMethod::Recursive; + if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { + auto name = readString(from); + auto camStr = readString(from); + auto refs = readStorePaths(*store, from); + + logger->startWork(); + StorePath path = [&]() -> StorePath { + // NB: FramedSource must be out of scope before logger->stopWork(); + ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + FramedSource source(from); + return std::visit(overloaded { + [&](TextHashMethod &_) -> StorePath { + // We could stream this by changing Store + std::string contents = source.drain(); + return store->addTextToStore(name, contents, refs); + }, + [&](FixedOutputHashMethod &fohm) -> StorePath { + return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType); + }, + }, contentAddressMethod); + }(); + logger->stopWork(); + + to << store->printStorePath(path); + + } else { + HashType hashAlgo; + std::string baseName; + FileIngestionMethod method; + { + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; + if (recursive > (uint8_t) FileIngestionMethod::Recursive) + throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); + method = FileIngestionMethod { recursive }; + /* Compatibility hack. */ + if (!fixed) { + hashAlgoRaw = "sha256"; + method = FileIngestionMethod::Recursive; + } + hashAlgo = parseHashType(hashAlgoRaw); } - hashAlgo = parseHashType(hashAlgoRaw); + + StringSink saved; + TeeSource savedNARSource(from, saved); + RetrieveRegularNARSink savedRegular { saved }; + + if (method == FileIngestionMethod::Recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNARSource); + } else + parseDump(savedRegular, from); + + logger->startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); + + // FIXME: try to stream directly from `from`. + StringSource dumpSource { *saved.s }; + auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); + logger->stopWork(); + + to << store->printStorePath(path); } - - StringSink saved; - TeeSource savedNARSource(from, saved); - RetrieveRegularNARSink savedRegular { saved }; - - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNARSource); - } else - parseDump(savedRegular, from); - - logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - // FIXME: try to stream directly from `from`. - StringSource dumpSource { *saved.s }; - auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); - logger->stopWork(); - - to << store->printStorePath(path); break; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 993b05dc0..14ad4767d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -526,6 +526,69 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } +StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashType, RepairFlag repair) +{ + if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon"); + StorePathSet references; + + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + + conn->to + << wopAddToStore + << name + << renderContentAddressMethod(FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }); + writeStorePaths(*this, conn->to, references); + + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + + return parseStorePath(readString(conn->from)); + + } + else { + if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); + + conn->to + << wopAddToStore + << name + << ((hashType == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (method == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashType); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + if (method == FileIngestionMethod::Recursive) { + dump.drainInto(conn->to); + } else { + std::string contents = dump.drain(); + dumpString(contents, conn->to); + } + } + conn->to.warn = false; + conn.processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + + return parseStorePath(readString(conn->from)); + } +} + + void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { @@ -579,46 +642,6 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - - Path srcPath(absPath(_srcPath)); - - conn->to - << wopAddToStore - << name - << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (method == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashAlgo); - - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - dumpPath(srcPath, conn->to, filter); - } - conn->to.warn = false; - conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) - try { - conn.processStderr(); - } catch (EndOfFile & e) { } - throw; - } - - return parseStorePath(readString(conn->from)); -} - - StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 91c748006..14c22e764 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -63,13 +63,12 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; - StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 13cf8d4ab..21e05d91c 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x118 +#define PROTOCOL_VERSION 0x119 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From c602ebfb34de3626fa0b9110face6ea4b171ac0f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 20:19:15 +0200 Subject: [PATCH 025/123] Refactor wopAddToStore to make wopAddTextToStore obsolete --- src/libstore/remote-store.cc | 91 ++++++++++++++++++--------------- src/libstore/remote-store.hh | 2 + src/libstore/worker-protocol.hh | 2 +- 3 files changed, 53 insertions(+), 42 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 14ad4767d..95e6a3291 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -526,12 +526,8 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } -StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair) +StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references) { - if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon"); - StorePathSet references; - auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { @@ -539,7 +535,7 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, conn->to << wopAddToStore << name - << renderContentAddressMethod(FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }); + << renderContentAddressMethod(caMethod); writeStorePaths(*this, conn->to, references); conn.withFramedSink([&](Sink & sink) { @@ -552,42 +548,60 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); - conn->to - << wopAddToStore - << name - << ((hashType == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (method == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashType); + std::visit(overloaded { + [&](TextHashMethod thm) -> void { + std::string s = dump.drain(); + conn->to << wopAddTextToStore << name << s; + writeStorePaths(*this, conn->to, references); + conn.processStderr(); + }, + [&](FixedOutputHashMethod fohm) -> void { + conn->to + << wopAddToStore + << name + << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(fohm.hashType); - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - if (method == FileIngestionMethod::Recursive) { - dump.drainInto(conn->to); - } else { - std::string contents = dump.drain(); - dumpString(contents, conn->to); - } - } - conn->to.warn = false; - conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + dump.drainInto(conn->to); + } else { + std::string contents = dump.drain(); + dumpString(contents, conn->to); + } + } + conn->to.warn = false; conn.processStderr(); - } catch (EndOfFile & e) { } - throw; - } + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + } + }, caMethod); return parseStorePath(readString(conn->from)); } } +StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashType, RepairFlag repair) +{ + if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon"); + StorePathSet references; + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references); +} + void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) @@ -646,13 +660,8 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - conn->to << wopAddTextToStore << name << s; - writeStorePaths(*this, conn->to, references); - - conn.processStderr(); - return parseStorePath(readString(conn->from)); + StringSource source(s); + return addCAToStore(source, name, TextHashMethod{}, references); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 14c22e764..2c775cb79 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -63,6 +63,8 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; + StorePath addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references); + StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 21e05d91c..b100d1550 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -18,7 +18,7 @@ typedef enum { wopQueryReferences = 5, // obsolete wopQueryReferrers = 6, wopAddToStore = 7, - wopAddTextToStore = 8, + wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore wopBuildPaths = 9, wopEnsurePath = 10, wopAddTempRoot = 11, From ecc8088cb7e91e4c45787f290c8ff61547fb838a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 21:32:01 +0200 Subject: [PATCH 026/123] wopAddToStore: Throw to clarify unused refs Co-authored-by: John Ericson --- src/libstore/daemon.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0a08a2e12..f2d789699 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -366,6 +366,8 @@ static void performOp(TunnelLogger * logger, ref store, return store->addTextToStore(name, contents, refs); }, [&](FixedOutputHashMethod &fohm) -> StorePath { + if (!refs.empty()) + throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType); }, }, contentAddressMethod); From 8279178b078103f816bf85f5a153a4845d30cc83 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 22:01:35 +0200 Subject: [PATCH 027/123] Move FramedSink next to FramedSource --- src/libstore/remote-store.cc | 35 +-------------------------- src/libutil/serialise.hh | 46 ++++++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 95e6a3291..cfbf23ac4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -957,39 +957,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } - -struct FramedSink : nix::BufferedSink -{ - ConnectionHandle & conn; - std::exception_ptr & ex; - - FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) - { } - - ~FramedSink() - { - try { - conn->to << 0; - conn->to.flush(); - } catch (...) { - ignoreException(); - } - } - - void write(const unsigned char * data, size_t len) override - { - /* Don't send more data if the remote has - encountered an error. */ - if (ex) { - auto ex2 = ex; - ex = nullptr; - std::rethrow_exception(ex2); - } - conn->to << len; - conn->to(data, len); - }; -}; - void ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); @@ -1022,7 +989,7 @@ ConnectionHandle::withFramedSink(std::function fun) { }); { - FramedSink sink(*this, ex); + FramedSink sink((*this)->to, ex); fun(sink); sink.flush(); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6027d5961..3511e4cd8 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -406,8 +406,13 @@ struct StreamToSourceAdapter : Source }; -/* Like SizedSource, but guarantees that the whole frame is consumed after - destruction. This ensures that the original stream is in a known state. */ +/* A source that reads a distinct format of concatenated chunks back into its + logical form, in order to guarantee a known state to the original stream, + even in the event of errors. + + Use with FramedSink, which also allows the logical stream to be terminated + in the event of an exception. +*/ struct FramedSource : Source { Source & from; @@ -452,5 +457,42 @@ struct FramedSource : Source } }; +/* Write as chunks in the format expected by FramedSource. + + The exception_ptr reference can be used to terminate the stream when you + detect that an error has occurred on the remote end. +*/ +struct FramedSink : nix::BufferedSink +{ + BufferedSink & to; + std::exception_ptr & ex; + + FramedSink(BufferedSink & to, std::exception_ptr & ex) : to(to), ex(ex) + { } + + ~FramedSink() + { + try { + to << 0; + to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + to << len; + to(data, len); + }; +}; + } From fbf509c1137fd59ebb3e7a993c8da42c17deb68a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 22:04:05 +0200 Subject: [PATCH 028/123] parseContentAddressMethodPrefix: use string_view Co-authored-by: John Ericson --- src/libstore/content-address.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index bf56d3d77..74215c545 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -52,7 +52,7 @@ std::string renderContentAddressMethod(ContentAddressMethod cam) { Parses content address strings up to the hash. */ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) { - std::string wholeInput(rest); + std::string_view wholeInput { rest }; std::string_view prefix; { From 7c682640857106f18d7020c9c75ea39b1ef8dd2c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 18 Sep 2020 10:06:34 +0200 Subject: [PATCH 029/123] wopAddToStore: add RepairFlag --- src/libstore/daemon.cc | 5 ++++- src/libstore/remote-store.cc | 9 ++++----- src/libstore/remote-store.hh | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f2d789699..9f7350217 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -353,6 +353,9 @@ static void performOp(TunnelLogger * logger, ref store, auto name = readString(from); auto camStr = readString(from); auto refs = readStorePaths(*store, from); + bool repairBool; + from >> repairBool; + auto repair = RepairFlag{repairBool}; logger->startWork(); StorePath path = [&]() -> StorePath { @@ -368,7 +371,7 @@ static void performOp(TunnelLogger * logger, ref store, [&](FixedOutputHashMethod &fohm) -> StorePath { if (!refs.empty()) throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); - return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType); + return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); }, }, contentAddressMethod); }(); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cfbf23ac4..54d70e3e1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -526,7 +526,7 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } -StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references) +StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair) { auto conn(getConnection()); @@ -537,6 +537,7 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA << name << renderContentAddressMethod(caMethod); writeStorePaths(*this, conn->to, references); + conn->to << repair; conn.withFramedSink([&](Sink & sink) { dump.drainInto(sink); @@ -597,9 +598,8 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method, HashType hashType, RepairFlag repair) { - if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon"); StorePathSet references; - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references); + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair); } @@ -659,9 +659,8 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references); + return addCAToStore(source, name, TextHashMethod{}, references, repair); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 2c775cb79..23f7b8425 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -63,7 +63,7 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - StorePath addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references); + StorePath addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair); StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; From fa08db5c4c92ed4674cfacd131d1b45906a9b146 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 18 Sep 2020 11:22:13 +0200 Subject: [PATCH 030/123] wopAddToStore: return ValidPathInfo A ValidPathInfo is created anyway. By returning it we can save a roundtrip and we have a nicer interface. --- src/libstore/daemon.cc | 38 ++++++++++++++++++------------- src/libstore/remote-store.cc | 43 +++++++++++++++++++++--------------- src/libstore/remote-store.hh | 4 +++- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 9f7350217..8897a73f4 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -239,6 +239,18 @@ struct ClientSettings } }; +static void writeValidPathInfo(ref store, unsigned int clientVersion, Sink & to, std::shared_ptr info) { + to << (info->deriver ? store->printStorePath(*info->deriver) : "") + << info->narHash.to_string(Base16, false); + writeStorePaths(*store, to, info->references); + to << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs + << renderContentAddress(info->ca); + } +} + static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) @@ -358,26 +370,30 @@ static void performOp(TunnelLogger * logger, ref store, auto repair = RepairFlag{repairBool}; logger->startWork(); - StorePath path = [&]() -> StorePath { + auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); FramedSource source(from); + // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](TextHashMethod &_) -> StorePath { + [&](TextHashMethod &_) { // We could stream this by changing Store std::string contents = source.drain(); - return store->addTextToStore(name, contents, refs); + auto path = store->addTextToStore(name, contents, refs, repair); + return store->queryPathInfo(path); }, - [&](FixedOutputHashMethod &fohm) -> StorePath { + [&](FixedOutputHashMethod &fohm) { if (!refs.empty()) throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); - return store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + return store->queryPathInfo(path); }, }, contentAddressMethod); }(); logger->stopWork(); - to << store->printStorePath(path); + to << store->printStorePath(pathInfo->path); + writeValidPathInfo(store, clientVersion, to, pathInfo); } else { HashType hashAlgo; @@ -706,15 +722,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); - writeStorePaths(*store, to, info->references); - to << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << renderContentAddress(info->ca); - } + writeValidPathInfo(store, clientVersion, to, info); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 54d70e3e1..3936ddb1d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -421,11 +421,27 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } +ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) { + auto deriver = readString(conn->from); + auto narHash = Hash::parseAny(readString(conn->from), htSHA256); + auto info = make_ref(path, narHash); + if (deriver != "") info->deriver = parseStorePath(deriver); + info->references = readStorePaths(*this, conn->from); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings(conn->from); + info->ca = parseContentAddressOpt(readString(conn->from)); + } + return info; +} + + void RemoteStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - std::shared_ptr info; + std::shared_ptr info; { auto conn(getConnection()); conn->to << wopQueryPathInfo << printStorePath(path); @@ -441,17 +457,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - auto deriver = readString(conn->from); - auto narHash = Hash::parseAny(readString(conn->from), htSHA256); - info = std::make_shared(path, narHash); - if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = readStorePaths(*this, conn->from); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); - } + info = readValidPathInfo(conn, path); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -526,7 +532,7 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } -StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair) +ref RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair) { auto conn(getConnection()); @@ -543,8 +549,8 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA dump.drainInto(sink); }); - return parseStorePath(readString(conn->from)); - + auto path = parseStorePath(readString(conn->from)); + return readValidPathInfo(conn, path); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); @@ -591,7 +597,8 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA } }, caMethod); - return parseStorePath(readString(conn->from)); + auto path = parseStorePath(readString(conn->from)); + return queryPathInfo(path); } } @@ -599,7 +606,7 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method, HashType hashType, RepairFlag repair) { StorePathSet references; - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair); + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; } @@ -660,7 +667,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references, repair); + return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 23f7b8425..47f43ae8a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -63,7 +63,7 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - StorePath addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair); + ref addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair); StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; @@ -140,6 +140,8 @@ protected: virtual void narFromPath(const StorePath & path, Sink & sink) override; + ref readValidPathInfo(ConnectionHandle & conn, const StorePath & path); + private: std::atomic_bool failed{false}; From ca30abb3fb36440e5a13161c39647189036fc18f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 18 Sep 2020 11:58:45 +0200 Subject: [PATCH 031/123] Document addCAToStore/addToStoreFromDump source drainage Also checked that all usages satisfy the requirement and removed dead code. --- src/libstore/build.cc | 8 -------- src/libstore/remote-store.hh | 2 ++ src/libstore/store-api.hh | 3 ++- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6e55f83d5..061f07546 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2949,14 +2949,6 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf goal.addDependency(info.path); } - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override - { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); - goal.addDependency(path); - return path; - } - StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair = NoRepair) override { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 47f43ae8a..735a3c24e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -63,8 +63,10 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; + /* Add a content-addressable store path. `dump` will be drained. */ ref addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair); + /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4d3f07dfc..591140874 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -449,7 +449,8 @@ public: /* Like addToStore(), but the contents of the path are contained in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == - false). */ + false). + `dump` may be drained */ // FIXME: remove? virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) From 4e1a04733d5075fdc09dbc6767755d4487e96da7 Mon Sep 17 00:00:00 2001 From: Marwan Aljubeh Date: Mon, 21 Sep 2020 16:32:22 +0100 Subject: [PATCH 032/123] Use a better name for the config option --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ddc13898d..fcb9b0f63 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -883,7 +883,7 @@ public: "Path or URI of the global flake registry."}; Setting ignoreSymlinkStore{ - this, false, "ignore-symlink-store", + this, false, "allow-symlinked-store", R"( If set to `true`, Nix will stop complaining if the store directory (typically /nix/store) contains symlink components. From f80ffeb8c9291f7168f098fdaadc15408492f3c2 Mon Sep 17 00:00:00 2001 From: Marwan Aljubeh Date: Mon, 21 Sep 2020 17:29:08 +0100 Subject: [PATCH 033/123] Update the variable name accordingly --- src/libstore/globals.cc | 2 +- src/libstore/globals.hh | 2 +- src/libstore/local-store.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ca2e75603..ff6082aac 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -41,7 +41,7 @@ Settings::Settings() { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; - ignoreSymlinkStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; + allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); if (caFile == "") { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index fcb9b0f63..fd0c6cbcc 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -882,7 +882,7 @@ public: Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", "Path or URI of the global flake registry."}; - Setting ignoreSymlinkStore{ + Setting allowSymlinkedStore{ this, false, "allow-symlinked-store", R"( If set to `true`, Nix will stop complaining if the store directory diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 24b9ea7bd..cc9fcfe6e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -109,7 +109,7 @@ LocalStore::LocalStore(const Params & params) } /* Ensure that the store and its parents are not symlinks. */ - if (!settings.ignoreSymlinkStore) { + if (!settings.allowSymlinkedStore) { Path path = realStoreDir; struct stat st; while (path != "/") { From ba37299a0364c4b58d59e20cf7821ed9fade6dd6 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Sat, 19 Sep 2020 22:26:07 -0700 Subject: [PATCH 034/123] Serialize SandboxMode enum to string for JSON Rather than showing an integer as the default, instead show the boolean referenced in the description. The nix.conf.5 manpage used to show "default: 0", which is unnecessarily opaque and confusing (doesn't 0 mean false, even though the default is true?); now it properly shows that the default is true. --- src/libstore/globals.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 0beb9b2b7..f94d97cc8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -147,6 +147,12 @@ bool Settings::isWSL1() const string nixVersion = PACKAGE_VERSION; +NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { + {SandboxMode::smEnabled, true}, + {SandboxMode::smRelaxed, "relaxed"}, + {SandboxMode::smDisabled, false}, +}); + template<> void BaseSetting::set(const std::string & str) { if (str == "true") value = smEnabled; From d860295e116f9aa0f37e2637b5ec4b9c86fdf4d3 Mon Sep 17 00:00:00 2001 From: Michael Reilly Date: Sat, 19 Sep 2020 10:46:02 -0400 Subject: [PATCH 035/123] Bump nlohmann-json version to 3.9.1 --- src/libexpr/json-to-value.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 76e1a26bf..9ca5ac86d 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -115,6 +115,14 @@ public: { return handle_value(mkString, val.c_str()); } +#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 + bool binary(binary_t&) + { + // This function ought to be unreachable + assert(false); + return true; + } +#endif bool start_object(std::size_t len) { From 97b5154750d911a05992b97d7404d813d924de73 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 22 Sep 2020 10:04:25 +0200 Subject: [PATCH 036/123] Disable `FORTIFY_SOURCE` when compiling without optims Otherwise the build is cluttered with ``` /nix/store/fwpn2f7a4iqszyydw7ag61zlnp6xk5d3-glibc-2.30-dev/include/features.h:382:4: warning: #warning _FORTIFY_SOURCE requires compiling with optimization (-O) [-Wcpp] 382 | # warning _FORTIFY_SOURCE requires compiling with optimization (-O) | ^~~~~~~ ``` when building with `OPTIMIZE=0` --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f472ca7e5..c50d2c40f 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ OPTIMIZE = 1 ifeq ($(OPTIMIZE), 1) GLOBAL_CXXFLAGS += -O3 else - GLOBAL_CXXFLAGS += -O0 + GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE endif include mk/lib.mk From c1e79f870cf548e81d7cab0d296c6e923da9fa8c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 22 Sep 2020 10:37:43 +0200 Subject: [PATCH 037/123] Silence a compiler warning in serialise.hh Explicitely cast to `uint64_t` in `readNum` to avoid a "comparison between signed and unsigned" warning --- src/libutil/serialise.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6f4f4c855..7682a0f19 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -340,7 +340,7 @@ T readNum(Source & source) ((uint64_t) buf[6] << 48) | ((uint64_t) buf[7] << 56); - if (n > std::numeric_limits::max()) + if (n > (uint64_t)std::numeric_limits::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); return (T) n; From 35a0ac183858ecb03e313e088562c84fe211e20d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Sep 2020 11:40:19 +0200 Subject: [PATCH 038/123] Style fixes --- src/libstore/content-address.cc | 26 +++++++++++++++++--------- src/libstore/daemon.cc | 7 ++++++- src/libstore/remote-store.cc | 15 +++++++++++---- src/libstore/remote-store.hh | 7 ++++++- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 74215c545..90a3ad1f5 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,11 +4,13 @@ namespace nix { -std::string FixedOutputHash::printMethodAlgo() const { +std::string FixedOutputHash::printMethodAlgo() const +{ return makeFileIngestionPrefix(method) + printHashType(hash.type); } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) { +std::string makeFileIngestionPrefix(const FileIngestionMethod m) +{ switch (m) { case FileIngestionMethod::Flat: return ""; @@ -26,7 +28,8 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) + hash.to_string(Base32, true); } -std::string renderContentAddress(ContentAddress ca) { +std::string renderContentAddress(ContentAddress ca) +{ return std::visit(overloaded { [](TextHash th) { return "text:" + th.hash.to_string(Base32, true); @@ -37,7 +40,8 @@ std::string renderContentAddress(ContentAddress ca) { }, ca); } -std::string renderContentAddressMethod(ContentAddressMethod cam) { +std::string renderContentAddressMethod(ContentAddressMethod cam) +{ return std::visit(overloaded { [](TextHashMethod &th) { return std::string{"text:"} + printHashType(htSHA256); @@ -51,7 +55,8 @@ std::string renderContentAddressMethod(ContentAddressMethod cam) { /* Parses content address strings up to the hash. */ -static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) { +static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) +{ std::string_view wholeInput { rest }; std::string_view prefix; @@ -113,16 +118,19 @@ ContentAddress parseContentAddress(std::string_view rawCa) { }, caMethod); } -ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) { +ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +{ std::string_view asPrefix {std::string{caMethod} + ":"}; return parseContentAddressMethodPrefix(asPrefix); } -std::optional parseContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseContentAddress(rawCaOpt); +std::optional parseContentAddressOpt(std::string_view rawCaOpt) +{ + return rawCaOpt == "" ? std::optional() : parseContentAddress(rawCaOpt); }; -std::string renderContentAddress(std::optional ca) { +std::string renderContentAddress(std::optional ca) +{ return ca ? renderContentAddress(*ca) : ""; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8897a73f4..83f8968b0 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -239,7 +239,12 @@ struct ClientSettings } }; -static void writeValidPathInfo(ref store, unsigned int clientVersion, Sink & to, std::shared_ptr info) { +static void writeValidPathInfo( + ref store, + unsigned int clientVersion, + Sink & to, + std::shared_ptr info) +{ to << (info->deriver ? store->printStorePath(*info->deriver) : "") << info->narHash.to_string(Base16, false); writeStorePaths(*store, to, info->references); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 02a8b72b9..6f1f9769b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -422,7 +422,8 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } -ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) { +ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) +{ auto deriver = readString(conn->from); auto narHash = Hash::parseAny(readString(conn->from), htSHA256); auto info = make_ref(path, narHash); @@ -533,7 +534,12 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } -ref RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair) +ref RemoteStore::addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair) { auto conn(getConnection()); @@ -603,6 +609,7 @@ ref RemoteStore::addCAToStore(Source & dump, const string & } } + StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method, HashType hashType, RepairFlag repair) { @@ -964,8 +971,8 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -void -ConnectionHandle::withFramedSink(std::function fun) { +void ConnectionHandle::withFramedSink(std::function fun) +{ (*this)->to.flush(); std::exception_ptr ex; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 735a3c24e..ec04be985 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -64,7 +64,12 @@ public: SubstitutablePathInfos & infos) override; /* Add a content-addressable store path. `dump` will be drained. */ - ref addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair); + ref addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair); /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, const string & name, From 980edd1f3a31eefe297d073f6a7cff099f21eb4a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Sep 2020 15:28:20 +0200 Subject: [PATCH 039/123] RemoteStore::addCAToStore(): Don't hold connection while calling queryPathInfo() This leads to a deadlock if we're at the connection limit. --- src/libstore/remote-store.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6f1f9769b..be5eb4736 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -541,7 +541,8 @@ ref RemoteStore::addCAToStore( const StorePathSet & references, RepairFlag repair) { - auto conn(getConnection()); + std::optional conn_(getConnection()); + auto & conn = *conn_; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { @@ -605,6 +606,8 @@ ref RemoteStore::addCAToStore( } }, caMethod); auto path = parseStorePath(readString(conn->from)); + // Release our connection to prevent a deadlock in queryPathInfo(). + conn_.reset(); return queryPathInfo(path); } } From 993229cdaf2e2347a204c876ecd660fc94048101 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 22 Aug 2020 20:44:47 +0000 Subject: [PATCH 040/123] Deduplicate basic derivation goals too See comments for security concerns. Also optimize goal creation by not traversing map twice. --- src/libstore/build.cc | 90 +++++++++++++++++++++++++++------------ src/libstore/daemon.cc | 14 ++++++ src/libstore/store-api.hh | 36 ++++++++++++++-- 3 files changed, 110 insertions(+), 30 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 07c5bceb2..8b206e819 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -296,9 +296,21 @@ public: ~Worker(); /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, BuildMode buildMode = bmNormal); + + /* derivation goal */ +private: + std::shared_ptr makeDerivationGoalCommon( + const StorePath & drvPath, const StringSet & wantedOutputs, + std::function()> mkDrvGoal); +public: + std::shared_ptr makeDerivationGoal( + const StorePath & drvPath, + const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + std::shared_ptr makeBasicDerivationGoal( + const StorePath & drvPath, const BasicDerivation & drv, + const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + + /* substitution goal */ GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ @@ -949,10 +961,12 @@ private: friend struct RestrictedStore; public: - DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, - Worker & worker, BuildMode buildMode = bmNormal); + DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - Worker & worker, BuildMode buildMode = bmNormal); + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); ~DerivationGoal(); /* Whether we need to perform hash rewriting if there are valid output paths. */ @@ -1085,8 +1099,8 @@ private: const Path DerivationGoal::homeDir = "/homeless-shelter"; -DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, - Worker & worker, BuildMode buildMode) +DerivationGoal::DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) , drvPath(drvPath) @@ -1094,7 +1108,9 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want , buildMode(buildMode) { state = &DerivationGoal::getDerivation; - name = fmt("building of '%s'", worker.store.printStorePath(this->drvPath)); + name = fmt( + "building of '%s' from .drv file", + StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -1103,15 +1119,18 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - Worker & worker, BuildMode buildMode) + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) , drvPath(drvPath) + , wantedOutputs(wantedOutputs) , buildMode(buildMode) { this->drv = std::make_unique(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = fmt("building of %s", StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); + name = fmt( + "building of '%s' from in-memory derivation", + StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -5060,35 +5079,52 @@ Worker::~Worker() } -GoalPtr Worker::makeDerivationGoal(const StorePath & path, - const StringSet & wantedOutputs, BuildMode buildMode) +std::shared_ptr Worker::makeDerivationGoalCommon( + const StorePath & drvPath, + const StringSet & wantedOutputs, + std::function()> mkDrvGoal) { - GoalPtr goal = derivationGoals[path].lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, wantedOutputs, *this, buildMode); - derivationGoals.insert_or_assign(path, goal); + WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; + GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME + std::shared_ptr goal; + if (!abstract_goal) { + goal = mkDrvGoal(); + abstract_goal_weak = goal; wakeUp(goal); - } else - (dynamic_cast(goal.get()))->addWantedOutputs(wantedOutputs); + } else { + goal = std::dynamic_pointer_cast(abstract_goal); + assert(goal); + goal->addWantedOutputs(wantedOutputs); + } return goal; } -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, BuildMode buildMode) +std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, BuildMode buildMode) { - auto goal = std::make_shared(drvPath, drv, *this, buildMode); - wakeUp(goal); - return goal; + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { + return std::make_shared(drvPath, wantedOutputs, *this, buildMode); + }); +} + + +std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, + const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) +{ + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { + return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); + }); } GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { - GoalPtr goal = substitutionGoals[path].lock(); // FIXME + WeakGoalPtr & goal_weak = substitutionGoals[path]; + GoalPtr goal = goal_weak.lock(); // FIXME if (!goal) { goal = std::make_shared(path, *this, repair, ca); - substitutionGoals.insert_or_assign(path, goal); + goal_weak = goal; wakeUp(goal); } return goal; @@ -5519,7 +5555,7 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); BuildResult result; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 83f8968b0..ec3391a6d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -546,6 +546,20 @@ static void performOp(TunnelLogger * logger, ref store, are in fact content-addressed if we don't trust them. */ assert(derivationIsCA(drv.type()) || trusted); + /* Recompute the derivation path when we cannot trust the original. */ + if (!trusted) { + /* Recomputing the derivation path for input-address derivations + makes it harder to audit them after the fact, since we need the + original not-necessarily-resolved derivation to verify the drv + derivation as adequate claim to the input-addressed output + paths. */ + assert(derivationIsCA(drv.type())); + + Derivation drv2; + static_cast(drv2) = drv; + drvPath = writeDerivation(*store, Derivation { drv2 }); + } + auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); to << res.status << res.errorMsg; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 591140874..3ccee4f75 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -479,8 +479,38 @@ public: BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an - on-disk .drv file). Note that ‘drvPath’ is only used for - informational purposes. */ + on-disk .drv file). + + ‘drvPath’ is used to deduplicate worker goals so it is imperative that + is correct. That said, it doesn't literally need to be store path that + would be calculated from writing this derivation to the store: it is OK + if it instead is that of a Derivation which would resolve to this (by + taking the outputs of it's input derivations and adding them as input + sources) such that the build time referenceable-paths are the same. + + In the input-addressed case, we usually *do* use an "original" + unresolved derivations's path, as that is what will be used in the + `buildPaths` case. Also, the input-addressed output paths are verified + only by that contents of that specific unresolved derivation, so it is + nice to keep that information around so if the original derivation is + ever obtained later, it can be verified whether the trusted user in fact + used the proper output path. + + In the content-addressed case, we want to always use the + resolved drv path calculated from the provided derivation. This serves + two purposes: + + - It keeps the operation trustless, by ruling out a maliciously + invalid drv path corresponding to a non-resolution-equivalent + derivation. + + - For the floating case in particular, it ensures that the derivation + to output mapping respects the resolution equivalence relation, so + one cannot choose different resolution-equivalent derivations to + subvert dependency coherence (i.e. the property that one doesn't end + up with multiple different versions of dependencies without + explicitly choosing to allow it). + */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) = 0; @@ -517,7 +547,7 @@ public: - The collector isn't running, or it's just started but hasn't acquired the GC lock yet. In that case we get and release the lock right away, then exit. The collector scans the - permanent root and sees our's. + permanent root and sees ours. In either case the permanent root is seen by the collector. */ virtual void syncWithGC() { }; From a2f5c921d41c941315ce783f66469bfb20e2c1b3 Mon Sep 17 00:00:00 2001 From: ujjwal Date: Tue, 22 Sep 2020 23:37:06 +0530 Subject: [PATCH 041/123] fixed typo --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 01f464859..783b98ef0 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -367,7 +367,7 @@ LockedFlake lockFlake( /* If we have an --update-input flag for an input of this input, then we must fetch the flake to - to update it. */ + update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto hasChildUpdate = From 9fbc31a65bab50cd60a882517b3c8030485ce096 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 15 Aug 2020 16:41:28 +0000 Subject: [PATCH 042/123] Get rid of Hash::dummy from BinaryCacheStore --- src/libstore/binary-cache-store.cc | 102 +++++++++++++++++------------ src/libstore/binary-cache-store.hh | 7 ++ src/libstore/store-api.hh | 4 +- 3 files changed, 69 insertions(+), 44 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ebc0bd6a4..6679eb16f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -142,17 +142,10 @@ struct FileSource : FdSource } }; -void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs) +StorePath BinaryCacheStore::addToStoreCommon( + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, + std::function mkInfo) { - assert(info.narSize); - - if (!repair && isValidPath(info.path)) { - // FIXME: copyNAR -> null sink - narSource.drain(); - return; - } - auto [fdTemp, fnTemp] = createTempFile(); AutoDelete autoDelete(fnTemp); @@ -162,13 +155,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource /* Read the NAR simultaneously into a CompressionSink+FileSink (to write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ - HashSink fileHashSink(htSHA256); + HashSink fileHashSink { htSHA256 }; std::shared_ptr narAccessor; + HashSink narHashSink { htSHA256 }; { FdSink fileSink(fdTemp.get()); - TeeSink teeSink(fileSink, fileHashSink); - auto compressionSink = makeCompressionSink(compression, teeSink); - TeeSource teeSource(narSource, *compressionSink); + TeeSink teeSinkCompressed { fileSink, fileHashSink }; + auto compressionSink = makeCompressionSink(compression, teeSinkCompressed); + TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; + TeeSource teeSource { narSource, teeSinkUncompressed }; narAccessor = makeNarAccessor(teeSource); compressionSink->finish(); fileSink.flush(); @@ -176,9 +171,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource auto now2 = std::chrono::steady_clock::now(); + auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->narSize = info.narSize; - narInfo->narHash = info.narHash; + narInfo->narSize = info.narSize; // FIXME needed? + narInfo->narHash = info.narHash; // FIXME needed? narInfo->compression = compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; @@ -300,6 +296,41 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource writeNarInfo(narInfo); stats.narInfoWrite++; + + return narInfo->path; +} + +void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + if (!repair && isValidPath(info.path)) { + // FIXME: copyNAR -> null sink + narSource.drain(); + return; + } + + (void) addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) { + /* FIXME reinstate these, once we can correctly do hash modulo sink as + needed. */ + // assert(info.narHash == nar.first); + // assert(info.narSize == nar.second); + return info; + }}); +} + +StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) +{ + if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) + unsupported("addToStoreFromDump"); + return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { + ValidPathInfo info { + makeFixedOutputPath(method, nar.first, name), + nar.first, + }; + info.narSize = nar.second; + return info; + }); } bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) @@ -367,11 +398,9 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { - // FIXME: some cut&paste from LocalStore::addToStore(). - - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ + /* FIXME: Make BinaryCacheStore::addToStoreCommon support + non-recursive+sha256 so we can just use the default + implementation of this method in terms of addToStoreFromDump. */ StringSink sink; std::optional h; if (method == FileIngestionMethod::Recursive) { @@ -383,34 +412,25 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info { - makeFixedOutputPath(method, *h, name), - Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash - }; - auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs); - - return std::move(info.path); + return addToStoreFromDump(source, name, FileIngestionMethod::Recursive, htSHA256, repair); } StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - ValidPathInfo info { - computeStorePathForText(name, s, references), - Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash - }; - info.references = references; + auto path = computeStorePathForText(name, s, references); - if (repair || !isValidPath(info.path)) { - StringSink sink; - dumpString(s, sink); - auto source = StringSource { *sink.s }; - addToStore(info, source, repair, CheckSigs); - } + if (!repair && isValidPath(path)) + return path; - return std::move(info.path); + auto source = StringSource { s }; + return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { + ValidPathInfo info { path, nar.first }; + info.narSize = nar.second; + info.references = references; + return info; + }); } ref BinaryCacheStore::getFSAccessor() diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 4b779cdd4..ce69ad3b4 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -72,6 +72,10 @@ private: void writeNarInfo(ref narInfo); + StorePath addToStoreCommon( + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, + std::function mkInfo); + public: bool isValidPathUncached(const StorePath & path) override; @@ -85,6 +89,9 @@ public: void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 591140874..85a84d080 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -454,9 +454,7 @@ public: // FIXME: remove? virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) - { - throw Error("addToStoreFromDump() is not supported by this store"); - } + { unsupported("addToStoreFromDump"); } /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ From 8a2e10827f2f57c3b9a0829357363d5f09af65e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Sep 2020 14:08:23 +0200 Subject: [PATCH 043/123] Remove unused Flake::vOutputs field --- src/libexpr/flake/flake.cc | 5 ++--- src/libexpr/flake/flake.hh | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 783b98ef0..460eea5ea 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -215,10 +215,9 @@ static Flake getFlake( if (auto outputs = vInfo.attrs->get(sOutputs)) { expectType(state, tLambda, *outputs->value, *outputs->pos); - flake.vOutputs = allocRootValue(outputs->value); - if ((*flake.vOutputs)->lambda.fun->matchAttrs) { - for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) { + if (outputs->value->lambda.fun->matchAttrs) { + for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { .ref = parseFlakeRef(formal.name) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index c2bb2888b..69c779af8 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -34,7 +34,6 @@ struct Flake std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; - RootValue vOutputs; ~Flake(); }; From 21639b2d179e84805d5ab0949c7bdd74c9e2b7ae Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 23 Sep 2020 16:05:47 +0200 Subject: [PATCH 044/123] Use gold as the linker on Linux Saves ~7s in the linking phase --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 0304557e8..200417c3e 100644 --- a/flake.nix +++ b/flake.nix @@ -58,6 +58,7 @@ configureFlags = lib.optionals stdenv.isLinux [ "--with-sandbox-shell=${sh}/bin/busybox" + "LDFLAGS=-fuse-ld=gold" ]; buildDeps = From 412b3a54fb02cdf49cb084a925bd14c24e14aea8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 23 Sep 2020 10:36:55 -0400 Subject: [PATCH 045/123] Clarify FIXME in `BinaryCacheStore::addToStoreCommon` Co-authored-by: Robert Hensing --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 6679eb16f..817661869 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -311,7 +311,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource (void) addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) { /* FIXME reinstate these, once we can correctly do hash modulo sink as - needed. */ + needed. We need to throw here in case we uploaded a corrupted store path. */ // assert(info.narHash == nar.first); // assert(info.narSize == nar.second); return info; From 3f226f71c185b2fbaaabb01bd0f3ba3cd4a39612 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 23 Sep 2020 14:40:41 +0000 Subject: [PATCH 046/123] Return more info from `BinaryCacheStore::addToStoreCommon` We don't need it yet, but we could/should in the future, and it's a cost-free change since we already have the reference. I like it. Co-authored-by: Robert Hensing --- src/libstore/binary-cache-store.cc | 8 ++++---- src/libstore/binary-cache-store.hh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 817661869..f7a52a296 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -142,7 +142,7 @@ struct FileSource : FdSource } }; -StorePath BinaryCacheStore::addToStoreCommon( +ref BinaryCacheStore::addToStoreCommon( Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::function mkInfo) { @@ -297,7 +297,7 @@ StorePath BinaryCacheStore::addToStoreCommon( stats.narInfoWrite++; - return narInfo->path; + return narInfo; } void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, @@ -330,7 +330,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam }; info.narSize = nar.second; return info; - }); + })->path; } bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) @@ -430,7 +430,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s info.narSize = nar.second; info.references = references; return info; - }); + })->path; } ref BinaryCacheStore::getFSAccessor() diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index ce69ad3b4..5224d7ec8 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -72,7 +72,7 @@ private: void writeNarInfo(ref narInfo); - StorePath addToStoreCommon( + ref addToStoreCommon( Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::function mkInfo); From e8f0b1e996e95e4a1025976d6cfb7ece734129a2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Sep 2020 15:34:11 +0200 Subject: [PATCH 047/123] DerivationGoal::registerOutputs(): Fix bad format string --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 07c5bceb2..2e23dd979 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3858,7 +3858,7 @@ void DerivationGoal::registerOutputs() something like that. */ canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - debug("scanning for references for output %1 in temp location '%1%'", outputName, actualPath); + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); /* Pass blank Sink as we are not ready to hash data at this stage. */ NullSink blank; From d4f8163d104aee0f39e6d8b98160f8de12c18824 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Sep 2020 15:47:12 +0200 Subject: [PATCH 048/123] canonicalisePathMetaData_(): Change assertion to error message --- src/libstore/local-store.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c91f3fbf7..1f58977a5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -478,8 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe ensure that we don't fail on hard links within the same build (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { - assert(!S_ISDIR(st.st_mode)); - if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) + if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) throw BuildError("invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); From cec94738715275ec4761071cefe4a9ae1a565960 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Sep 2020 15:47:30 +0200 Subject: [PATCH 049/123] DerivationGoal::registerOutputs(): Don't canonicalize twice Fixes #4021. --- src/libstore/build.cc | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2e23dd979..2979ce010 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3913,7 +3913,6 @@ void DerivationGoal::registerOutputs() outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; }; - bool rewritten = false; std::optional referencesOpt = std::visit(overloaded { [&](AlreadyRegistered skippedFinalPath) -> std::optional { finish(skippedFinalPath.path); @@ -3943,8 +3942,6 @@ void DerivationGoal::registerOutputs() sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); StringSource source(*sink.s); restorePath(actualPath, source); - - rewritten = true; } }; @@ -4125,11 +4122,6 @@ void DerivationGoal::registerOutputs() } } - /* Get rid of all weird permissions. This also checks that - all files are owned by the build user, if applicable. */ - canonicalisePathMetaData(actualPath, - buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen); - if (buildMode == bmCheck) { if (!worker.store.isValidPath(newInfo.path)) continue; ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); From 2548347bba2d4e6f9947dadf95ed24a16f8a1cdc Mon Sep 17 00:00:00 2001 From: Dominique Martinet Date: Wed, 23 Sep 2020 17:39:19 +0200 Subject: [PATCH 050/123] libutil/archive: add preallocate-contents option Make archive preallocation (fallocate) optional because some filesystems like btrfs do not behave as expected with fallocate. See #3550. --- src/libutil/archive.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 14399dea3..cdf3d89ae 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -27,6 +27,8 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; + Setting preallocateContents{this, true, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; }; static ArchiveSettings archiveSettings; @@ -325,6 +327,9 @@ struct RestoreSink : ParseSink void preallocateContents(uint64_t len) { + if (!archiveSettings.preallocateContents) + return; + #if HAVE_POSIX_FALLOCATE if (len) { errno = posix_fallocate(fd.get(), 0, len); From 31ab4c3816675b5bf2f7b29dfb10112cd8ec9ceb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Sep 2020 19:09:58 +0200 Subject: [PATCH 051/123] Test whether build/repair results are read-only --- tests/repair.sh | 13 +++++-------- tests/simple.sh | 4 +++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/repair.sh b/tests/repair.sh index ec7ad5dca..ba019028d 100644 --- a/tests/repair.sh +++ b/tests/repair.sh @@ -13,14 +13,14 @@ hash=$(nix-hash $path2) chmod u+w $path2 touch $path2/bad -if nix-store --verify --check-contents -v; then - echo "nix-store --verify succeeded unexpectedly" >&2 - exit 1 -fi +(! nix-store --verify --check-contents -v) # The path can be repaired by rebuilding the derivation. nix-store --verify --check-contents --repair +(! [ -e $path2/bad ]) +(! [ -w $path2 ]) + nix-store --verify-path $path2 # Re-corrupt and delete the deriver. Now --verify --repair should @@ -30,10 +30,7 @@ touch $path2/bad nix-store --delete $(nix-store -qd $path2) -if nix-store --verify --check-contents --repair; then - echo "nix-store --verify --repair succeeded unexpectedly" >&2 - exit 1 -fi +(! nix-store --verify --check-contents --repair) nix-build dependencies.nix -o $TEST_ROOT/result --repair diff --git a/tests/simple.sh b/tests/simple.sh index 37631b648..15bd2bd16 100644 --- a/tests/simple.sh +++ b/tests/simple.sh @@ -10,13 +10,15 @@ outPath=$(nix-store -rvv "$drvPath") echo "output path is $outPath" +(! [ -w $outPath ]) + text=$(cat "$outPath"/hello) if test "$text" != "Hello World!"; then exit 1; fi # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. nix-store --delete $outPath -if test -e $outPath/hello; then false; fi +(! [ -e $outPath/hello ]) outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)" if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then From 688bd4fb500c70cda4ffe6864bbf59dbc4da49a2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Sep 2020 19:10:16 +0200 Subject: [PATCH 052/123] After rewriting a path, make it read-only --- src/libstore/build.cc | 47 ++++++++++++++----------------------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2979ce010..cf04fbe56 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3735,15 +3735,12 @@ void DerivationGoal::runChild() } -static void moveCheckToStore(const Path & src, const Path & dst) +/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if + it's a directory and we're not root (to be able to update the + directory's parent link ".."). */ +static void movePath(const Path & src, const Path & dst) { - /* For the rename of directory to succeed, we must be running as root or - the directory must be made temporarily writable (to update the - directory's parent link ".."). */ - struct stat st; - if (lstat(src.c_str(), &st) == -1) { - throw SysError("getting attributes of path '%1%'", src); - } + auto st = lstat(src); bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); @@ -3942,6 +3939,10 @@ void DerivationGoal::registerOutputs() sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); StringSource source(*sink.s); restorePath(actualPath, source); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, -1, inodesSeen); } }; @@ -4024,7 +4025,7 @@ void DerivationGoal::registerOutputs() [&](DerivationOutputInputAddressed output) { /* input-addressed case */ auto requiredFinalPath = output.path; - /* Preemtively add rewrite rule for final hash, as that is + /* Preemptively add rewrite rule for final hash, as that is what the NAR hash will use rather than normalized-self references */ if (scratchPath != requiredFinalPath) outputRewrites.insert_or_assign( @@ -4098,27 +4099,9 @@ void DerivationGoal::registerOutputs() else. No moving needed. */ assert(newInfo.ca); } else { - /* Temporarily add write perm so we can move, will be fixed - later. */ - { - struct stat st; - auto & mode = st.st_mode; - if (lstat(actualPath.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", actualPath); - mode |= 0200; - /* Try to change the perms, but only if the file isn't a - symlink as symlinks permissions are mostly ignored and - calling `chmod` on it will just forward the call to the - target of the link. */ - if (!S_ISLNK(st.st_mode)) - if (chmod(actualPath.c_str(), mode) == -1) - throw SysError("changing mode of '%1%' to %2$o", actualPath, mode); - } - if (rename( - actualPath.c_str(), - worker.store.toRealPath(finalDestPath).c_str()) == -1) - throw SysError("moving build output '%1%' from it's temporary location to the Nix store", finalDestPath); - actualPath = worker.store.toRealPath(finalDestPath); + auto destPath = worker.store.toRealPath(finalDestPath); + movePath(actualPath, destPath); + actualPath = destPath; } } @@ -4128,9 +4111,9 @@ void DerivationGoal::registerOutputs() if (newInfo.narHash != oldInfo.narHash) { worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { - Path dst = worker.store.toRealPath(finalDestPath + checkSuffix); + auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); deletePath(dst); - moveCheckToStore(actualPath, dst); + movePath(actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), From 236d9ee7f72ca4238f5f44c244fd2b885691c6ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Sep 2020 19:17:28 +0200 Subject: [PATCH 053/123] lstat() cleanup --- src/libexpr/parser.y | 3 +-- src/libstore/build.cc | 9 ++------- src/libstore/gc.cc | 4 +--- src/libstore/local-store.cc | 18 +++++------------- src/libstore/optimise-store.cc | 12 +++--------- src/libstore/profiles.cc | 5 +---- src/libutil/archive.cc | 4 +--- .../resolve-system-dependencies.cc | 6 +----- 8 files changed, 15 insertions(+), 46 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 24b21f7da..a4c84c526 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -614,8 +614,7 @@ Path resolveExprPath(Path path) // Basic cycle/depth limit to avoid infinite loops. if (++followCount >= maxFollow) throw Error("too many symbolic links encountered while traversing the path '%s'", path); - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%s'", path); + st = lstat(path); if (!S_ISLNK(st.st_mode)) break; path = absPath(readLink(path), dirOf(path)); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cf04fbe56..6b53f529a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2367,10 +2367,7 @@ void DerivationGoal::startBuilder() for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); Path r = worker.store.toRealPath(p); - struct stat st; - if (lstat(r.c_str(), &st)) - throw SysError("getting attributes of path '%s'", p); - if (S_ISDIR(st.st_mode)) + if (S_ISDIR(lstat(r).st_mode)) dirsInChroot.insert_or_assign(p, r); else linkOrCopy(r, chrootRootDir + p); @@ -3144,9 +3141,7 @@ void DerivationGoal::addDependency(const StorePath & path) if (pathExists(target)) throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - struct stat st; - if (lstat(source.c_str(), &st)) - throw SysError("getting attributes of path '%s'", source); + auto st = lstat(source); if (S_ISDIR(st.st_mode)) { diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 08b53c702..518a357ef 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -663,9 +663,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (name == "." || name == "..") continue; Path path = linksDir + "/" + name; - struct stat st; - if (lstat(path.c_str(), &st) == -1) - throw SysError("statting '%1%'", path); + auto st = lstat(path); if (st.st_nlink != 1) { actualSize += st.st_size; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1f58977a5..ee997ef3a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -114,8 +114,7 @@ LocalStore::LocalStore(const Params & params) Path path = realStoreDir; struct stat st; while (path != "/") { - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); + st = lstat(path); if (S_ISLNK(st.st_mode)) throw Error( "the path '%1%' is a symlink; " @@ -419,10 +418,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct void canonicaliseTimestampAndPermissions(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); - canonicaliseTimestampAndPermissions(path, st); + canonicaliseTimestampAndPermissions(path, lstat(path)); } @@ -440,9 +436,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe } #endif - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); /* Really make sure that the path is of a supported type. */ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) @@ -521,9 +515,7 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino /* On platforms that don't have lchown(), the top-level path can't be a symlink, since we can't change its ownership. */ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); if (st.st_uid != geteuid()) { assert(S_ISLNK(st.st_mode)); @@ -1454,7 +1446,7 @@ static void makeMutable(const Path & path) { checkInterrupt(); - struct stat st = lstat(path); + auto st = lstat(path); if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index e4b4b6213..c032a5e22 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -17,9 +17,7 @@ namespace nix { static void makeWritable(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) throw SysError("changing writability of '%1%'", path); } @@ -94,9 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for @@ -187,9 +183,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Yes! We've seen a file with the same contents. Replace the current file with a hard link to that file. */ - struct stat stLink; - if (lstat(linkPath.c_str(), &stLink)) - throw SysError("getting attributes of path '%1%'", linkPath); + auto stLink = lstat(linkPath); if (st.st_ino == stLink.st_ino) { debug(format("'%1%' is already linked to '%2%'") % path % linkPath); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index c20386e2b..c3809bad7 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -39,13 +39,10 @@ std::pair> findGenerations(Path pro for (auto & i : readDirectory(profileDir)) { if (auto n = parseName(profileName, i.name)) { auto path = profileDir + "/" + i.name; - struct stat st; - if (lstat(path.c_str(), &st) != 0) - throw SysError("statting '%1%'", path); gens.push_back({ .number = *n, .path = path, - .creationTime = st.st_mtime + .creationTime = lstat(path).st_mtime }); } } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 14399dea3..4f59c8ed5 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -66,9 +66,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting attributes of path '%1%'", path); + auto st = lstat(path); sink << "("; diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 434ad80a6..d30227e4e 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -111,11 +111,7 @@ std::set runResolver(const Path & filename) bool isSymlink(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st) == -1) - throw SysError("getting attributes of path '%1%'", path); - - return S_ISLNK(st.st_mode); + return S_ISLNK(lstat(path).st_mode); } Path resolveSymlink(const Path & path) From 9a24ece122eb19f3b69f072f6ce3c39c5ae4d0ce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Sep 2020 20:21:08 +0200 Subject: [PATCH 054/123] Fix exception --- src/libstore/build.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6b53f529a..f0820e711 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1675,7 +1675,7 @@ void DerivationGoal::tryLocalBuild() { } -void replaceValidPath(const Path & storePath, const Path tmpPath) +void replaceValidPath(const Path & storePath, const Path & tmpPath) { /* We can't atomically replace storePath (the original) with tmpPath (the replacement), so we have to move it out of the @@ -1685,8 +1685,9 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) if (pathExists(storePath)) rename(storePath.c_str(), oldPath.c_str()); if (rename(tmpPath.c_str(), storePath.c_str()) == -1) { + auto ex = SysError("moving '%s' to '%s'", tmpPath, storePath); rename(oldPath.c_str(), storePath.c_str()); // attempt to recover - throw SysError("moving '%s' to '%s'", tmpPath, storePath); + throw ex; } deletePath(oldPath); } From 4ce8a3ed452f06b18a40cffefc37d47c916927a8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Sep 2020 21:29:10 +0200 Subject: [PATCH 055/123] Hopefully fix EPERM on macOS --- src/libstore/build.cc | 72 ++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f0820e711..db7dbc17e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1675,6 +1675,33 @@ void DerivationGoal::tryLocalBuild() { } +static void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError("setting permissions on '%s'", path); +} + + +/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if + it's a directory and we're not root (to be able to update the + directory's parent link ".."). */ +static void movePath(const Path & src, const Path & dst) +{ + auto st = lstat(src); + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError("renaming '%1%' to '%2%'", src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + void replaceValidPath(const Path & storePath, const Path & tmpPath) { /* We can't atomically replace storePath (the original) with @@ -1683,12 +1710,20 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) we're repairing (say) Glibc, we end up with a broken system. */ Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); if (pathExists(storePath)) - rename(storePath.c_str(), oldPath.c_str()); - if (rename(tmpPath.c_str(), storePath.c_str()) == -1) { - auto ex = SysError("moving '%s' to '%s'", tmpPath, storePath); - rename(oldPath.c_str(), storePath.c_str()); // attempt to recover - throw ex; + movePath(storePath, oldPath); + + try { + movePath(tmpPath, storePath); + } catch (...) { + try { + // attempt to recover + movePath(oldPath, storePath); + } catch (...) { + ignoreException(); + } + throw; } + deletePath(oldPath); } @@ -2006,13 +2041,6 @@ HookReply DerivationGoal::tryBuildHook() } -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - - int childEntry(void * arg) { ((DerivationGoal *) arg)->runChild(); @@ -3731,26 +3759,6 @@ void DerivationGoal::runChild() } -/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if - it's a directory and we're not root (to be able to update the - directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) -{ - auto st = lstat(src); - - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); - - if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); - - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); - - if (changePerm) - chmod_(dst, st.st_mode); -} - - void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output From bd5f3dbe118d569ffb201ce14394572ac5fc412c Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 12:30:03 -0700 Subject: [PATCH 056/123] Fixes fall-through to report correct description of hash-file command. --- src/nix/hash.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0eca4f8ea..494f00a20 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -44,6 +44,7 @@ struct CmdHash : Command switch (mode) { case FileIngestionMethod::Flat: d = "print cryptographic hash of a regular file"; + break; case FileIngestionMethod::Recursive: d = "print cryptographic hash of the NAR serialisation of a path"; }; From ed218e1d6cf755fc3011c0954eb7031f95d3d732 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 25 Sep 2020 00:07:42 +0300 Subject: [PATCH 057/123] Fix max-jobs option After 0ed946aa616bbf7ffe7f90d3309abdd27d875b10, max-jobs setting (-j/--max-jobs) stopped working. The reason was that nrLocalBuilds (which compared to maxBuildJobs to figure out whether the limit is reached or not) is not incremented yet when tryBuild is started; So, the solution is to move the check to tryLocalBuild. Closes https://github.com/nixos/nix/issues/3763 --- src/libstore/build.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index db7dbc17e..3fc24b221 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1612,6 +1612,13 @@ void DerivationGoal::tryToBuild() actLock.reset(); + state = &DerivationGoal::tryLocalBuild; + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::tryLocalBuild() { + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); + /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1622,12 +1629,6 @@ void DerivationGoal::tryToBuild() return; } - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} - -void DerivationGoal::tryLocalBuild() { - /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ if (settings.buildUsersGroup != "" && getuid() == 0) { From 4d863a9fcb9460a9e4978466e03d2982d32e39f0 Mon Sep 17 00:00:00 2001 From: Paul Opiyo <59094545+paulopiyo777@users.noreply.github.com> Date: Thu, 24 Sep 2020 18:05:47 -0500 Subject: [PATCH 058/123] Remove redundant value checks std::optional had redundant checks for whether it had a value. An object is emplaced either way so it can be dereferenced without repeating a value check --- src/libexpr/flake/flake.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 460eea5ea..760ed1a6e 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -48,17 +48,17 @@ static std::tuple fetchOrSubstituteTree( resolvedRef = originalRef.resolve(state.store); auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); - flakeCache.push_back({resolvedRef, fetchedResolved.value()}); - fetched.emplace(fetchedResolved.value()); + flakeCache.push_back({resolvedRef, *fetchedResolved}); + fetched.emplace(*fetchedResolved); } else { throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); } } - flakeCache.push_back({originalRef, fetched.value()}); + flakeCache.push_back({originalRef, *fetched}); } - auto [tree, lockedRef] = fetched.value(); + auto [tree, lockedRef] = *fetched; debug("got tree '%s' from '%s'", state.store->printStorePath(tree.storePath), lockedRef); From 83fec38fc93922192ada7c0409fec76578ef8dfb Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 22:41:24 -0700 Subject: [PATCH 059/123] Update document generation for empty json object values. --- doc/manual/generate-options.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix index 7afe279c3..3c31a4eec 100644 --- a/doc/manual/generate-options.nix +++ b/doc/manual/generate-options.nix @@ -13,7 +13,12 @@ concatStrings (map then "*empty*" else if isBool option.value then (if option.value then "`true`" else "`false`") - else "`" + toString option.value + "`") + "\n\n" + else + # n.b. a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is "null" in + # JSON, but that converts to "{ }" here. + (if isAttrs option.value then "`\"\"`" + else "`" + toString option.value + "`")) + "\n\n" + (if option.aliases != [] then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n" else "") From a439e9488df6c13d0e44dd4816df98487d69f4c6 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 22:42:59 -0700 Subject: [PATCH 060/123] Support StringMap configuration settings. Allows Configuration values that are space-separated key=value pairs. --- src/libutil/config.cc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 309d23b40..b14feb10d 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -268,6 +268,26 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } +template<> void BaseSetting::set(const std::string & str) +{ + auto kvpairs = tokenizeString(str); + for (auto & s : kvpairs) + { + auto eq = s.find_first_of('='); + if (std::string::npos != eq) + value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); + // else ignored + } +} + +template<> std::string BaseSetting::to_string() const +{ + Strings kvstrs; + std::transform(value.begin(), value.end(), back_inserter(kvstrs), + [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; }); + return concatStringsSep(" ", kvstrs); +} + template class BaseSetting; template class BaseSetting; template class BaseSetting; @@ -278,6 +298,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; +template class BaseSetting; void PathSetting::set(const std::string & str) { From c2f48cfcee501dd15690245d481d154444456f66 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 22:46:03 -0700 Subject: [PATCH 061/123] Complete conversion of "url" to "host" with associated variable renaming. Completes the change begun in commit 56f1e0d to consistently use the "host" attribute for "github" and "gitlab" inputs instead of a "url" attribute. --- src/libfetchers/github.cc | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index d8d0351b9..eb758bc5f 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -65,9 +65,9 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains multiple branch/tag names", url.url); ref = value; } - else if (name == "url") { + else if (name == "host") { if (!std::regex_match(value, urlRegex)) - throw BadURL("URL '%s' contains an invalid instance url", url.url); + throw BadURL("URL '%s' contains an invalid instance host", url.url); host_url = value; } // FIXME: barf on unsupported attributes @@ -82,7 +82,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); if (ref) input.attrs.insert_or_assign("ref", *ref); - if (host_url) input.attrs.insert_or_assign("url", *host_url); + if (host_url) input.attrs.insert_or_assign("host", *host_url); return input; } @@ -92,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != type()) return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") throw Error("unsupported input attribute '%s'", name); getStrAttr(attrs, "owner"); @@ -208,9 +208,9 @@ struct GitHubInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); Headers headers; std::string accessToken = settings.githubAccessToken.get(); @@ -230,9 +230,9 @@ struct GitHubInputScheme : GitArchiveInputScheme { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); std::string accessToken = settings.githubAccessToken.get(); @@ -246,9 +246,9 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) .clone(destDir); } @@ -284,10 +284,14 @@ struct GitLabInputScheme : GitArchiveInputScheme DownloadUrl getDownloadUrl(const Input & input) const override { - // FIXME: This endpoint has a rate limit threshold of 5 requests per minute - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + // This endpoint has a rate limit threshold that may be + // server-specific and vary based whether the user is + // authenticated via an accessToken or not, but the usual rate + // is 10 reqs/sec/ip-addr. See + // https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); std::string accessToken = settings.gitlabAccessToken.get(); @@ -302,10 +306,10 @@ struct GitLabInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) .clone(destDir); } From 8fba2a8b54283ea1cf56ae75faf4ced5f3e8e4a1 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 22:49:44 -0700 Subject: [PATCH 062/123] Update to use access-tokens configuration for github/gitlab access. This change provides support for using access tokens with other instances of GitHub and GitLab beyond just github.com and gitlab.com (especially company-specific or foundation-specific instances). This change also provides the ability to specify the type of access token being used, where different types may have different handling, based on the forge type. --- src/libfetchers/github.cc | 100 ++++++++++++++++++++++---------------- src/libstore/globals.hh | 50 ++++++++++++++++++- 2 files changed, 107 insertions(+), 43 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index eb758bc5f..0e0655367 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "types.hh" +#include #include namespace nix::fetchers { @@ -12,13 +13,10 @@ namespace nix::fetchers { struct DownloadUrl { std::string url; - std::optional> access_token_header; + Headers headers; - DownloadUrl(const std::string & url) - : url(url) { } - - DownloadUrl(const std::string & url, const std::pair & access_token_header) - : url(url), access_token_header(access_token_header) { } + DownloadUrl(const std::string & url, const Headers & headers) + : url(url), headers(headers) { } }; // A github or gitlab url @@ -29,7 +27,7 @@ struct GitArchiveInputScheme : InputScheme { virtual std::string type() = 0; - virtual std::pair accessHeaderFromToken(const std::string & token) const = 0; + virtual std::optional > accessHeaderFromToken(const std::string & token) const = 0; std::optional inputFromURL(const ParsedURL & url) override { @@ -144,6 +142,27 @@ struct GitArchiveInputScheme : InputScheme return input; } + std::optional getAccessToken(const std::string &host) const { + auto tokens = settings.accessTokens.get(); + auto pat = tokens.find(host); + if (pat == tokens.end()) + return std::nullopt; + return pat->second; + } + + Headers makeHeadersWithAuthTokens(const std::string & host) const { + Headers headers; + auto accessToken = getAccessToken(host); + if (accessToken) { + auto hdr = accessHeaderFromToken(*accessToken); + if (hdr) + headers.push_back(*hdr); + else + warn("Unrecognized access token for host '%s'", host); + } + return headers; + } + virtual Hash getRevFromRef(nix::ref store, const Input & input) const = 0; virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; @@ -175,12 +194,7 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - Headers headers; - if (url.access_token_header) { - headers.push_back(*url.access_token_header); - } - - auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true); + auto [tree, lastModified] = downloadTarball(store, url.url, url.headers, "source", true); input.attrs.insert_or_assign("lastModified", lastModified); @@ -202,7 +216,13 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string type() override { return "github"; } - std::pair accessHeaderFromToken(const std::string & token) const { + std::optional > accessHeaderFromToken(const std::string & token) const { + // Github supports PAT/OAuth2 tokens and HTTP Basic + // Authentication. The former simply specifies the token, the + // latter can use the token as the password. Only the first + // is used here. See + // https://developer.github.com/v3/#authentication and + // https://docs.github.com/en/developers/apps/authorizing-oath-apps return std::pair("Authorization", fmt("token %s", token)); } @@ -212,10 +232,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers; - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - headers.push_back(accessHeaderFromToken(accessToken)); + Headers headers = makeHeadersWithAuthTokens(host); auto json = nlohmann::json::parse( readFile( @@ -235,13 +252,8 @@ struct GitHubInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") { - auto auth_header = accessHeaderFromToken(accessToken); - return DownloadUrl(url, auth_header); - } else { - return DownloadUrl(url); - } + Headers headers = makeHeadersWithAuthTokens(host); + return DownloadUrl(url, headers); } void clone(const Input & input, const Path & destDir) override @@ -258,20 +270,32 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string type() override { return "gitlab"; } - std::pair accessHeaderFromToken(const std::string & token) const { - return std::pair("Authorization", fmt("Bearer %s", token)); + std::optional > accessHeaderFromToken(const std::string & token) const { + // Gitlab supports 4 kinds of authorization, two of which are + // relevant here: OAuth2 and PAT (Private Access Token). The + // user can indicate which token is used by specifying the + // token as :, where type is "OAuth2" or "PAT". + // If the is unrecognized, this will fall back to + // treating this simply has :. See + // https://docs.gitlab.com/12.10/ee/api/README.html#authentication + auto fldsplit = token.find_first_of(':'); + // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ... + if ("OAuth2" == token.substr(0, fldsplit)) + return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1))); + if ("PAT" == token.substr(0, fldsplit)) + return std::make_pair("Private-token", token.substr(fldsplit+1)); + warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit)); + return std::nullopt; } Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); + // See rate limiting note below auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers; - std::string accessToken = settings.gitlabAccessToken.get(); - if (accessToken != "") - headers.push_back(accessHeaderFromToken(accessToken)); + Headers headers = makeHeadersWithAuthTokens(host); auto json = nlohmann::json::parse( readFile( @@ -294,14 +318,8 @@ struct GitLabInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - std::string accessToken = settings.gitlabAccessToken.get(); - if (accessToken != "") { - auto auth_header = accessHeaderFromToken(accessToken); - return DownloadUrl(url, auth_header); - } else { - return DownloadUrl(url); - } - + Headers headers = makeHeadersWithAuthTokens(host); + return DownloadUrl(url, headers); } void clone(const Input & input, const Path & destDir) override diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b2e7610ee..646422399 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -863,8 +863,54 @@ public: Setting githubAccessToken{this, "", "github-access-token", "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."}; - Setting gitlabAccessToken{this, "", "gitlab-access-token", - "GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."}; + Setting accessTokens{this, {}, "access-tokens", + R"( + Access tokens used to access protected GitHub, GitLab, or + other locations requiring token-based authentication. + + Access tokens are specified as a string made up of + space-separated `host=token` values. The specific token + used is selected by matching the `host` portion against the + "host" specification of the input. The actual use of the + `token` value is determined by the type of resource being + accessed: + + * Github: the token value is the OAUTH-TOKEN string obtained + as the Personal Access Token from the Github server (see + https://docs.github.com/en/developers/apps/authorizing-oath-apps). + + * Gitlab: the token value is either the OAuth2 token or the + Personal Access Token (these are different types tokens + for gitlab, see + https://docs.gitlab.com/12.10/ee/api/README.html#authentication). + The `token` value should be `type:tokenstring` where + `type` is either `OAuth2` or `PAT` to indicate which type + of token is being specified. + + Example `~/.config/nix/nix.conf`: + + ``` + personal-access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + ``` + + Example `~/code/flake.nix`: + + ```nix + input.foo = { + type="gitlab"; + host="gitlab.mycompany.com"; + owner="mycompany"; + repo="pro"; + }; + ``` + + This example specifies three tokens, one each for accessing + github.com, gitlab.mycompany.com, and sourceforge.net. + + The `input.foo` uses the "gitlab" fetcher, which might + requires specifying the token type along with the token + value. + )"}; Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; From 7b2ae472ff05a39cd635ac10dbbce3cd17b60c93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 25 Sep 2020 10:27:40 +0200 Subject: [PATCH 063/123] expectArg(): Respect the 'optional' flag --- src/libutil/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 3c1f87f7e..f41242e17 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -192,7 +192,7 @@ public: { expectArgs({ .label = label, - .optional = true, + .optional = optional, .handler = {dest} }); } From ef2a14be190f7162e85e9bdd44dd45bd9ddfe391 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Fri, 25 Sep 2020 08:08:27 -0700 Subject: [PATCH 064/123] Fix reference to older name for access-tokens config value. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 646422399..959ebe360 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -890,7 +890,7 @@ public: Example `~/.config/nix/nix.conf`: ``` - personal-access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" ``` Example `~/code/flake.nix`: From 5a35cc29bffc88b88f883dfcdd1bb251eab53ecd Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Fri, 25 Sep 2020 08:09:56 -0700 Subject: [PATCH 065/123] Re-add support for github-access-token, but mark as deprecated. --- src/libfetchers/github.cc | 10 ++++++++++ src/libstore/globals.hh | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 0e0655367..443644639 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -146,7 +146,17 @@ struct GitArchiveInputScheme : InputScheme auto tokens = settings.accessTokens.get(); auto pat = tokens.find(host); if (pat == tokens.end()) + { + if ("github.com" == host) + { + auto oldcfg = settings.githubAccessToken.get(); + if (!oldcfg.empty()) { + warn("using deprecated 'github-access-token' config value; please use 'access-tokens' instead"); + return oldcfg; + } + } return std::nullopt; + } return pat->second; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 959ebe360..bd36ffc17 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -861,7 +861,7 @@ public: )"}; Setting githubAccessToken{this, "", "github-access-token", - "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."}; + "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes (deprecated, please use 'access-tokens' instead)."}; Setting accessTokens{this, {}, "access-tokens", R"( From cb186f1e7536c9448455bfbf8dec16ad6600e88e Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Fri, 25 Sep 2020 09:36:18 -0700 Subject: [PATCH 066/123] Use "?dir=..." portion of "registry add" local path specification. The registry targets generally follow a URL formatting schema with support for a query parameter of "?dir=subpath" to specify a sub-path location below the URL root. Alternatively, an absolute path can be specified. This specification mode accepts the query parameter but ignores/drops it. It would probably be better to either (a) disallow the query parameter for the path form, or (b) recognize the query parameter and add to the path. This patch implements (b) for consistency, and to make it easier for tooling that might switch between a remote git reference and a local path reference. See also issue #4050. --- src/libexpr/flake/flakeref.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index d5c2ffe66..762e27e1f 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -157,7 +157,8 @@ std::pair parseFlakeRefWithFragment( } else { if (!hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); - path = canonPath(path); + auto query = decodeQuery(match[2]); + path = canonPath(path + "/" + get(query, "dir").value_or("")); } fetchers::Attrs attrs; From 5db83dd771b92bcacb0cd4dea7d4e06f767769ca Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 26 Sep 2020 03:21:36 +0000 Subject: [PATCH 067/123] BinaryCacheStore::addTextToStore include CA field --- src/libstore/binary-cache-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f7a52a296..a3a73fe27 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -419,7 +419,8 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - auto path = computeStorePathForText(name, s, references); + auto textHash = hashString(htSHA256, s); + auto path = makeTextPath(name, textHash, references); if (!repair && isValidPath(path)) return path; @@ -428,6 +429,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { path, nar.first }; info.narSize = nar.second; + info.ca = TextHash { textHash }; info.references = references; return info; })->path; From 1832436526307ac92baec0146b89e9a5cf3aca35 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 26 Sep 2020 04:56:29 +0000 Subject: [PATCH 068/123] Fix up BinaryCacheStore::addToStore taking a path --- src/libstore/binary-cache-store.cc | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a3a73fe27..b34b10fd1 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -401,19 +401,30 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default implementation of this method in terms of addToStoreFromDump. */ - StringSink sink; - std::optional h; + + HashSink sink { hashAlgo }; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); - h = hashString(hashAlgo, *sink.s); } else { - auto s = readFile(srcPath); - dumpString(s, sink); - h = hashString(hashAlgo, s); + readFile(srcPath, sink); } + auto h = sink.finish().first; - auto source = StringSource { *sink.s }; - return addToStoreFromDump(source, name, FileIngestionMethod::Recursive, htSHA256, repair); + auto source = sinkToSource([&](Sink & sink) { + dumpPath(srcPath, sink, filter); + }); + return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { + ValidPathInfo info { + makeFixedOutputPath(method, h, name), + nar.first, + }; + info.narSize = nar.second; + info.ca = FixedOutputHash { + .method = method, + .hash = h, + }; + return info; + })->path; } StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, From 8b4a542d1767e0df7b3c0902b766f34352cb0958 Mon Sep 17 00:00:00 2001 From: Mateusz Piotrowski <0mp@FreeBSD.org> Date: Sat, 26 Sep 2020 13:33:04 +0200 Subject: [PATCH 069/123] Fix a typo (#4073) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3cf4e44fa..11fe5f932 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ for more details. ## Installation -On Linux and macOS the easiest way to Install Nix is to run the following shell command +On Linux and macOS the easiest way to install Nix is to run the following shell command (as a user other than root): ```console From 25fffdda865e51f68e72e0ca1775800b60391820 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 26 Sep 2020 10:17:30 -0400 Subject: [PATCH 070/123] Remove redundant nar hash and size setting Co-authored-by: Robert Hensing --- src/libstore/binary-cache-store.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b34b10fd1..59d02cab5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -173,8 +173,6 @@ ref BinaryCacheStore::addToStoreCommon( auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->narSize = info.narSize; // FIXME needed? - narInfo->narHash = info.narHash; // FIXME needed? narInfo->compression = compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; From a76fb07314d3c5dea06ac2c1a36f8af1e76c2dde Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 26 Sep 2020 17:38:11 +0200 Subject: [PATCH 071/123] libmain/progress-bar: don't trim whitespace on the left When running `nix build -L` it can be fairly hard to read the output if the build program intentionally renders whitespace on the left. A typical example is `g++` displaying compilation errors. With this patch, the whitespace on the left is retained to make the log more readable: ``` foo> no configure script, doing nothing foo> building foo> foobar.cc: In function 'int main()': foo> foobar.cc:5:5: error: 'wrong_func' was not declared in this scope foo> 5 | wrong_func(1); foo> | ^~~~~~~~~~ error: --- Error ------------------------------------------------------------------------------------- nix error: --- Error --- nix-daemon builder for '/nix/store/i1q76cw6cyh91raaqg5p5isd1l2x6rx2-foo-1.0.drv' failed with exit code 1 ``` --- src/libmain/progress-bar.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index be3c06a38..07b45b3b5 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -256,7 +256,7 @@ public: } else if (type == resBuildLogLine || type == resPostBuildLogLine) { - auto lastLine = trim(getS(fields, 0)); + auto lastLine = chomp(getS(fields, 0)); if (!lastLine.empty()) { auto i = state->its.find(act); assert(i != state->its.end()); From 5885b0cfd878b4b60556c5b03bbe52244d04191a Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Sun, 27 Sep 2020 13:04:06 -0700 Subject: [PATCH 072/123] Miscellaneous spelling fixes in comments. (#4071) --- src/libexpr/flake/flake.cc | 2 +- src/libfetchers/fetchers.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 760ed1a6e..b4ede542c 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -247,7 +247,7 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup } /* Compute an in-memory lock file for the specified top-level flake, - and optionally write it to file, it the flake is writable. */ + and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( EvalState & state, const FlakeRef & topRef, diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 89b1e6e7d..191e91978 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -73,7 +73,7 @@ public: StorePath computeStorePath(Store & store) const; - // Convience functions for common attributes. + // Convenience functions for common attributes. std::string getType() const; std::optional getNarHash() const; std::optional getRef() const; From 3655875483306fa893ec2b01295151819a00ccaa Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 27 Sep 2020 22:35:03 +0200 Subject: [PATCH 073/123] doc/manual: update hacking docs (#4078) * By default, build artifacts should be installed into `outputs/` rather than `inst/`[1]. * Add instructions on how to run unit-tests. [1] 733d2e9402807e54d503c3113e854bfddb3d44e0 --- doc/manual/src/hacking.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/hacking.md b/doc/manual/src/hacking.md index 5bd884ce8..2a1e55e5b 100644 --- a/doc/manual/src/hacking.md +++ b/doc/manual/src/hacking.md @@ -39,17 +39,17 @@ To build Nix itself in this shell: ```console [nix-shell]$ ./bootstrap.sh -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/inst +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out [nix-shell]$ make -j $NIX_BUILD_CORES ``` -To install it in `$(pwd)/inst` and test it: +To install it in `$(pwd)/outputs` and test it: ```console [nix-shell]$ make install -[nix-shell]$ make installcheck -[nix-shell]$ ./inst/bin/nix --version -nix (Nix) 2.4 +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 3.0 ``` To run a functional test: @@ -58,6 +58,12 @@ To run a functional test: make tests/test-name-should-auto-complete.sh.test ``` +To run the unit-tests for C++ code: + +``` +make check +``` + If you have a flakes-enabled Nix you can replace: ```console From 095a91f55a89d856e51ff099686e2bbe6fb9a384 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 05:37:07 +0000 Subject: [PATCH 074/123] Bump cachix/install-nix-action from v10 to v11 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from v10 to v11. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v10...95a8068e317b8def9482980abe762f36c77ccc99) Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f504a8ea..2f5c548d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v10 + - uses: cachix/install-nix-action@v11 with: skip_adding_nixpkgs_channel: true #- run: nix flake check From ed66d010659a710646f5f1015c97a58614a713b3 Mon Sep 17 00:00:00 2001 From: Mateusz Piotrowski <0mp@FreeBSD.org> Date: Mon, 28 Sep 2020 15:23:21 +0200 Subject: [PATCH 075/123] Fix tar invocation on FreeBSD tar(1) on FreeBSD does not use standard output or input when the -f flag is not provided. Instead, it defaults to /dev/sa0 on FreeBSD. Make this tar invocation a bit more robust and explicitly tell tar(1) to use standard output. This is one of the issues discovered while porting Nix to FreeBSD. It has been tested and committed locally to FreeBSD ports: https://svnweb.freebsd.org/ports/head/sysutils/nix/Makefile?revision=550026&view=markup#l108 --- tests/tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tarball.sh b/tests/tarball.sh index 88a1a07a0..fe65a22e4 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -17,7 +17,7 @@ test_tarball() { local compressor="$2" tarball=$TEST_ROOT/tarball.tar$ext - (cd $TEST_ROOT && tar c tarball) | $compressor > $tarball + (cd $TEST_ROOT && tar cf - tarball) | $compressor > $tarball nix-env -f file://$tarball -qa --out-path | grep -q dependencies From 6c31297d80b71ab9c2c1084fae22f726f7d89daa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 28 Sep 2020 11:32:58 -0400 Subject: [PATCH 076/123] Update src/libstore/binary-cache-store.cc Co-authored-by: Eelco Dolstra --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 59d02cab5..f6224d6a0 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -307,7 +307,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource return; } - (void) addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) { + addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) { /* FIXME reinstate these, once we can correctly do hash modulo sink as needed. We need to throw here in case we uploaded a corrupted store path. */ // assert(info.narHash == nar.first); From 80e335bb5837d7bfbf1f14d4f3d39525013c4f4d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 28 Sep 2020 15:43:56 +0000 Subject: [PATCH 077/123] Use `drvPath2` and give it a better name --- src/libstore/build.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0a6f2ccb5..0499273a4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4315,19 +4315,19 @@ void DerivationGoal::registerOutputs() but it's fine to do in all cases. */ bool isCaFloating = drv->type() == DerivationType::CAFloating; - auto drvPath2 = drvPath; + auto drvPathResolved = drvPath; if (!useDerivation && isCaFloating) { /* Once a floating CA derivations reaches this point, it must already be resolved, so we don't bother trying to downcast drv to get would would just be an empty inputDrvs field. */ Derivation drv2 { *drv }; - drvPath2 = writeDerivation(worker.store, drv2); + drvPathResolved = writeDerivation(worker.store, drv2); } if (useDerivation || isCaFloating) for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path); + worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); } From c89fa3f644eec0652e97c11d9246f9580e5929fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 28 Sep 2020 21:08:14 +0300 Subject: [PATCH 078/123] Update .github/workflows/test.yml --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2f5c548d9..adc56e223 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,6 @@ jobs: with: fetch-depth: 0 - uses: cachix/install-nix-action@v11 - with: skip_adding_nixpkgs_channel: true #- run: nix flake check - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) From f1428484be248c7ba3f96379d8bf256b8df1a706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 28 Sep 2020 21:08:24 +0300 Subject: [PATCH 079/123] Update .github/workflows/test.yml --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index adc56e223..829111b67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,5 @@ jobs: with: fetch-depth: 0 - uses: cachix/install-nix-action@v11 - skip_adding_nixpkgs_channel: true #- run: nix flake check - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) From 00135e13f49e9b20e3ef03f2516e7cc277c40ca9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 28 Sep 2020 18:19:10 +0000 Subject: [PATCH 080/123] Clarify comment a bit --- src/libstore/derivations.hh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 74601134e..d48266774 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -128,11 +128,12 @@ struct Derivation : BasicDerivation std::string unparse(const Store & store, bool maskOutputs, std::map * actualInputs = nullptr) const; - /* Return the underlying basic derivation but with + /* Return the underlying basic derivation but with these changes: - 1. input drv outputs moved to input sources. + 1. Input drvs are emptied, but the outputs of them that were used are + added directly to input sources. - 2. placeholders replaced with realized input store paths. */ + 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); Derivation() = default; From de86abbf3f9f0d18f7506b1d50072597786332ea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 29 Sep 2020 12:55:06 +0200 Subject: [PATCH 081/123] Cleanup --- src/libfetchers/github.cc | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 7bf155c69..3734c4278 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -14,12 +14,6 @@ struct DownloadUrl { std::string url; std::optional> access_token_header; - - DownloadUrl(const std::string & url) - : url(url) { } - - DownloadUrl(const std::string & url, const std::pair & access_token_header) - : url(url), access_token_header(access_token_header) { } }; // A github or gitlab host @@ -239,9 +233,9 @@ struct GitHubInputScheme : GitArchiveInputScheme std::string accessToken = settings.githubAccessToken.get(); if (accessToken != "") { auto auth_header = accessHeaderFromToken(accessToken); - return DownloadUrl(url, auth_header); + return DownloadUrl { url, auth_header }; } else { - return DownloadUrl(url); + return DownloadUrl { url }; } } @@ -294,9 +288,9 @@ struct GitLabInputScheme : GitArchiveInputScheme std::string accessToken = settings.gitlabAccessToken.get(); if (accessToken != "") { auto auth_header = accessHeaderFromToken(accessToken); - return DownloadUrl(url, auth_header); + return DownloadUrl { url, auth_header }; } else { - return DownloadUrl(url); + return DownloadUrl { url }; } } From 5999978a053b3ec16f448c40b54a1a62ceb82c90 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 29 Sep 2020 13:05:19 +0200 Subject: [PATCH 082/123] Make Headers an optional argument --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops/fetchTree.cc | 4 ++-- src/libfetchers/fetchers.hh | 8 ++++---- src/libfetchers/github.cc | 6 +++--- src/libfetchers/registry.cc | 2 +- src/libfetchers/tarball.cc | 15 ++++++++------- src/libstore/filetransfer.hh | 3 --- src/nix-channel/nix-channel.cc | 6 +++--- 9 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index d71aa22f1..10c1a6975 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -76,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s) if (isUri(s)) { return state.store->toRealPath( fetchers::downloadTarball( - state.store, resolveUri(s), Headers {}, "source", false).first.storePath); + state.store, resolveUri(s), "source", false).first.storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e879280c0..a4c84c526 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -718,7 +718,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), Headers {}, "source", false).first.storePath) }; + store, resolveUri(elem.second), "source", false).first.storePath) }; } catch (FileTransferError & e) { logWarning({ .name = "Entry download", diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 3001957b4..06e8304b8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -201,8 +201,8 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, Headers {}, name, (bool) expectedHash).first.storePath - : fetchers::downloadFile(state.store, *url, Headers{}, name, (bool) expectedHash).storePath; + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath + : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; auto path = state.store->toRealPath(storePath); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 0adc2c9f5..36d44f6e1 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -118,15 +118,15 @@ struct DownloadFileResult DownloadFileResult downloadFile( ref store, const std::string & url, - const Headers & headers, const std::string & name, - bool immutable); + bool immutable, + const Headers & headers = {}); std::pair downloadTarball( ref store, const std::string & url, - const Headers & headers, const std::string & name, - bool immutable); + bool immutable, + const Headers & headers = {}); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 3734c4278..ec99481e1 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -175,7 +175,7 @@ struct GitArchiveInputScheme : InputScheme headers.push_back(*url.access_token_header); } - auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true); + auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, headers); input.attrs.insert_or_assign("lastModified", lastModified); @@ -215,7 +215,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, headers, "source", false).storePath))); + downloadFile(store, url, "source", false, headers).storePath))); auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; @@ -271,7 +271,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, headers, "source", false).storePath))); + downloadFile(store, url, "source", false, headers).storePath))); auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 551e7684a..4367ee810 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -145,7 +145,7 @@ static std::shared_ptr getGlobalRegistry(ref store) auto path = settings.flakeRegistry.get(); if (!hasPrefix(path, "/")) { - auto storePath = downloadFile(store, path, Headers {}, "flake-registry.json", false).storePath; + auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json"); path = store->toRealPath(storePath); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index cf6d6e3d2..ca49482a9 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -12,9 +12,9 @@ namespace nix::fetchers { DownloadFileResult downloadFile( ref store, const std::string & url, - const Headers & headers, const std::string & name, - bool immutable) + bool immutable, + const Headers & headers) { // FIXME: check store @@ -38,7 +38,8 @@ DownloadFileResult downloadFile( if (cached && !cached->expired) return useCached(); - FileTransferRequest request(url, headers); + FileTransferRequest request(url); + request.headers = headers; if (cached) request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); FileTransferResult res; @@ -112,9 +113,9 @@ DownloadFileResult downloadFile( std::pair downloadTarball( ref store, const std::string & url, - const Headers & headers, const std::string & name, - bool immutable) + bool immutable, + const Headers & headers) { Attrs inAttrs({ {"type", "tarball"}, @@ -130,7 +131,7 @@ std::pair downloadTarball( getIntAttr(cached->infoAttrs, "lastModified") }; - auto res = downloadFile(store, url, headers, name, immutable); + auto res = downloadFile(store, url, name, immutable, headers); std::optional unpackedStorePath; time_t lastModified; @@ -225,7 +226,7 @@ struct TarballInputScheme : InputScheme std::pair fetch(ref store, const Input & input) override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), Headers {}, "source", false).first; + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; return {std::move(tree), input}; } }; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 7e302ff39..c89c51a21 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -66,9 +66,6 @@ struct FileTransferRequest FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } - FileTransferRequest(const std::string & uri, Headers headers) - : uri(uri), headers(headers) { } - std::string verb() { return data ? "upload" : "download"; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 760bbea86..e48f7af9a 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -94,7 +94,7 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, Headers {}, std::string(baseNameOf(url)), false); + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); auto filename = store->toRealPath(result.storePath); url = result.effectiveUrl; @@ -119,9 +119,9 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", Headers {}, "nixexprs.tar.xz", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", Headers {}, "nixexprs.tar.bz2", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } } From 64e9b3c83b7cf7f3c7348426666ccca2ca395d28 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 29 Sep 2020 23:33:16 +0200 Subject: [PATCH 083/123] nix registry list: Show 'dir' attribute Issue #4050. --- src/libexpr/flake/flakeref.cc | 6 +++--- src/libfetchers/fetchers.cc | 8 ++++++++ src/libfetchers/fetchers.hh | 2 ++ src/nix/registry.cc | 4 ++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index d5c2ffe66..87b202643 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -16,10 +16,10 @@ const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRege std::string FlakeRef::to_string() const { - auto url = input.toURL(); + std::map extraQuery; if (subdir != "") - url.query.insert_or_assign("dir", subdir); - return url.to_string(); + extraQuery.insert_or_assign("dir", subdir); + return input.toURLString(extraQuery); } fetchers::Attrs FlakeRef::toAttrs() const diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index eaa635595..49851f7bc 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -69,6 +69,14 @@ ParsedURL Input::toURL() const return scheme->toURL(*this); } +std::string Input::toURLString(const std::map & extraQuery) const +{ + auto url = toURL(); + for (auto & attr : extraQuery) + url.query.insert(attr); + return url.to_string(); +} + std::string Input::to_string() const { return toURL().to_string(); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 36d44f6e1..cc31a31b9 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -39,6 +39,8 @@ public: ParsedURL toURL() const; + std::string toURLString(const std::map & extraQuery = {}) const; + std::string to_string() const; Attrs toAttrs() const; diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 367268683..cb11ec195 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -31,8 +31,8 @@ struct CmdRegistryList : StoreCommand registry->type == Registry::User ? "user " : registry->type == Registry::System ? "system" : "global", - entry.from.to_string(), - entry.to.to_string()); + entry.from.toURLString(), + entry.to.toURLString(attrsToQuery(entry.extraAttrs))); } } } From 5e7838512e2b8de3c8fe271b8beae5ca9e1efaf9 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Tue, 29 Sep 2020 16:20:54 -0700 Subject: [PATCH 084/123] Remove github-access-token in favor of access-token. --- src/libfetchers/github.cc | 10 ---------- src/libstore/globals.hh | 3 --- 2 files changed, 13 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 142b8b87c..8286edf75 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -147,17 +147,7 @@ struct GitArchiveInputScheme : InputScheme auto tokens = settings.accessTokens.get(); auto pat = tokens.find(host); if (pat == tokens.end()) - { - if ("github.com" == host) - { - auto oldcfg = settings.githubAccessToken.get(); - if (!oldcfg.empty()) { - warn("using deprecated 'github-access-token' config value; please use 'access-tokens' instead"); - return oldcfg; - } - } return std::nullopt; - } return pat->second; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 3b8ccadf3..0f0c0fe6f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -859,9 +859,6 @@ public: are loaded as plugins (non-recursively). )"}; - Setting githubAccessToken{this, "", "github-access-token", - "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes (deprecated, please use 'access-tokens' instead)."}; - Setting accessTokens{this, {}, "access-tokens", R"( Access tokens used to access protected GitHub, GitLab, or From 45a0ed82f089158a79c8c25ef844c55e4a74fc35 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Sep 2020 00:39:06 +0000 Subject: [PATCH 085/123] Revert "Use template structs instead of phantoms" This reverts commit 9ab07e99f527d1fa3adfa02839da477a1528d64b. --- src/libstore/build.cc | 4 +- src/libstore/daemon.cc | 38 ++++---- src/libstore/derivations.cc | 4 +- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 14 +-- src/libstore/remote-store.cc | 70 +++++++------- src/libstore/worker-protocol.hh | 151 ++++++++++++++++--------------- src/nix-store/nix-store.cc | 16 ++-- 8 files changed, 153 insertions(+), 148 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 56d454b6b..928858203 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1976,7 +1976,7 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - WorkerProto::write(worker.store, hook->sink, inputPaths); + nix::worker_proto::write(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1987,7 +1987,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known->isValid()) continue; missingPaths.insert(status.known->path); } - WorkerProto::write(worker.store, hook->sink, missingPaths); + nix::worker_proto::write(worker.store, hook->sink, missingPaths); } hook->sink = FdSink(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 98fd2048d..72203d1b2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -247,7 +247,7 @@ static void writeValidPathInfo( { to << (info->deriver ? store->printStorePath(*info->deriver) : "") << info->narHash.to_string(Base16, false); - WorkerProto::write(*store, to, info->references); + nix::worker_proto::write(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate @@ -272,11 +272,11 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryValidPaths: { - auto paths = WorkerProto::read(*store, from); + auto paths = nix::worker_proto::read(*store, from, Phantom {}); logger->startWork(); auto res = store->queryValidPaths(paths); logger->stopWork(); - WorkerProto::write(*store, to, res); + nix::worker_proto::write(*store, to, res); break; } @@ -292,11 +292,11 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQuerySubstitutablePaths: { - auto paths = WorkerProto::read(*store, from); + auto paths = nix::worker_proto::read(*store, from, Phantom {}); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - WorkerProto::write(*store, to, res); + nix::worker_proto::write(*store, to, res); break; } @@ -325,7 +325,7 @@ static void performOp(TunnelLogger * logger, ref store, paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - WorkerProto::write(*store, to, paths); + nix::worker_proto::write(*store, to, paths); break; } @@ -343,7 +343,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - WorkerProto>>::write(*store, to, outputs); + nix::worker_proto::write(*store, to, outputs); break; } @@ -369,7 +369,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { auto name = readString(from); auto camStr = readString(from); - auto refs = WorkerProto::read(*store, from); + auto refs = nix::worker_proto::read(*store, from, Phantom {}); bool repairBool; from >> repairBool; auto repair = RepairFlag{repairBool}; @@ -449,7 +449,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - auto refs = WorkerProto::read(*store, from); + auto refs = nix::worker_proto::read(*store, from, Phantom {}); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -608,7 +608,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = WorkerProto::read(*store, from); + options.pathsToDelete = nix::worker_proto::read(*store, from, Phantom {}); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -677,7 +677,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - WorkerProto::write(*store, to, i->second.references); + nix::worker_proto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -688,11 +688,11 @@ static void performOp(TunnelLogger * logger, ref store, SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = WorkerProto::read(*store, from); + auto paths = nix::worker_proto::read(*store, from, Phantom {}); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = WorkerProto::read(*store, from); + pathsMap = nix::worker_proto::read(*store, from, Phantom {}); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); @@ -700,7 +700,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - WorkerProto::write(*store, to, i.second.references); + nix::worker_proto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -710,7 +710,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - WorkerProto::write(*store, to, paths); + nix::worker_proto::write(*store, to, paths); break; } @@ -782,7 +782,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::read(*store, from); + info.references = nix::worker_proto::read(*store, from, Phantom {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); @@ -835,9 +835,9 @@ static void performOp(TunnelLogger * logger, ref store, uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - WorkerProto::write(*store, to, willBuild); - WorkerProto::write(*store, to, willSubstitute); - WorkerProto::write(*store, to, unknown); + nix::worker_proto::write(*store, to, willBuild); + nix::worker_proto::write(*store, to, willSubstitute); + nix::worker_proto::write(*store, to, unknown); to << downloadSize << narSize; break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index faffe01e7..f8e7d773b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -584,7 +584,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = WorkerProto::read(store, in); + drv.inputSrcs = nix::worker_proto::read(store, in, Phantom {}); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -622,7 +622,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.output); } - WorkerProto::write(store, out, drv.inputSrcs); + nix::worker_proto::write(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index b59b34dee..40a6f3c63 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - WorkerProto::write(*this, teeSink, info->references); + nix::worker_proto::write(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - auto references = WorkerProto::read(*this, source); + auto references = nix::worker_proto::read(*this, source, Phantom {}); auto deriver = readString(source); auto narHash = hashString(htSHA256, *saved.s); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fdf3f91b9..e056859fd 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -122,7 +122,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::read(*this, conn->from); + info->references = nix::worker_proto::read(*this, conn->from, Phantom {}); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -156,7 +156,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, conn->to, info.references); + nix::worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -185,7 +185,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, conn->to, info.references); + nix::worker_proto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -301,10 +301,10 @@ public: conn->to << cmdQueryClosure << includeOutputs; - WorkerProto::write(*this, conn->to, paths); + nix::worker_proto::write(*this, conn->to, paths); conn->to.flush(); - for (auto & i : WorkerProto::read(*this, conn->from)) + for (auto & i : nix::worker_proto::read(*this, conn->from, Phantom {})) out.insert(i); } @@ -317,10 +317,10 @@ public: << cmdQueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, conn->to, paths); + nix::worker_proto::write(*this, conn->to, paths); conn->to.flush(); - return WorkerProto::read(*this, conn->from); + return nix::worker_proto::read(*this, conn->from, Phantom {}); } void connect() override diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 273466137..43853db4e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -24,61 +24,65 @@ namespace nix { -std::string WorkerProto::read(const Store & store, Source & from) +namespace worker_proto { + +std::string read(const Store & store, Source & from, Phantom _) { return readString(from); } -void WorkerProto::write(const Store & store, Sink & out, const std::string & str) +void write(const Store & store, Sink & out, const std::string & str) { out << str; } -StorePath WorkerProto::read(const Store & store, Source & from) +StorePath read(const Store & store, Source & from, Phantom _) { return store.parseStorePath(readString(from)); } -void WorkerProto::write(const Store & store, Sink & out, const StorePath & storePath) +void write(const Store & store, Sink & out, const StorePath & storePath) { out << store.printStorePath(storePath); } -ContentAddress WorkerProto::read(const Store & store, Source & from) +ContentAddress read(const Store & store, Source & from, Phantom _) { return parseContentAddress(readString(from)); } -void WorkerProto::write(const Store & store, Sink & out, const ContentAddress & ca) +void write(const Store & store, Sink & out, const ContentAddress & ca) { out << renderContentAddress(ca); } -std::optional WorkerProto>::read(const Store & store, Source & from) +std::optional read(const Store & store, Source & from, Phantom> _) { auto s = readString(from); return s == "" ? std::optional {} : store.parseStorePath(s); } -void WorkerProto>::write(const Store & store, Sink & out, const std::optional & storePathOpt) +void write(const Store & store, Sink & out, const std::optional & storePathOpt) { out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); } -std::optional WorkerProto>::read(const Store & store, Source & from) +std::optional read(const Store & store, Source & from, Phantom> _) { return parseContentAddressOpt(readString(from)); } -void WorkerProto>::write(const Store & store, Sink & out, const std::optional & caOpt) +void write(const Store & store, Sink & out, const std::optional & caOpt) { out << (caOpt ? renderContentAddress(*caOpt) : ""); } +} + /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -325,9 +329,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - WorkerProto::write(*this, conn->to, paths); + nix::worker_proto::write(*this, conn->to, paths); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom {}); } } @@ -337,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return nix::worker_proto::read(*this, conn->from, Phantom {}); } @@ -354,9 +358,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - WorkerProto::write(*this, conn->to, paths); + nix::worker_proto::write(*this, conn->to, paths); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom {}); } } @@ -378,7 +382,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::read(*this, conn->from); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -391,9 +395,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - WorkerProto::write(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); } else - WorkerProto::write(*this, conn->to, pathsMap); + worker_proto::write(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -401,7 +405,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::read(*this, conn->from); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -416,7 +420,7 @@ ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, auto narHash = Hash::parseAny(readString(conn->from), htSHA256); auto info = make_ref(path, narHash); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::read(*this, conn->from); + info->references = worker_proto::read(*this, conn->from, Phantom {}); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -460,7 +464,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : WorkerProto::read(*this, conn->from)) + for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) referrers.insert(i); } @@ -470,7 +474,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom {}); } @@ -482,7 +486,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom {}); } @@ -492,7 +496,7 @@ std::map> RemoteStore::queryPartialDerivat auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return WorkerProto>>::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom>> {}); } else { // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an @@ -537,7 +541,7 @@ ref RemoteStore::addCAToStore( << wopAddToStore << name << renderContentAddressMethod(caMethod); - WorkerProto::write(*this, conn->to, references); + worker_proto::write(*this, conn->to, references); conn->to << repair; conn.withFramedSink([&](Sink & sink) { @@ -554,7 +558,7 @@ ref RemoteStore::addCAToStore( [&](TextHashMethod thm) -> void { std::string s = dump.drain(); conn->to << wopAddTextToStore << name << s; - WorkerProto::write(*this, conn->to, references); + worker_proto::write(*this, conn->to, references); conn.processStderr(); }, [&](FixedOutputHashMethod fohm) -> void { @@ -623,7 +627,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, sink, info.references); + worker_proto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -633,7 +637,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = WorkerProto::read(*this, conn->from); + auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); assert(importedPaths.size() <= 1); } @@ -642,7 +646,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -764,7 +768,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << wopCollectGarbage << options.action; - WorkerProto::write(*this, conn->to, options.pathsToDelete); + worker_proto::write(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -826,9 +830,9 @@ void RemoteStore::queryMissing(const std::vector & targets ss.push_back(p.to_string(*this)); conn->to << ss; conn.processStderr(); - willBuild = WorkerProto::read(*this, conn->from); - willSubstitute = WorkerProto::read(*this, conn->from); - unknown = WorkerProto::read(*this, conn->from); + willBuild = worker_proto::read(*this, conn->from, Phantom {}); + willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); + unknown = worker_proto::read(*this, conn->from, Phantom {}); conn->from >> downloadSize >> narSize; return; } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index bb87cf3ec..fd6c2b2cf 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -66,96 +66,95 @@ typedef enum { class Store; struct Source; +/* To guide overloading */ template -struct WorkerProto { - static T read(const Store & store, Source & from); - static void write(const Store & store, Sink & out, const T & t); -}; +struct Phantom {}; -#define MAKE_WORKER_PROTO(T) \ - template<> \ - struct WorkerProto< T > { \ - static T read(const Store & store, Source & from); \ - static void write(const Store & store, Sink & out, const T & t); \ - } -MAKE_WORKER_PROTO(std::string); -MAKE_WORKER_PROTO(StorePath); -MAKE_WORKER_PROTO(ContentAddress); +namespace worker_proto { +/* FIXME maybe move more stuff inside here */ + +#define MAKE_WORKER_PROTO(TEMPLATE, T) \ + TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \ + TEMPLATE void write(const Store & store, Sink & out, const T & str) + +MAKE_WORKER_PROTO(, std::string); +MAKE_WORKER_PROTO(, StorePath); +MAKE_WORKER_PROTO(, ContentAddress); + +MAKE_WORKER_PROTO(template, std::set); +MAKE_WORKER_PROTO(template, std::optional); + +#define X_ template +#define Y_ std::map +MAKE_WORKER_PROTO(X_, Y_); +#undef X_ +#undef Y_ template -struct WorkerProto> { - - static std::set read(const Store & store, Source & from) - { - std::set resSet; - auto size = readNum(from); - while (size--) { - resSet.insert(WorkerProto::read(store, from)); - } - return resSet; +std::set read(const Store & store, Source & from, Phantom> _) +{ + std::set resSet; + auto size = readNum(from); + while (size--) { + resSet.insert(read(store, from, Phantom {})); } + return resSet; +} - static void write(const Store & store, Sink & out, const std::set & resSet) - { - out << resSet.size(); - for (auto & key : resSet) { - WorkerProto::write(store, out, key); - } +template +void write(const Store & store, Sink & out, const std::set & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + write(store, out, key); } - -}; +} template -struct WorkerProto> { - - static std::map read(const Store & store, Source & from) - { - std::map resMap; - auto size = readNum(from); - while (size--) { - auto k = WorkerProto::read(store, from); - auto v = WorkerProto::read(store, from); - resMap.insert_or_assign(std::move(k), std::move(v)); - } - return resMap; +std::map read(const Store & store, Source & from, Phantom> _) +{ + std::map resMap; + auto size = readNum(from); + while (size--) { + auto k = read(store, from, Phantom {}); + auto v = read(store, from, Phantom {}); + resMap.insert_or_assign(std::move(k), std::move(v)); } + return resMap; +} - static void write(const Store & store, Sink & out, const std::map & resMap) - { - out << resMap.size(); - for (auto & i : resMap) { - WorkerProto::write(store, out, i.first); - WorkerProto::write(store, out, i.second); - } +template +void write(const Store & store, Sink & out, const std::map & resMap) +{ + out << resMap.size(); + for (auto & i : resMap) { + write(store, out, i.first); + write(store, out, i.second); } - -}; +} template -struct WorkerProto> { - - static std::optional read(const Store & store, Source & from) - { - auto tag = readNum(from); - switch (tag) { - case 0: - return std::nullopt; - case 1: - return WorkerProto::read(store, from); - default: - throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag); - } +std::optional read(const Store & store, Source & from, Phantom> _) +{ + auto tag = readNum(from); + switch (tag) { + case 0: + return std::nullopt; + case 1: + return read(store, from, Phantom {}); + default: + throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag); } +} - static void write(const Store & store, Sink & out, const std::optional & optVal) - { - out << (uint64_t) (optVal ? 1 : 0); - if (optVal) - WorkerProto::write(store, out, *optVal); - } - -}; +template +void write(const Store & store, Sink & out, const std::optional & optVal) +{ + out << (uint64_t) (optVal ? 1 : 0); + if (optVal) + nix::worker_proto::write(store, out, *optVal); +} /* Specialization which uses and empty string for the empty case, taking advantage of the fact these types always serialize to non-empty strings. @@ -163,7 +162,9 @@ struct WorkerProto> { std::optional, where <= is the compatability partial order, T is one of the types below. */ -MAKE_WORKER_PROTO(std::optional); -MAKE_WORKER_PROTO(std::optional); +MAKE_WORKER_PROTO(, std::optional); +MAKE_WORKER_PROTO(, std::optional); + +} } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 141dab478..4dcdebe2f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -822,7 +822,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = WorkerProto::read(*store, in); + auto paths = nix::worker_proto::read(*store, in, Phantom {}); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -852,19 +852,19 @@ static void opServe(Strings opFlags, Strings opArgs) } } - WorkerProto::write(*store, out, store->queryValidPaths(paths)); + nix::worker_proto::write(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - auto paths = WorkerProto::read(*store, in); + auto paths = nix::worker_proto::read(*store, in, Phantom {}); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, out, info->references); + nix::worker_proto::write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -892,7 +892,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdExportPaths: { readInt(in); // obsolete - store->exportPaths(WorkerProto::read(*store, in), out); + store->exportPaths(nix::worker_proto::read(*store, in, Phantom {}), out); break; } @@ -941,9 +941,9 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(WorkerProto::read(*store, in), + store->computeFSClosure(nix::worker_proto::read(*store, in, Phantom {}), closure, false, includeOutputs); - WorkerProto::write(*store, out, closure); + nix::worker_proto::write(*store, out, closure); break; } @@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::read(*store, in); + info.references = nix::worker_proto::read(*store, in, Phantom {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); From b7597016529cebdc3c9432a101c1f8d9227713cc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Sep 2020 00:41:18 +0000 Subject: [PATCH 086/123] nix::worker_proto -> worker_proto --- src/libstore/build.cc | 4 ++-- src/libstore/daemon.cc | 38 ++++++++++++++++---------------- src/libstore/derivations.cc | 4 ++-- src/libstore/export-import.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 14 ++++++------ src/libstore/remote-store.cc | 6 ++--- src/libstore/worker-protocol.hh | 2 +- src/nix-store/nix-store.cc | 16 +++++++------- 8 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 928858203..c688acd58 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1976,7 +1976,7 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - nix::worker_proto::write(worker.store, hook->sink, inputPaths); + worker_proto::write(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1987,7 +1987,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known->isValid()) continue; missingPaths.insert(status.known->path); } - nix::worker_proto::write(worker.store, hook->sink, missingPaths); + worker_proto::write(worker.store, hook->sink, missingPaths); } hook->sink = FdSink(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 72203d1b2..0713c4853 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -247,7 +247,7 @@ static void writeValidPathInfo( { to << (info->deriver ? store->printStorePath(*info->deriver) : "") << info->narHash.to_string(Base16, false); - nix::worker_proto::write(*store, to, info->references); + worker_proto::write(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate @@ -272,11 +272,11 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryValidPaths: { - auto paths = nix::worker_proto::read(*store, from, Phantom {}); + auto paths = worker_proto::read(*store, from, Phantom {}); logger->startWork(); auto res = store->queryValidPaths(paths); logger->stopWork(); - nix::worker_proto::write(*store, to, res); + worker_proto::write(*store, to, res); break; } @@ -292,11 +292,11 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQuerySubstitutablePaths: { - auto paths = nix::worker_proto::read(*store, from, Phantom {}); + auto paths = worker_proto::read(*store, from, Phantom {}); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - nix::worker_proto::write(*store, to, res); + worker_proto::write(*store, to, res); break; } @@ -325,7 +325,7 @@ static void performOp(TunnelLogger * logger, ref store, paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - nix::worker_proto::write(*store, to, paths); + worker_proto::write(*store, to, paths); break; } @@ -343,7 +343,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - nix::worker_proto::write(*store, to, outputs); + worker_proto::write(*store, to, outputs); break; } @@ -369,7 +369,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { auto name = readString(from); auto camStr = readString(from); - auto refs = nix::worker_proto::read(*store, from, Phantom {}); + auto refs = worker_proto::read(*store, from, Phantom {}); bool repairBool; from >> repairBool; auto repair = RepairFlag{repairBool}; @@ -449,7 +449,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - auto refs = nix::worker_proto::read(*store, from, Phantom {}); + auto refs = worker_proto::read(*store, from, Phantom {}); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -608,7 +608,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = nix::worker_proto::read(*store, from, Phantom {}); + options.pathsToDelete = worker_proto::read(*store, from, Phantom {}); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -677,7 +677,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - nix::worker_proto::write(*store, to, i->second.references); + worker_proto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -688,11 +688,11 @@ static void performOp(TunnelLogger * logger, ref store, SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = nix::worker_proto::read(*store, from, Phantom {}); + auto paths = worker_proto::read(*store, from, Phantom {}); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = nix::worker_proto::read(*store, from, Phantom {}); + pathsMap = worker_proto::read(*store, from, Phantom {}); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); @@ -700,7 +700,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - nix::worker_proto::write(*store, to, i.second.references); + worker_proto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -710,7 +710,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - nix::worker_proto::write(*store, to, paths); + worker_proto::write(*store, to, paths); break; } @@ -782,7 +782,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = nix::worker_proto::read(*store, from, Phantom {}); + info.references = worker_proto::read(*store, from, Phantom {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); @@ -835,9 +835,9 @@ static void performOp(TunnelLogger * logger, ref store, uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - nix::worker_proto::write(*store, to, willBuild); - nix::worker_proto::write(*store, to, willSubstitute); - nix::worker_proto::write(*store, to, unknown); + worker_proto::write(*store, to, willBuild); + worker_proto::write(*store, to, willSubstitute); + worker_proto::write(*store, to, unknown); to << downloadSize << narSize; break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f8e7d773b..7ffc94818 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -584,7 +584,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = nix::worker_proto::read(store, in, Phantom {}); + drv.inputSrcs = worker_proto::read(store, in, Phantom {}); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -622,7 +622,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.output); } - nix::worker_proto::write(store, out, drv.inputSrcs); + worker_proto::write(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 40a6f3c63..02c839520 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - nix::worker_proto::write(*this, teeSink, info->references); + worker_proto::write(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - auto references = nix::worker_proto::read(*this, source, Phantom {}); + auto references = worker_proto::read(*this, source, Phantom {}); auto deriver = readString(source); auto narHash = hashString(htSHA256, *saved.s); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index e056859fd..6db44047d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -122,7 +122,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = nix::worker_proto::read(*this, conn->from, Phantom {}); + info->references = worker_proto::read(*this, conn->from, Phantom {}); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -156,7 +156,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - nix::worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -185,7 +185,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig conn->to << exportMagic << printStorePath(info.path); - nix::worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -301,10 +301,10 @@ public: conn->to << cmdQueryClosure << includeOutputs; - nix::worker_proto::write(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn->to.flush(); - for (auto & i : nix::worker_proto::read(*this, conn->from, Phantom {})) + for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) out.insert(i); } @@ -317,10 +317,10 @@ public: << cmdQueryValidPaths << false // lock << maybeSubstitute; - nix::worker_proto::write(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn->to.flush(); - return nix::worker_proto::read(*this, conn->from, Phantom {}); + return worker_proto::read(*this, conn->from, Phantom {}); } void connect() override diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 43853db4e..049d4e954 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -329,7 +329,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - nix::worker_proto::write(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); } @@ -341,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return nix::worker_proto::read(*this, conn->from, Phantom {}); + return worker_proto::read(*this, conn->from, Phantom {}); } @@ -358,7 +358,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - nix::worker_proto::write(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index fd6c2b2cf..2934c1d67 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -153,7 +153,7 @@ void write(const Store & store, Sink & out, const std::optional & optVal) { out << (uint64_t) (optVal ? 1 : 0); if (optVal) - nix::worker_proto::write(store, out, *optVal); + worker_proto::write(store, out, *optVal); } /* Specialization which uses and empty string for the empty case, taking diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4dcdebe2f..da91caef1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -822,7 +822,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = nix::worker_proto::read(*store, in, Phantom {}); + auto paths = worker_proto::read(*store, in, Phantom {}); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -852,19 +852,19 @@ static void opServe(Strings opFlags, Strings opArgs) } } - nix::worker_proto::write(*store, out, store->queryValidPaths(paths)); + worker_proto::write(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - auto paths = nix::worker_proto::read(*store, in, Phantom {}); + auto paths = worker_proto::read(*store, in, Phantom {}); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - nix::worker_proto::write(*store, out, info->references); + worker_proto::write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -892,7 +892,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdExportPaths: { readInt(in); // obsolete - store->exportPaths(nix::worker_proto::read(*store, in, Phantom {}), out); + store->exportPaths(worker_proto::read(*store, in, Phantom {}), out); break; } @@ -941,9 +941,9 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(nix::worker_proto::read(*store, in, Phantom {}), + store->computeFSClosure(worker_proto::read(*store, in, Phantom {}), closure, false, includeOutputs); - nix::worker_proto::write(*store, out, closure); + worker_proto::write(*store, out, closure); break; } @@ -958,7 +958,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = nix::worker_proto::read(*store, in, Phantom {}); + info.references = worker_proto::read(*store, in, Phantom {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); From 274357eb6acd9a0812872f32dd487354d51f0f13 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Sep 2020 12:09:18 +0200 Subject: [PATCH 087/123] Simplify --- src/libfetchers/github.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8286edf75..42eb346a5 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -15,9 +15,6 @@ struct DownloadUrl { std::string url; Headers headers; - - DownloadUrl(const std::string & url, const Headers & headers) - : url(url), headers(headers) { } }; // A github or gitlab host @@ -254,7 +251,7 @@ struct GitHubInputScheme : GitArchiveInputScheme input.getRev()->to_string(Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); - return DownloadUrl(url, headers); + return DownloadUrl { url, headers }; } void clone(const Input & input, const Path & destDir) override @@ -320,7 +317,7 @@ struct GitLabInputScheme : GitArchiveInputScheme input.getRev()->to_string(Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); - return DownloadUrl(url, headers); + return DownloadUrl { url, headers }; } void clone(const Input & input, const Path & destDir) override From 20a1e20d9194527d725898c745d1243d3de16277 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Sep 2020 12:11:22 +0200 Subject: [PATCH 088/123] Style --- src/libfetchers/github.cc | 19 +++++++++++-------- src/libstore/globals.hh | 8 ++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 42eb346a5..8610fe447 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -140,15 +140,16 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional getAccessToken(const std::string &host) const { + std::optional getAccessToken(const std::string & host) const + { auto tokens = settings.accessTokens.get(); - auto pat = tokens.find(host); - if (pat == tokens.end()) - return std::nullopt; - return pat->second; + if (auto token = get(tokens, host)) + return *token; + return {}; } - Headers makeHeadersWithAuthTokens(const std::string & host) const { + Headers makeHeadersWithAuthTokens(const std::string & host) const + { Headers headers; auto accessToken = getAccessToken(host); if (accessToken) { @@ -214,7 +215,8 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string type() override { return "github"; } - std::optional > accessHeaderFromToken(const std::string & token) const { + std::optional > accessHeaderFromToken(const std::string & token) const + { // Github supports PAT/OAuth2 tokens and HTTP Basic // Authentication. The former simply specifies the token, the // latter can use the token as the password. Only the first @@ -268,7 +270,8 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string type() override { return "gitlab"; } - std::optional > accessHeaderFromToken(const std::string & token) const { + std::optional > accessHeaderFromToken(const std::string & token) const + { // Gitlab supports 4 kinds of authorization, two of which are // relevant here: OAuth2 and PAT (Private Access Token). The // user can indicate which token is used by specifying the diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0f0c0fe6f..8c63c5b34 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -893,10 +893,10 @@ public: ```nix input.foo = { - type="gitlab"; - host="gitlab.mycompany.com"; - owner="mycompany"; - repo="pro"; + type = "gitlab"; + host = "gitlab.mycompany.com"; + owner = "mycompany"; + repo = "pro"; }; ``` From 924712eef1fe86d349635ba666d413632e62519c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Sep 2020 17:48:49 +0200 Subject: [PATCH 089/123] Installer: Set a known umask Fixes #1560, #2377. --- scripts/install-nix-from-closure.sh | 2 ++ scripts/install.in | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 6efd8af18..14fb91534 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -2,6 +2,8 @@ set -e +umask 0022 + dest="/nix" self="$(dirname "$0")" nix="@nix@" diff --git a/scripts/install.in b/scripts/install.in index 1d26c4ff0..39fae37e3 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -10,6 +10,8 @@ oops() { exit 1 } +umask 0022 + tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \ oops "Can't create temporary directory for downloading the Nix binary tarball")" cleanup() { From f3280004e2be3f7533190ddf71ee1ec7c0d7d60d Mon Sep 17 00:00:00 2001 From: DavHau Date: Thu, 1 Oct 2020 11:34:13 +0700 Subject: [PATCH 090/123] add more examples to --help of `nix run` --- src/nix/run.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/nix/run.cc b/src/nix/run.cc index cbaba9d90..e6584346e 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -167,6 +167,14 @@ struct CmdRun : InstallableCommand, RunCommon "To run Blender:", "nix run blender-bin" }, + Example{ + "To run vim from nixpkgs:", + "nix run nixpkgs#vim" + }, + Example{ + "To run vim from nixpkgs with arguments:", + "nix run nixpkgs#vim -- --help" + }, }; } From d5d196b0a15fdba50363b89a61eb894a49153b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Fri, 2 Oct 2020 12:10:31 +0200 Subject: [PATCH 091/123] Fix profile update in nix command --- src/nix/command.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/command.cc b/src/nix/command.cc index 37a4bc785..ba7de9fdd 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -152,7 +152,7 @@ void MixProfile::updateProfile(const Buildables & buildables) for (auto & output : bfd.outputs) { /* Output path should be known because we just tried to build it. */ - assert(!output.second); + assert(output.second); result.push_back(*output.second); } }, From 4abd5554ad0be209b103378a26fa8784aee3a0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Fri, 2 Oct 2020 15:40:55 +0200 Subject: [PATCH 092/123] Fix macOS sandbox build --- doc/manual/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 3b8e7e2df..d308c7cf6 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -40,7 +40,7 @@ $(d)/nix.json: $(bindir)/nix @mv $@.tmp $@ $(d)/conf-file.json: $(bindir)/nix - $(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp + $(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix From 88a667e49e10af4a9e2daa51badbed63ad19d817 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Oct 2020 17:53:30 +0200 Subject: [PATCH 093/123] Fix s3:// store Fixes https://github.com/NixOS/nixos-org-configurations/issues/123. --- src/libstore/http-binary-cache-store.cc | 1 + src/libstore/store-api.hh | 1 + src/libutil/url-parts.hh | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 86be7c006..3d3d91e5e 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -73,6 +73,7 @@ public: if (forceHttp) ret.insert("file"); return ret; } + protected: void maybeDisable() diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ba202375b..854446987 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -830,6 +830,7 @@ struct StoreFactory std::function (const std::string & scheme, const std::string & uri, const Store::Params & params)> create; std::function ()> getConfig; }; + struct Implementations { static std::vector * registered; diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 64e06cfbc..68be15cb0 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -7,7 +7,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z+.-]+)"; +const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; From d0bb544128140d51b5a5a17d36eeb9fd13838586 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 10:18:44 +0200 Subject: [PATCH 094/123] Add missing #pragma once --- src/libexpr/primops.hh | 2 ++ src/libstore/daemon.hh | 2 ++ src/libstore/parsed-derivations.hh | 2 ++ src/libutil/topo-sort.hh | 2 ++ 4 files changed, 8 insertions(+) diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index ed5e2ea58..9d42d6539 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -1,3 +1,5 @@ +#pragma once + #include "eval.hh" #include diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 841ace316..67755d54e 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,3 +1,5 @@ +#pragma once + #include "serialise.hh" #include "store-api.hh" diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 3fa09f34f..c9fbe68c4 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,3 +1,5 @@ +#pragma once + #include "store-api.hh" #include diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh index 7a68ff169..7418be5e0 100644 --- a/src/libutil/topo-sort.hh +++ b/src/libutil/topo-sort.hh @@ -1,3 +1,5 @@ +#pragma once + #include "error.hh" namespace nix { From 6691256e790b2917ccd06f16472b667426d0f413 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 10:40:49 +0200 Subject: [PATCH 095/123] Factor out common showBytes() --- src/libmain/shared.cc | 12 +++--------- src/libstore/optimise-store.cc | 12 +++--------- src/libutil/util.cc | 7 +++++++ src/libutil/util.hh | 3 +++ 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 22ae51e47..2247aeca4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -386,18 +386,12 @@ RunPager::~RunPager() } -string showBytes(uint64_t bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); -} - - PrintFreed::~PrintFreed() { if (show) - std::cout << format("%1% store paths deleted, %2% freed\n") - % results.paths.size() - % showBytes(results.bytesFreed); + std::cout << fmt("%d store paths deleted, %s freed\n", + results.paths.size(), + showBytes(results.bytesFreed)); } Exit::~Exit() { } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index c032a5e22..a0d482ddf 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -276,21 +276,15 @@ void LocalStore::optimiseStore(OptimiseStats & stats) } } -static string showBytes(uint64_t bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); -} - void LocalStore::optimiseStore() { OptimiseStats stats; optimiseStore(stats); - printInfo( - format("%1% freed by hard-linking %2% files") - % showBytes(stats.bytesFreed) - % stats.filesLinked); + printInfo("%s freed by hard-linking %d files", + showBytes(stats.bytesFreed), + stats.filesLinked); } void LocalStore::optimisePath(const Path & path) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9e7142e01..f07e99885 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1641,4 +1641,11 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) return fdSocket; } + +string showBytes(uint64_t bytes) +{ + return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); +} + + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b8e201203..129d59a97 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -573,4 +573,7 @@ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; +std::string showBytes(uint64_t bytes); + + } From 636455c471bfde89dbdf33e10308962b79d50f07 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 11:16:32 +0200 Subject: [PATCH 096/123] Remove 'using namespace fetchers' --- src/libexpr/flake/flake.cc | 2 +- src/libexpr/flake/lockfile.cc | 4 ++-- src/libexpr/flake/lockfile.hh | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index b4ede542c..bae4d65e5 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -12,7 +12,7 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +typedef std::pair FetchedFlake; typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 78431f000..bb46e1bb4 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -13,12 +13,12 @@ FlakeRef getFlakeRef( { auto i = json.find(attr); if (i != json.end()) { - auto attrs = jsonToAttrs(*i); + auto attrs = fetchers::jsonToAttrs(*i); // FIXME: remove when we drop support for version 5. if (info) { auto j = json.find(info); if (j != json.end()) { - for (auto k : jsonToAttrs(*j)) + for (auto k : fetchers::jsonToAttrs(*j)) attrs.insert_or_assign(k.first, k.second); } } diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 9ec8b39c3..627794d8c 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -11,8 +11,6 @@ class StorePath; namespace nix::flake { -using namespace fetchers; - typedef std::vector InputPath; struct LockedNode; From b4db315a56860b10a7d19339f2310bc1149219cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 13:26:00 +0200 Subject: [PATCH 097/123] mk/precompiled-headers.mk: Fix clang test "clang++" includes the string "g++" so this test didn't work properly. However the separate handling of clang might not be needed anymore... --- mk/precompiled-headers.mk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 1fdb4b3a4..bfee35a2c 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -21,18 +21,18 @@ clean-files += $(GCH) $(PCH) ifeq ($(PRECOMPILE_HEADERS), 1) - ifeq ($(findstring g++,$(CXX)), g++) - - GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch - - GLOBAL_ORDER_AFTER += $(GCH) - - else ifeq ($(findstring clang++,$(CXX)), clang++) + ifeq ($(findstring clang++,$(CXX)), clang++) GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch GLOBAL_ORDER_AFTER += $(PCH) + else ifeq ($(findstring g++,$(CXX)), g++) + + GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch + + GLOBAL_ORDER_AFTER += $(GCH) + else $(error Don't know how to precompile headers on $(CXX)) From 0856c0a0b4ff9b72b4e786e700785bb331298582 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 13:27:09 +0200 Subject: [PATCH 098/123] mk/precompiled-headers.mk: Remove special handling for clang --- .gitignore | 1 - local.mk | 2 +- mk/precompiled-headers.mk | 27 +++------------------------ 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 4711fa7fa..b087cd8d5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ perl/Makefile.config /aclocal.m4 /autom4te.cache /precompiled-headers.h.gch -/precompiled-headers.h.pch /config.* /configure /stamp-h1 diff --git a/local.mk b/local.mk index b1ca832a6..fe314b902 100644 --- a/local.mk +++ b/local.mk @@ -11,6 +11,6 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations $(foreach i, config.h $(wildcard src/lib*/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) -$(GCH) $(PCH): src/libutil/util.hh config.h +$(GCH): src/libutil/util.hh config.h GCH_CXXFLAGS = -I src/libutil diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index bfee35a2c..cdd3daecd 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -10,33 +10,12 @@ $(GCH): precompiled-headers.h @mkdir -p "$(dir $@)" $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) -PCH = $(buildprefix)precompiled-headers.h.pch - -$(PCH): precompiled-headers.h - @rm -f $@ - @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) - -clean-files += $(GCH) $(PCH) +clean-files += $(GCH) ifeq ($(PRECOMPILE_HEADERS), 1) - ifeq ($(findstring clang++,$(CXX)), clang++) + GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch - GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch - - GLOBAL_ORDER_AFTER += $(PCH) - - else ifeq ($(findstring g++,$(CXX)), g++) - - GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch - - GLOBAL_ORDER_AFTER += $(GCH) - - else - - $(error Don't know how to precompile headers on $(CXX)) - - endif + GLOBAL_ORDER_AFTER += $(GCH) endif From 0419cd26954d9d7c5b5ace96501b41cc81d66eed Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 13:34:58 +0200 Subject: [PATCH 099/123] Remove unneeded -lboost_* flags --- src/nix/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/local.mk b/src/nix/local.mk index ab4e9121b..f37b73384 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -19,7 +19,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr nix_LIBS = libexpr libmain libfetchers libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -llowdown +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ From 85c8be6286e8f5464813a8a29b0b78d892498745 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 13:36:55 +0200 Subject: [PATCH 100/123] Remove static variable name clashes This was useful for an experiment with building Nix as a single compilation unit. It's not very useful otherwise but also doesn't hurt... --- src/build-remote/build-remote.cc | 4 ++-- src/libexpr/eval.cc | 2 +- src/libexpr/primops/context.cc | 10 +++++----- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libexpr/primops/fromTOML.cc | 2 +- src/libfetchers/git.cc | 2 +- src/libfetchers/github.cc | 4 ++-- src/libfetchers/indirect.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/path.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/dummy-store.cc | 2 +- src/libstore/filetransfer.cc | 2 +- src/libstore/globals.cc | 2 +- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/ssh-store.cc | 2 +- src/libutil/archive.cc | 2 +- src/libutil/args.cc | 6 +++--- src/libutil/logging.cc | 2 +- src/nix-build/nix-build.cc | 6 +++--- src/nix-channel/nix-channel.cc | 4 ++-- src/nix-collect-garbage/nix-collect-garbage.cc | 4 ++-- src/nix-copy-closure/nix-copy-closure.cc | 4 ++-- src/nix-daemon/nix-daemon.cc | 4 ++-- src/nix-env/nix-env.cc | 4 ++-- src/nix-instantiate/nix-instantiate.cc | 4 ++-- src/nix-prefetch-url/nix-prefetch-url.cc | 4 ++-- src/nix-store/nix-store.cc | 9 +++++++-- src/nix/add-to-store.cc | 2 +- src/nix/build.cc | 2 +- src/nix/cat.cc | 4 ++-- src/nix/copy.cc | 2 +- src/nix/describe-stores.cc | 2 +- src/nix/develop.cc | 4 ++-- src/nix/diff-closures.cc | 2 +- src/nix/doctor.cc | 2 +- src/nix/dump-path.cc | 2 +- src/nix/edit.cc | 2 +- src/nix/eval.cc | 2 +- src/nix/flake.cc | 2 +- src/nix/hash.cc | 14 +++++++------- src/nix/log.cc | 2 +- src/nix/ls.cc | 4 ++-- src/nix/make-content-addressable.cc | 2 +- src/nix/optimise-store.cc | 2 +- src/nix/path-info.cc | 2 +- src/nix/ping-store.cc | 2 +- src/nix/profile.cc | 2 +- src/nix/registry.cc | 2 +- src/nix/repl.cc | 2 +- src/nix/run.cc | 4 ++-- src/nix/search.cc | 2 +- src/nix/show-config.cc | 2 +- src/nix/show-derivation.cc | 2 +- src/nix/sigs.cc | 4 ++-- src/nix/upgrade-nix.cc | 2 +- src/nix/verify.cc | 2 +- src/nix/why-depends.cc | 2 +- 63 files changed, 97 insertions(+), 92 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index ce5127113..9a0e9f08d 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -44,7 +44,7 @@ static bool allSupportedLocally(Store & store, const std::set& requ return true; } -static int _main(int argc, char * * argv) +static int main_build_remote(int argc, char * * argv) { { logger = makeJSONLogger(*logger); @@ -297,4 +297,4 @@ connected: } } -static RegisterLegacyCommand s1("build-remote", _main); +static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 883fc27a7..d6366050c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2081,7 +2081,7 @@ Strings EvalSettings::getDefaultNixPath() EvalSettings evalSettings; -static GlobalConfig::Register r1(&evalSettings); +static GlobalConfig::Register rEvalSettings(&evalSettings); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index dbb93bae6..b570fca31 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, mkString(v, s, PathSet()); } -static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -21,7 +21,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, mkBool(v, !context.empty()); } -static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a @@ -42,7 +42,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po mkString(v, s, context2); } -static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); /* Extract the context of a string as a structured Nix value. @@ -127,7 +127,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, v.attrs->sort(); } -static RegisterPrimOp r4("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); /* Append the given context to a given string. @@ -191,6 +191,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg mkString(v, orig, context); } -static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 1a064ed5c..a77035c16 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -87,6 +87,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar state.allowedPaths->insert(tree.actualPath); } -static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 06e8304b8..7cd4d0fbf 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -152,7 +152,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V fetchTree(state, pos, args, v, std::nullopt); } -static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); +static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, std::string name) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index b00827a4b..77bff44ae 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -88,6 +88,6 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va } } -static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ad7638d73..a6411b02b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -452,6 +452,6 @@ struct GitInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8610fe447..90893075e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -334,7 +334,7 @@ struct GitLabInputScheme : GitArchiveInputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); -static auto r2 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 74332ae3d..10e59919a 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -100,6 +100,6 @@ struct IndirectInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index d80c2ea7a..7d3d52751 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -293,6 +293,6 @@ struct MercurialInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 99d4b4e8f..bcb904c0d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -102,6 +102,6 @@ struct PathInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ca49482a9..8c0f20475 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -231,6 +231,6 @@ struct TarballInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 49641c2ac..736be8413 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -63,6 +63,6 @@ struct DummyStore : public Store, public virtual DummyStoreConfig { unsupported("buildDerivation"); } }; -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regDummyStore; } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index cd619672f..c2c65af05 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -31,7 +31,7 @@ namespace nix { FileTransferSettings fileTransferSettings; -static GlobalConfig::Register r1(&fileTransferSettings); +static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); std::string resolveUri(const std::string & uri) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index c5734852d..1238dc530 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -25,7 +25,7 @@ namespace nix { Settings settings; -static GlobalConfig::Register r1(&settings); +static GlobalConfig::Register rSettings(&settings); Settings::Settings() : nixPrefix(NIX_PREFIX) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3d3d91e5e..9d2a89f96 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -181,6 +181,6 @@ protected: }; -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regHttpBinaryCacheStore; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 6db44047d..467169ce8 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -335,6 +335,6 @@ public: } }; -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b5744448e..7d979c5c2 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -105,6 +105,6 @@ std::set LocalBinaryCacheStore::uriSchemes() return {"file"}; } -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 049d4e954..40ef72293 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1008,6 +1008,6 @@ void ConnectionHandle::withFramedSink(std::function fun) } -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d43f267e0..552c4aac7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -439,7 +439,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache }; -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 6d6eca98d..08d0bd565 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -83,6 +83,6 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regSSHStore; } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6ad9fa238..f1479329f 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -33,7 +33,7 @@ struct ArchiveSettings : Config static ArchiveSettings archiveSettings; -static GlobalConfig::Register r1(&archiveSettings); +static GlobalConfig::Register rArchiveSettings(&archiveSettings); const std::string narVersionMagic1 = "nix-archive-1"; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 147602415..2760b830b 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -277,7 +277,7 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< }; } -static void completePath(std::string_view prefix, bool onlyDirs) +static void _completePath(std::string_view prefix, bool onlyDirs) { pathCompletions = true; glob_t globbuf; @@ -300,12 +300,12 @@ static void completePath(std::string_view prefix, bool onlyDirs) void completePath(size_t, std::string_view prefix) { - completePath(prefix, false); + _completePath(prefix, false); } void completeDir(size_t, std::string_view prefix) { - completePath(prefix, true); + _completePath(prefix, true); } Strings argvToStrings(int argc, char * * argv) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index cbbf64395..8a6752e22 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -10,7 +10,7 @@ namespace nix { LoggerSettings loggerSettings; -static GlobalConfig::Register r1(&loggerSettings); +static GlobalConfig::Register rLoggerSettings(&loggerSettings); static thread_local ActivityId curActivity = 0; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 3a8d67f21..a79b1086b 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -67,7 +67,7 @@ std::vector shellwords(const string & s) return res; } -static void _main(int argc, char * * argv) +static void main_nix_build(int argc, char * * argv) { auto dryRun = false; auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); @@ -540,5 +540,5 @@ static void _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-build", _main); -static RegisterLegacyCommand s2("nix-shell", _main); +static RegisterLegacyCommand r_nix_build("nix-build", main_nix_build); +static RegisterLegacyCommand r_nix_shell("nix-shell", main_nix_build); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index e48f7af9a..309970df6 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -153,7 +153,7 @@ static void update(const StringSet & channelNames) replaceSymlink(profile, channelLink); } -static int _main(int argc, char ** argv) +static int main_nix_channel(int argc, char ** argv) { { // Figure out the name of the `.nix-channels' file to use @@ -250,4 +250,4 @@ static int _main(int argc, char ** argv) } } -static RegisterLegacyCommand s1("nix-channel", _main); +static RegisterLegacyCommand r_nix_channel("nix-channel", main_nix_channel); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index bcf1d8518..57092b887 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -49,7 +49,7 @@ void removeOldGenerations(std::string dir) } } -static int _main(int argc, char * * argv) +static int main_nix_collect_garbage(int argc, char * * argv) { { bool removeOld = false; @@ -92,4 +92,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-collect-garbage", _main); +static RegisterLegacyCommand r_nix_collect_garbage("nix-collect-garbage", main_nix_collect_garbage); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index b10184718..10990f7b5 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -4,7 +4,7 @@ using namespace nix; -static int _main(int argc, char ** argv) +static int main_nix_copy_closure(int argc, char ** argv) { { auto gzip = false; @@ -65,4 +65,4 @@ static int _main(int argc, char ** argv) } } -static RegisterLegacyCommand s1("nix-copy-closure", _main); +static RegisterLegacyCommand r_nix_copy_closure("nix-copy-closure", main_nix_copy_closure); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 6e652ccbf..fc6195cf0 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -265,7 +265,7 @@ static void daemonLoop(char * * argv) } -static int _main(int argc, char * * argv) +static int main_nix_daemon(int argc, char * * argv) { { auto stdio = false; @@ -330,4 +330,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-daemon", _main); +static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 3e7c453fb..e6667e7f5 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1330,7 +1330,7 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) } -static int _main(int argc, char * * argv) +static int main_nix_env(int argc, char * * argv) { { Strings opFlags, opArgs; @@ -1460,4 +1460,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-env", _main); +static RegisterLegacyCommand r_nix_env("nix-env", main_nix_env); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 539092cbe..18a0049a6 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -83,7 +83,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, } -static int _main(int argc, char * * argv) +static int main_nix_instantiate(int argc, char * * argv) { { Strings files; @@ -192,4 +192,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-instantiate", _main); +static RegisterLegacyCommand r_nix_instantiate("nix-instantiate", main_nix_instantiate); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 377ae03a8..3bdee55a7 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -48,7 +48,7 @@ string resolveMirrorUri(EvalState & state, string uri) } -static int _main(int argc, char * * argv) +static int main_nix_prefetch_url(int argc, char * * argv) { { HashType ht = htSHA256; @@ -229,4 +229,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-prefetch-url", _main); +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index da91caef1..14baabc36 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -24,6 +24,9 @@ #endif +namespace nix_store { + + using namespace nix; using std::cin; using std::cout; @@ -1025,7 +1028,7 @@ static void opVersion(Strings opFlags, Strings opArgs) /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ -static int _main(int argc, char * * argv) +static int main_nix_store(int argc, char * * argv) { { Strings opFlags, opArgs; @@ -1121,4 +1124,6 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-store", _main); +static RegisterLegacyCommand r_nix_store("nix-store", main_nix_store); + +} diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 023ffa4ed..7fe87d757 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -87,4 +87,4 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; -static auto r1 = registerCommand("add-to-store"); +static auto rCmdAddToStore = registerCommand("add-to-store"); diff --git a/src/nix/build.cc b/src/nix/build.cc index 4605eb13e..d85a482db 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -88,4 +88,4 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile } }; -static auto r1 = registerCommand("build"); +static auto rCmdBuild = registerCommand("build"); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 97306107c..eef172cfc 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -72,5 +72,5 @@ struct CmdCatNar : StoreCommand, MixCat } }; -static auto r1 = registerCommand("cat-store"); -static auto r2 = registerCommand("cat-nar"); +static auto rCmdCatStore = registerCommand("cat-store"); +static auto rCmdCatNar = registerCommand("cat-nar"); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 811c8ab34..cb31aac8f 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -106,4 +106,4 @@ struct CmdCopy : StorePathsCommand } }; -static auto r1 = registerCommand("copy"); +static auto rCmdCopy = registerCommand("copy"); diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc index 0cc2d9337..1dd384c0e 100644 --- a/src/nix/describe-stores.cc +++ b/src/nix/describe-stores.cc @@ -41,4 +41,4 @@ struct CmdDescribeStores : Command, MixJSON } }; -static auto r1 = registerCommand("describe-stores"); +static auto rDescribeStore = registerCommand("describe-stores"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index f29fa71d2..a46ea39b6 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -443,5 +443,5 @@ struct CmdPrintDevEnv : Common } }; -static auto r1 = registerCommand("print-dev-env"); -static auto r2 = registerCommand("develop"); +static auto rCmdPrintDevEnv = registerCommand("print-dev-env"); +static auto rCmdDevelop = registerCommand("develop"); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 0dc99d05e..30e7b20e1 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -143,4 +143,4 @@ struct CmdDiffClosures : SourceExprCommand } }; -static auto r1 = registerCommand("diff-closures"); +static auto rCmdDiffClosures = registerCommand("diff-closures"); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 683e91446..4588ac05e 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -131,4 +131,4 @@ struct CmdDoctor : StoreCommand } }; -static auto r1 = registerCommand("doctor"); +static auto rCmdDoctor = registerCommand("doctor"); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index e1de71bf8..6fd197531 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -30,4 +30,4 @@ struct CmdDumpPath : StorePathCommand } }; -static auto r1 = registerCommand("dump-path"); +static auto rDumpPath = registerCommand("dump-path"); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 378a3739c..51c16f5a9 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -54,4 +54,4 @@ struct CmdEdit : InstallableCommand } }; -static auto r1 = registerCommand("edit"); +static auto rCmdEdit = registerCommand("edit"); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index a8ca446be..43ce46546 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -90,4 +90,4 @@ struct CmdEval : MixJSON, InstallableCommand } }; -static auto r1 = registerCommand("eval"); +static auto rCmdEval = registerCommand("eval"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ae6f4c5f9..d45f13029 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -965,4 +965,4 @@ struct CmdFlake : NixMultiCommand } }; -static auto r1 = registerCommand("flake"); +static auto rCmdFlake = registerCommand("flake"); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 494f00a20..1d23bb0e2 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -79,8 +79,8 @@ struct CmdHash : Command } }; -static RegisterCommand r1("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); -static RegisterCommand r2("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); +static RegisterCommand rCmdHashFile("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); +static RegisterCommand rCmdHashPath("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); struct CmdToBase : Command { @@ -112,10 +112,10 @@ struct CmdToBase : Command } }; -static RegisterCommand r3("to-base16", [](){ return make_ref(Base16); }); -static RegisterCommand r4("to-base32", [](){ return make_ref(Base32); }); -static RegisterCommand r5("to-base64", [](){ return make_ref(Base64); }); -static RegisterCommand r6("to-sri", [](){ return make_ref(SRI); }); +static RegisterCommand rCmdToBase16("to-base16", [](){ return make_ref(Base16); }); +static RegisterCommand rCmdToBase32("to-base32", [](){ return make_ref(Base32); }); +static RegisterCommand rCmdToBase64("to-base64", [](){ return make_ref(Base64); }); +static RegisterCommand rCmdToSRI("to-sri", [](){ return make_ref(SRI); }); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) @@ -167,4 +167,4 @@ static int compatNixHash(int argc, char * * argv) return 0; } -static RegisterLegacyCommand s1("nix-hash", compatNixHash); +static RegisterLegacyCommand r_nix_hash("nix-hash", compatNixHash); diff --git a/src/nix/log.cc b/src/nix/log.cc index 33380dcf5..33a3053f5 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -64,4 +64,4 @@ struct CmdLog : InstallableCommand } }; -static auto r1 = registerCommand("log"); +static auto rCmdLog = registerCommand("log"); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 76c8bc9a3..baca54431 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -152,5 +152,5 @@ struct CmdLsNar : Command, MixLs } }; -static auto r1 = registerCommand("ls-store"); -static auto r2 = registerCommand("ls-nar"); +static auto rCmdLsStore = registerCommand("ls-store"); +static auto rCmdLsNar = registerCommand("ls-nar"); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 38b60fc38..df3ec5194 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -108,4 +108,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand("make-content-addressable"); +static auto rCmdMakeContentAddressable = registerCommand("make-content-addressable"); diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index b45951879..51a7a9756 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -31,4 +31,4 @@ struct CmdOptimiseStore : StoreCommand } }; -static auto r1 = registerCommand("optimise-store"); +static auto rCmdOptimiseStore = registerCommand("optimise-store"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 0c12efaf0..63cf885f9 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -127,4 +127,4 @@ struct CmdPathInfo : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand("path-info"); +static auto rCmdPathInfo = registerCommand("path-info"); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 127397a29..8db78d591 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -29,4 +29,4 @@ struct CmdPingStore : StoreCommand } }; -static auto r1 = registerCommand("ping-store"); +static auto rCmdPingStore = registerCommand("ping-store"); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 7ce4dfe4c..01aef2f9b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -467,4 +467,4 @@ struct CmdProfile : NixMultiCommand } }; -static auto r1 = registerCommand("profile"); +static auto rCmdProfile = registerCommand("profile"); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index cb11ec195..8e8983ad0 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -143,4 +143,4 @@ struct CmdRegistry : virtual NixMultiCommand } }; -static auto r1 = registerCommand("registry"); +static auto rCmdRegistry = registerCommand("registry"); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 329999475..9ff386b1d 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -825,6 +825,6 @@ struct CmdRepl : StoreCommand, MixEvalArgs } }; -static auto r1 = registerCommand("repl"); +static auto rCmdRepl = registerCommand("repl"); } diff --git a/src/nix/run.cc b/src/nix/run.cc index e6584346e..790784382 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -140,7 +140,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment } }; -static auto r1 = registerCommand("shell"); +static auto rCmdShell = registerCommand("shell"); struct CmdRun : InstallableCommand, RunCommon { @@ -209,7 +209,7 @@ struct CmdRun : InstallableCommand, RunCommon } }; -static auto r2 = registerCommand("run"); +static auto rCmdRun = registerCommand("run"); void chrootHelper(int argc, char * * argv) { diff --git a/src/nix/search.cc b/src/nix/search.cc index 430979274..d4326dc84 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -185,4 +185,4 @@ struct CmdSearch : InstallableCommand, MixJSON } }; -static auto r1 = registerCommand("search"); +static auto rCmdSearch = registerCommand("search"); diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 3ed1ad2aa..1ef54a33a 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -30,4 +30,4 @@ struct CmdShowConfig : Command, MixJSON } }; -static auto r1 = registerCommand("show-config"); +static auto rShowConfig = registerCommand("show-config"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index b9f33499b..2542537d3 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -123,4 +123,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static auto r1 = registerCommand("show-derivation"); +static auto rCmdShowDerivation = registerCommand("show-derivation"); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 7821a5432..44916c77f 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -92,7 +92,7 @@ struct CmdCopySigs : StorePathsCommand } }; -static auto r1 = registerCommand("copy-sigs"); +static auto rCmdCopySigs = registerCommand("copy-sigs"); struct CmdSignPaths : StorePathsCommand { @@ -144,4 +144,4 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto r2 = registerCommand("sign-paths"); +static auto rCmdSignPaths = registerCommand("sign-paths"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a880bdae0..66ecc5b34 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -158,4 +158,4 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand } }; -static auto r1 = registerCommand("upgrade-nix"); +static auto rCmdUpgradeNix = registerCommand("upgrade-nix"); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 26f755fd9..ec7333d03 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -189,4 +189,4 @@ struct CmdVerify : StorePathsCommand } }; -static auto r1 = registerCommand("verify"); +static auto rCmdVerify = registerCommand("verify"); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 7e630b745..63bf087e6 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -263,4 +263,4 @@ struct CmdWhyDepends : SourceExprCommand } }; -static auto r1 = registerCommand("why-depends"); +static auto rCmdWhyDepends = registerCommand("why-depends"); From ad143c5b3b7e713b89f0437ce17c20ac642ca530 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 14:00:42 +0200 Subject: [PATCH 101/123] Shut up some clang warnings --- src/libexpr/eval-cache.hh | 2 +- src/libfetchers/fetchers.hh | 3 +++ src/libfetchers/github.cc | 6 +++--- src/libstore/dummy-store.cc | 3 +-- src/libstore/store-api.hh | 2 ++ 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 8ffffc0ed..e23e45c94 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -11,7 +11,7 @@ namespace nix::eval_cache { MakeError(CachedEvalError, EvalError); -class AttrDb; +struct AttrDb; class AttrCursor; class EvalCache : public std::enable_shared_from_this diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index cc31a31b9..e8ae59143 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -86,6 +86,9 @@ public: struct InputScheme { + virtual ~InputScheme() + { } + virtual std::optional inputFromURL(const ParsedURL & url) = 0; virtual std::optional inputFromAttrs(const Attrs & attrs) = 0; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 90893075e..92ff224f7 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -25,7 +25,7 @@ struct GitArchiveInputScheme : InputScheme { virtual std::string type() = 0; - virtual std::optional > accessHeaderFromToken(const std::string & token) const = 0; + virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; std::optional inputFromURL(const ParsedURL & url) override { @@ -215,7 +215,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string type() override { return "github"; } - std::optional > accessHeaderFromToken(const std::string & token) const + std::optional> accessHeaderFromToken(const std::string & token) const override { // Github supports PAT/OAuth2 tokens and HTTP Basic // Authentication. The former simply specifies the token, the @@ -270,7 +270,7 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string type() override { return "gitlab"; } - std::optional > accessHeaderFromToken(const std::string & token) const + std::optional> accessHeaderFromToken(const std::string & token) const override { // Gitlab supports 4 kinds of authorization, two of which are // relevant here: OAuth2 and PAT (Private Access Token). The diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 736be8413..98b745c3a 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -18,8 +18,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig DummyStore(const Params & params) : StoreConfig(params) , Store(params) - { - } + { } string getUri() override { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 854446987..450c0f554 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -194,6 +194,8 @@ struct StoreConfig : public Config */ StoreConfig() { assert(false); } + virtual ~StoreConfig() { } + virtual const std::string name() = 0; const PathSetting storeDir_{this, false, settings.nixStore, From d761485010cc9d071da17a5d90cf4126a4bf499d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 18:52:46 +0200 Subject: [PATCH 102/123] Prevent a deadlock when user namespace setup fails Observed on Centos 7 when user namespaces are disabled: DerivationGoal::startBuilder() throws an exception, ~DerivationGoal() waits for the child process to exit, but the child process hangs forever in drainFD(userNamespaceSync.readSide.get()) in DerivationGoal::runChild(). Not sure why the SIGKILL doesn't get through. Issue #4092. --- src/libstore/build.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 87e01a378..9497e7b3e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2686,6 +2686,12 @@ void DerivationGoal::startBuilder() userNamespaceSync.readSide = -1; + /* Close the write side to prevent runChild() from hanging + reading from this. */ + Finally cleanup([&]() { + userNamespaceSync.writeSide = -1; + }); + pid_t tmp; if (!string2Int(readLine(builderOut.readSide.get()), tmp)) abort(); pid = tmp; @@ -2712,7 +2718,6 @@ void DerivationGoal::startBuilder() /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); - userNamespaceSync.writeSide = -1; } else #endif From 57d960dcd174f0910c7fca02ee3afe80cb2b7844 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 7 Oct 2020 12:48:35 +0000 Subject: [PATCH 103/123] Remove generic std::optional suppport from worker proto See comment for rational; I think it's good to leave a comment lest anyone is tempted to add such a sum-type instance again. Fixes #4113 --- src/libstore/worker-protocol.hh | 48 +++++++++++---------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 2934c1d67..564fb978a 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -83,7 +83,6 @@ MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); MAKE_WORKER_PROTO(template, std::set); -MAKE_WORKER_PROTO(template, std::optional); #define X_ template #define Y_ std::map @@ -91,6 +90,22 @@ MAKE_WORKER_PROTO(X_, Y_); #undef X_ #undef Y_ +/* These use the empty string for the null case, relying on the fact + that the underlying types never serialize to the empty string. + + We do this instead of a generic std::optional instance because + ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + the same reason, we don't have a std::variant instances (ordinal + tags 0...n). + + We could the generic instances and then these as specializations for + compatability, but that's proven a bit finnicky, and also makes the + worker protocol harder to implement in other languages where such + specializations may not be allowed. + */ +MAKE_WORKER_PROTO(, std::optional); +MAKE_WORKER_PROTO(, std::optional); + template std::set read(const Store & store, Source & from, Phantom> _) { @@ -134,37 +149,6 @@ void write(const Store & store, Sink & out, const std::map & resMap) } } -template -std::optional read(const Store & store, Source & from, Phantom> _) -{ - auto tag = readNum(from); - switch (tag) { - case 0: - return std::nullopt; - case 1: - return read(store, from, Phantom {}); - default: - throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag); - } -} - -template -void write(const Store & store, Sink & out, const std::optional & optVal) -{ - out << (uint64_t) (optVal ? 1 : 0); - if (optVal) - worker_proto::write(store, out, *optVal); -} - -/* Specialization which uses and empty string for the empty case, taking - advantage of the fact these types always serialize to non-empty strings. - This is done primarily for backwards compatability, so that T <= - std::optional, where <= is the compatability partial order, T is one of - the types below. - */ -MAKE_WORKER_PROTO(, std::optional); -MAKE_WORKER_PROTO(, std::optional); - } } From 27ca87c46a57e1e6603028c1902189b53635d576 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 16:33:19 +0200 Subject: [PATCH 104/123] Formatting --- src/libutil/error.cc | 26 +++++++++++++------------- src/libutil/error.hh | 4 ++-- src/libutil/fmt.hh | 30 +++++++++++++++++------------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index fd6f69b7f..803a72953 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -11,13 +11,13 @@ const std::string nativeSystem = SYSTEM; BaseError & BaseError::addTrace(std::optional e, hintformat hint) { - err.traces.push_front(Trace { .pos = e, .hint = hint}); + err.traces.push_front(Trace { .pos = e, .hint = hint }); return *this; } // c++ std::exception descendants must have a 'const char* what()' function. // This stringifies the error and caches it for use by what(), or similarly by msg(). -const string& BaseError::calcWhat() const +const string & BaseError::calcWhat() const { if (what_.has_value()) return *what_; @@ -34,12 +34,12 @@ const string& BaseError::calcWhat() const std::optional ErrorInfo::programName = std::nullopt; -std::ostream& operator<<(std::ostream &os, const hintformat &hf) +std::ostream & operator<<(std::ostream & os, const hintformat & hf) { return os << hf.str(); } -string showErrPos(const ErrPos &errPos) +string showErrPos(const ErrPos & errPos) { if (errPos.line > 0) { if (errPos.column > 0) { @@ -53,7 +53,7 @@ string showErrPos(const ErrPos &errPos) } } -std::optional getCodeLines(const ErrPos &errPos) +std::optional getCodeLines(const ErrPos & errPos) { if (errPos.line <= 0) return std::nullopt; @@ -92,13 +92,13 @@ std::optional getCodeLines(const ErrPos &errPos) return loc; } } - catch (EndOfFile &eof) { + catch (EndOfFile & eof) { if (loc.errLineOfCode.has_value()) return loc; else return std::nullopt; } - catch (std::exception &e) { + catch (std::exception & e) { printError("error reading nix file: %s\n%s", errPos.file, e.what()); return std::nullopt; } @@ -137,10 +137,10 @@ std::optional getCodeLines(const ErrPos &errPos) } // print lines of code to the ostream, indicating the error column. -void printCodeLines(std::ostream &out, - const string &prefix, - const ErrPos &errPos, - const LinesOfCode &loc) +void printCodeLines(std::ostream & out, + const string & prefix, + const ErrPos & errPos, + const LinesOfCode & loc) { // previous line of code. if (loc.prevLineOfCode.has_value()) { @@ -186,7 +186,7 @@ void printCodeLines(std::ostream &out, } } -void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out) +void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out) { if (pos) { @@ -212,7 +212,7 @@ void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out) } } -std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace) +std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { auto errwidth = std::max(getWindowSize().second, 20); string prefix = ""; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f3babcbde..d1b6d82bb 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -107,7 +107,7 @@ struct Trace { struct ErrorInfo { Verbosity level; string name; - string description; + string description; // FIXME: remove? it seems to be barely used std::optional hint; std::optional errPos; std::list traces; @@ -169,7 +169,7 @@ public: #endif const string & msg() const { return calcWhat(); } - const ErrorInfo & info() { calcWhat(); return err; } + const ErrorInfo & info() const { calcWhat(); return err; } template BaseError & addTrace(std::optional e, const string &fs, const Args & ... args) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 6e69bdce2..85c0e9429 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -76,11 +76,11 @@ template struct yellowtxt { yellowtxt(const T &s) : value(s) {} - const T &value; + const T & value; }; template -std::ostream& operator<<(std::ostream &out, const yellowtxt &y) +std::ostream & operator<<(std::ostream & out, const yellowtxt & y) { return out << ANSI_YELLOW << y.value << ANSI_NORMAL; } @@ -88,12 +88,12 @@ std::ostream& operator<<(std::ostream &out, const yellowtxt &y) template struct normaltxt { - normaltxt(const T &s) : value(s) {} - const T &value; + normaltxt(const T & s) : value(s) {} + const T & value; }; template -std::ostream& operator<<(std::ostream &out, const normaltxt &y) +std::ostream & operator<<(std::ostream & out, const normaltxt & y) { return out << ANSI_NORMAL << y.value; } @@ -101,26 +101,30 @@ std::ostream& operator<<(std::ostream &out, const normaltxt &y) class hintformat { public: - hintformat(const string &format) :fmt(format) + hintformat(const string & format) : fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } - hintformat(const hintformat &hf) - : fmt(hf.fmt) - {} + hintformat(const hintformat & hf) + : fmt(hf.fmt) + { } + + hintformat(format && fmt) + : fmt(std::move(fmt)) + { } template - hintformat& operator%(const T &value) + hintformat & operator%(const T & value) { fmt % yellowtxt(value); return *this; } template - hintformat& operator%(const normaltxt &value) + hintformat & operator%(const normaltxt & value) { fmt % value.value; return *this; @@ -135,7 +139,7 @@ private: format fmt; }; -std::ostream& operator<<(std::ostream &os, const hintformat &hf); +std::ostream & operator<<(std::ostream & os, const hintformat & hf); template inline hintformat hintfmt(const std::string & fs, const Args & ... args) From be149acfdaae06667bc58df467c04c41827b69d0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 16:34:03 +0200 Subject: [PATCH 105/123] Serialize exceptions from the sandbox process to the parent Fixes #4118. --- src/libstore/build.cc | 16 ++++++++++----- src/libutil/serialise.cc | 42 ++++++++++++++++++++++++++++++++++++++++ src/libutil/serialise.hh | 3 +++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9497e7b3e..fa7283588 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2737,9 +2737,12 @@ void DerivationGoal::startBuilder() /* Check if setting up the build environment failed. */ while (true) { string msg = readLine(builderOut.readSide.get()); + if (string(msg, 0, 1) == "\2") break; if (string(msg, 0, 1) == "\1") { - if (msg.size() == 1) break; - throw Error(string(msg, 1)); + FdSource source(builderOut.readSide.get()); + auto ex = readError(source); + ex.addTrace({}, "while setting up the build environment"); + throw ex; } debug(msg); } @@ -3785,7 +3788,7 @@ void DerivationGoal::runChild() args.push_back(rewriteStrings(i, inputRewrites)); /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\1\n")); + writeFull(STDERR_FILENO, string("\2\n")); /* Execute the program. This should not return. */ if (drv->isBuiltin()) { @@ -3815,8 +3818,11 @@ void DerivationGoal::runChild() throw SysError("executing '%1%'", drv->builder); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); + } catch (Error & e) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); _exit(1); } } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index a469a1e73..5c9f6f901 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -266,6 +266,24 @@ Sink & operator << (Sink & sink, const StringSet & s) return sink; } +Sink & operator << (Sink & sink, const Error & ex) +{ + auto info = ex.info(); + sink + << "Error" + << info.level + << info.name + << info.description + << (info.hint ? info.hint->str() : "") + << 0 // FIXME: info.errPos + << info.traces.size(); + for (auto & trace : info.traces) { + sink << 0; // FIXME: trace.pos + sink << trace.hint.str(); + } + return sink; +} + void readPadding(size_t len, Source & source) { @@ -319,6 +337,30 @@ template Paths readStrings(Source & source); template PathSet readStrings(Source & source); +Error readError(Source & source) +{ + auto type = readString(source); + assert(type == "Error"); + ErrorInfo info; + info.level = (Verbosity) readInt(source); + info.name = readString(source); + info.description = readString(source); + auto hint = readString(source); + if (hint != "") info.hint = hintformat(std::move(format("%s") % hint)); + auto havePos = readNum(source); + assert(havePos == 0); + auto nrTraces = readNum(source); + for (size_t i = 0; i < nrTraces; ++i) { + havePos = readNum(source); + assert(havePos == 0); + info.traces.push_back(Trace { + .hint = hintformat(std::move(format("%s") % readString(source))) + }); + } + return Error(std::move(info)); +} + + void StringSink::operator () (const unsigned char * data, size_t len) { static bool warned = false; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index b41e58f33..d7fe0b81e 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -321,6 +321,7 @@ inline Sink & operator << (Sink & sink, uint64_t n) Sink & operator << (Sink & sink, const string & s); Sink & operator << (Sink & sink, const Strings & s); Sink & operator << (Sink & sink, const StringSet & s); +Sink & operator << (Sink & in, const Error & ex); MakeError(SerialisationError, Error); @@ -382,6 +383,8 @@ Source & operator >> (Source & in, bool & b) return in; } +Error readError(Source & source); + /* An adapter that converts a std::basic_istream into a source. */ struct StreamToSourceAdapter : Source From c43e882f54b5f09278dfb0d315239531c0676b4e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 17:13:54 +0200 Subject: [PATCH 106/123] Serialize exceptions from the daemon to the client --- src/libstore/daemon.cc | 23 ++++++++++++++++------- src/libstore/remote-store.cc | 10 +++++++--- src/libstore/worker-protocol.hh | 2 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ae2fbec35..99d8add92 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -101,17 +101,20 @@ struct TunnelLogger : public Logger /* stopWork() means that we're done; stop sending stderr to the client. */ - void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) + void stopWork(const Error * ex = nullptr) { auto state(state_.lock()); state->canSendStderr = false; - if (success) + if (!ex) to << STDERR_LAST; else { - to << STDERR_ERROR << msg; - if (status != 0) to << status; + if (GET_PROTOCOL_MINOR(clientVersion) >= 26) { + to << STDERR_ERROR << *ex; + } else { + to << STDERR_ERROR << ex->what() << ex->status; + } } } @@ -935,10 +938,11 @@ void processConnection( during addTextToStore() / importPath(). If that happens, just send the error message and exit. */ bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; - tunnelLogger->stopWork(false, e.msg(), e.status); + tunnelLogger->stopWork(&e); if (!errorAllowed) throw; } catch (std::bad_alloc & e) { - tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + auto ex = Error("Nix daemon out of memory"); + tunnelLogger->stopWork(&ex); throw; } @@ -947,8 +951,13 @@ void processConnection( assert(!tunnelLogger->state_.lock()->canSendStderr); }; + } catch (Error & e) { + tunnelLogger->stopWork(&e); + to.flush(); + return; } catch (std::exception & e) { - tunnelLogger->stopWork(false, e.what(), 1); + auto ex = Error(e.what()); + tunnelLogger->stopWork(&ex); to.flush(); return; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 40ef72293..23b1942ce 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -925,9 +925,13 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * } else if (msg == STDERR_ERROR) { - string error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + return std::make_exception_ptr(readError(from)); + } else { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } } else if (msg == STDERR_NEXT) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 564fb978a..b3705578e 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x119 +#define PROTOCOL_VERSION 0x11a #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From e705c2429445a4a05076fc067bb349bdd6752bd9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 17:28:43 +0200 Subject: [PATCH 107/123] Tweak error messages --- src/libstore/build.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index fa7283588..0585905e0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2355,7 +2355,8 @@ void DerivationGoal::startBuilder() worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); } catch (InvalidPath & e) { } catch (Error & e) { - throw Error("while processing 'sandbox-paths': %s", e.what()); + e.addTrace({}, "while processing 'sandbox-paths'"); + throw; } for (auto & i : closure) { auto p = worker.store.printStorePath(i); @@ -3809,7 +3810,7 @@ void DerivationGoal::runChild() throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { - writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + writeFull(STDERR_FILENO, e.what() + "\n"); _exit(1); } } From f66bbd8c7bb1472facf8917e58e3cd4f6ddfa1b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 21:25:06 +0200 Subject: [PATCH 108/123] Doh --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0585905e0..e1774237b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3810,7 +3810,7 @@ void DerivationGoal::runChild() throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + "\n"); + writeFull(STDERR_FILENO, e.what() + std::string("\n")); _exit(1); } } From 6aa64627c8e431c3b187f7bb44c943d06e39b929 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 22:02:36 +0200 Subject: [PATCH 109/123] Support user namespaces being disabled If max_user_namespaces is set to 0, then don't run the build in a user namespace. Fixes #4092. --- src/libstore/build.cc | 54 ++++++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e1774237b..a645c429d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -831,6 +831,10 @@ private: paths to the sandbox as a result of recursive Nix calls. */ AutoCloseFD sandboxMountNamespace; + /* On Linux, whether we're doing the build in its own user + namespace. */ + bool usingUserNamespace = true; + /* The build hook. */ std::unique_ptr hook; @@ -920,8 +924,8 @@ private: result. */ std::map prevInfos; - const uid_t sandboxUid = 1000; - const gid_t sandboxGid = 100; + uid_t sandboxUid = 1000; + gid_t sandboxGid = 100; const static Path homeDir; @@ -2629,6 +2633,24 @@ void DerivationGoal::startBuilder() options.allowVfork = false; + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + static bool userNamespacesEnabled = + pathExists(maxUserNamespaces) + && trim(readFile(maxUserNamespaces)) != "0"; + + usingUserNamespace = userNamespacesEnabled; + + if (usingUserNamespace) { + sandboxUid = 1000; + sandboxGid = 100; + } else { + debug("note: not using a user namespace"); + if (!buildUser) + throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); + sandboxUid = buildUser->getUID(); + sandboxGid = buildUser->getGID(); + } + Pid helper = startProcess([&]() { /* Drop additional groups here because we can't do it @@ -2647,9 +2669,11 @@ void DerivationGoal::startBuilder() PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); if (stack == MAP_FAILED) throw SysError("allocating stack"); - int flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (privateNetwork) flags |= CLONE_NEWNET; + if (usingUserNamespace) + flags |= CLONE_NEWUSER; pid_t child = clone(childEntry, stack + stackSize, flags, this); if (child == -1 && errno == EINVAL) { @@ -2697,19 +2721,21 @@ void DerivationGoal::startBuilder() if (!string2Int(readLine(builderOut.readSide.get()), tmp)) abort(); pid = tmp; - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + if (usingUserNamespace) { + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - (format("%d %d 1") % sandboxUid % hostUid).str()); + writeFile("/proc/" + std::to_string(pid) + "/uid_map", + (format("%d %d 1") % sandboxUid % hostUid).str()); - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - (format("%d %d 1") % sandboxGid % hostGid).str()); + writeFile("/proc/" + std::to_string(pid) + "/gid_map", + (format("%d %d 1") % sandboxGid % hostGid).str()); + } /* Save the mount namespace of the child. We have to do this *before* the child does a chroot. */ @@ -2745,7 +2771,7 @@ void DerivationGoal::startBuilder() ex.addTrace({}, "while setting up the build environment"); throw ex; } - debug(msg); + debug("sandbox setup: " + msg); } } From 97ffc1e0139e124b7e36b5d1a62b90300f231118 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 22:46:01 +0200 Subject: [PATCH 110/123] Dynamically disable user namespaces if CLONE_NEWUSER fails This makes builds work inside nixos-enter. Fixes #3145. --- src/libstore/build.cc | 45 ++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index a645c429d..05a9ef088 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -924,8 +924,8 @@ private: result. */ std::map prevInfos; - uid_t sandboxUid = 1000; - gid_t sandboxGid = 100; + uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } const static Path homeDir; @@ -2428,15 +2428,14 @@ void DerivationGoal::startBuilder() "root:x:0:0:Nix build user:%3%:/noshell\n" "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid, sandboxGid, settings.sandboxBuildDir)); + sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ writeFile(chrootRootDir + "/etc/group", - (format( - "root:x:0:\n" + fmt("root:x:0:\n" "nixbld:!:%1%:\n" - "nogroup:x:65534:\n") % sandboxGid).str()); + "nogroup:x:65534:\n", sandboxGid())); /* Create /etc/hosts with localhost entry. */ if (!(derivationIsImpure(derivationType))) @@ -2640,17 +2639,6 @@ void DerivationGoal::startBuilder() usingUserNamespace = userNamespacesEnabled; - if (usingUserNamespace) { - sandboxUid = 1000; - sandboxGid = 100; - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - sandboxUid = buildUser->getUID(); - sandboxGid = buildUser->getGID(); - } - Pid helper = startProcess([&]() { /* Drop additional groups here because we can't do it @@ -2682,11 +2670,12 @@ void DerivationGoal::startBuilder() flags &= ~CLONE_NEWPID; child = clone(childEntry, stack + stackSize, flags, this); } - if (child == -1 && (errno == EPERM || errno == EINVAL)) { + if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { /* Some distros patch Linux to not allow unprivileged * user namespaces. If we get EPERM or EINVAL, try * without CLONE_NEWUSER and see if that works. */ + usingUserNamespace = false; flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } @@ -2697,7 +2686,8 @@ void DerivationGoal::startBuilder() _exit(1); if (child == -1) throw SysError("cloning builder process"); - writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); + writeFull(builderOut.writeSide.get(), + fmt("%d %d\n", usingUserNamespace, child)); _exit(0); }, options); @@ -2718,7 +2708,10 @@ void DerivationGoal::startBuilder() }); pid_t tmp; - if (!string2Int(readLine(builderOut.readSide.get()), tmp)) abort(); + auto ss = tokenizeString>(readLine(builderOut.readSide.get())); + assert(ss.size() == 2); + usingUserNamespace = ss[0] == "1"; + if (!string2Int(ss[1], tmp)) abort(); pid = tmp; if (usingUserNamespace) { @@ -2729,12 +2722,16 @@ void DerivationGoal::startBuilder() uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); writeFile("/proc/" + std::to_string(pid) + "/uid_map", - (format("%d %d 1") % sandboxUid % hostUid).str()); + fmt("%d %d 1", sandboxUid(), hostUid)); writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", - (format("%d %d 1") % sandboxGid % hostGid).str()); + fmt("%d %d 1", sandboxGid(), hostGid)); + } else { + debug("note: not using a user namespace"); + if (!buildUser) + throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); } /* Save the mount namespace of the child. We have to do this @@ -3595,9 +3592,9 @@ void DerivationGoal::runChild() /* Switch to the sandbox uid/gid in the user namespace, which corresponds to the build user or calling user in the parent namespace. */ - if (setgid(sandboxGid) == -1) + if (setgid(sandboxGid()) == -1) throw SysError("setgid failed"); - if (setuid(sandboxUid) == -1) + if (setuid(sandboxUid()) == -1) throw SysError("setuid failed"); setUser = false; From eaef251b2ba416fcde8f0f0b72ce68c548ce35c3 Mon Sep 17 00:00:00 2001 From: Horki Date: Thu, 8 Oct 2020 13:40:47 +0200 Subject: [PATCH 111/123] rust: small patches --- nix-rust/src/lib.rs | 1 + nix-rust/src/store/path.rs | 35 +++++++++++++++++------------------ nix-rust/src/util/base32.rs | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 27ea69fbd..101de106f 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -1,3 +1,4 @@ +#[allow(improper_ctypes_definitions)] #[cfg(not(test))] mod c; mod error; diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs index 47b5975c0..99f7a1f18 100644 --- a/nix-rust/src/store/path.rs +++ b/nix-rust/src/store/path.rs @@ -19,9 +19,9 @@ impl StorePath { } Self::new_from_base_name( path.file_name() - .ok_or(Error::BadStorePath(path.into()))? + .ok_or_else(|| Error::BadStorePath(path.into()))? .to_str() - .ok_or(Error::BadStorePath(path.into()))?, + .ok_or_else(|| Error::BadStorePath(path.into()))?, ) } @@ -34,7 +34,7 @@ impl StorePath { pub fn new_from_base_name(base_name: &str) -> Result { if base_name.len() < STORE_PATH_HASH_CHARS + 1 - || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8 + || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != b'-' { return Err(Error::BadStorePath(base_name.into())); } @@ -65,7 +65,7 @@ impl StorePathHash { Ok(Self(bytes)) } - pub fn hash<'a>(&'a self) -> &'a [u8; STORE_PATH_HASH_BYTES] { + pub fn hash(&self) -> &[u8; STORE_PATH_HASH_BYTES] { &self.0 } } @@ -98,7 +98,7 @@ pub struct StorePathName(String); impl StorePathName { pub fn new(s: &str) -> Result { - if s.len() == 0 { + if s.is_empty() { return Err(Error::StorePathNameEmpty); } @@ -106,25 +106,24 @@ impl StorePathName { return Err(Error::StorePathNameTooLong); } - if s.starts_with('.') - || !s.chars().all(|c| { - c.is_ascii_alphabetic() - || c.is_ascii_digit() - || c == '+' - || c == '-' - || c == '.' - || c == '_' - || c == '?' - || c == '=' - }) - { + let is_good_path_name = s.chars().all(|c| { + c.is_ascii_alphabetic() + || c.is_ascii_digit() + || c == '+' + || c == '-' + || c == '.' + || c == '_' + || c == '?' + || c == '=' + }); + if s.starts_with('.') || !is_good_path_name { return Err(Error::BadStorePathName); } Ok(Self(s.to_string())) } - pub fn name<'a>(&'a self) -> &'a str { + pub fn name(&self) -> &str { &self.0 } } diff --git a/nix-rust/src/util/base32.rs b/nix-rust/src/util/base32.rs index efd4a2901..7e71dc920 100644 --- a/nix-rust/src/util/base32.rs +++ b/nix-rust/src/util/base32.rs @@ -13,7 +13,7 @@ pub fn decoded_len(input_len: usize) -> usize { input_len * 5 / 8 } -static BASE32_CHARS: &'static [u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz"; +static BASE32_CHARS: &[u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz"; lazy_static! { static ref BASE32_CHARS_REVERSE: Box<[u8; 256]> = { From 58dadf295499588e492dab6bdc5934dc27ad3e64 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Oct 2020 17:30:40 +0200 Subject: [PATCH 112/123] Remove stray 'Title:' from the manual Closes #4096. --- doc/manual/generate-manpage.nix | 3 +-- doc/manual/local.mk | 15 ++++++++++++--- doc/manual/src/command-ref/conf-file-prefix.md | 2 -- doc/manual/src/command-ref/nix-build.md | 2 -- doc/manual/src/command-ref/nix-channel.md | 2 -- doc/manual/src/command-ref/nix-collect-garbage.md | 2 -- doc/manual/src/command-ref/nix-copy-closure.md | 2 -- doc/manual/src/command-ref/nix-daemon.md | 2 -- doc/manual/src/command-ref/nix-env.md | 2 -- doc/manual/src/command-ref/nix-hash.md | 2 -- doc/manual/src/command-ref/nix-instantiate.md | 2 -- doc/manual/src/command-ref/nix-prefetch-url.md | 2 -- doc/manual/src/command-ref/nix-shell.md | 2 -- doc/manual/src/command-ref/nix-store.md | 2 -- 14 files changed, 13 insertions(+), 29 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 4709c0e2c..db266750a 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -52,5 +52,4 @@ in command: -"Title: nix\n\n" -+ showCommand { command = "nix"; section = "#"; def = command; } +showCommand { command = "nix"; section = "#"; def = command; } diff --git a/doc/manual/local.mk b/doc/manual/local.mk index d308c7cf6..7d9a1a3e8 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -18,13 +18,22 @@ dist-files += $(man-pages) nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr $(d)/%.1: $(d)/src/command-ref/%.md - $(trace-gen) lowdown -sT man $^ -o $@ + @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp + @cat $^ >> $^.tmp + $(trace-gen) lowdown -sT man $^.tmp -o $@ + @rm $^.tmp $(d)/%.8: $(d)/src/command-ref/%.md - $(trace-gen) lowdown -sT man $^ -o $@ + @printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp + @cat $^ >> $^.tmp + $(trace-gen) lowdown -sT man $^.tmp -o $@ + @rm $^.tmp $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md - $(trace-gen) lowdown -sT man $^ -o $@ + @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp + @cat $^ >> $^.tmp + $(trace-gen) lowdown -sT man $^.tmp -o $@ + @rm $^.tmp $(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix $(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 04c6cd859..9987393d2 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -1,5 +1,3 @@ -Title: nix.conf - # Name `nix.conf` - Nix configuration file diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index 4bcb8db40..4565bfbc2 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -1,5 +1,3 @@ -Title: nix-build - # Name `nix-build` - build a Nix expression diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index f0e205967..4ca12d2cc 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -1,5 +1,3 @@ -Title: nix-channel - # Name `nix-channel` - manage Nix channels diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 62a6b7ca0..296165993 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -1,5 +1,3 @@ -Title: nix-collect-garbage - # Name `nix-collect-garbage` - delete unreachable store paths diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index 5ce320af7..dcb844a72 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -1,5 +1,3 @@ -Title: nix-copy-closure - # Name `nix-copy-closure` - copy a closure to or from a remote machine via SSH diff --git a/doc/manual/src/command-ref/nix-daemon.md b/doc/manual/src/command-ref/nix-daemon.md index bd5d25026..e91cb01dd 100644 --- a/doc/manual/src/command-ref/nix-daemon.md +++ b/doc/manual/src/command-ref/nix-daemon.md @@ -1,5 +1,3 @@ -Title: nix-daemon - # Name `nix-daemon` - Nix multi-user support daemon diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index ee838581b..1c23bb0ad 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -1,5 +1,3 @@ -Title: nix-env - # Name `nix-env` - manipulate or query Nix user environments diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md index d3f91f8e9..7ed82cdfc 100644 --- a/doc/manual/src/command-ref/nix-hash.md +++ b/doc/manual/src/command-ref/nix-hash.md @@ -1,5 +1,3 @@ -Title: nix-hash - # Name `nix-hash` - compute the cryptographic hash of a path diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index d09f5ed6a..c369397b6 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -1,5 +1,3 @@ -Title: nix-instantiate - # Name `nix-instantiate` - instantiate store derivations from Nix expressions diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 1307c7c37..78c612cd4 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -1,5 +1,3 @@ -Title: nix-prefetch-url - # Name `nix-prefetch-url` - copy a file from a URL into the store and print its hash diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index fc42a202a..45a5ff08c 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -1,5 +1,3 @@ -Title: nix-shell - # Name `nix-shell` - start an interactive shell based on a Nix expression diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 4680339e4..827adbd05 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -1,5 +1,3 @@ -Title: nix-store - # Name `nix-store` - manipulate or query the Nix store From 04e5d0e7040fdfbbc084634c0694ae7da89765d9 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 9 Oct 2020 09:39:51 +0200 Subject: [PATCH 113/123] Add a description in the completion outputs Make nix output completions in the form `completion\tdescription`. This can't be used by bash (afaik), but other shells like zsh or fish can display it along the completion choices --- misc/bash/completion.sh | 5 +++-- src/libmain/common-args.cc | 2 +- src/libutil/args.cc | 29 ++++++++++++++++++++--------- src/libutil/args.hh | 12 +++++++++++- src/nix/installables.cc | 12 ++++++------ src/nix/main.cc | 2 +- 6 files changed, 42 insertions(+), 20 deletions(-) diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index bc184edd6..bea2a40bc 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -4,13 +4,14 @@ function _complete_nix { _get_comp_words_by_ref -n ':=&' words cword cur local have_type while IFS= read -r line; do + local completion=${line%% *} if [[ -z $have_type ]]; then have_type=1 - if [[ $line = filenames ]]; then + if [[ $completion = filenames ]]; then compopt -o filenames fi else - COMPREPLY+=("$line") + COMPREPLY+=("$completion") fi done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}") __ltrim_colon_completions "$cur" diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 3411e2d7a..9151a0344 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -44,7 +44,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->insert(s.first); + completions->add(s.first, s.second.description); } } }); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 2760b830b..66d8df085 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -17,8 +17,19 @@ void Args::addFlag(Flag && flag_) if (flag->shortName) shortFlags[flag->shortName] = flag; } +void Completions::add(std::string completion, std::string description) +{ + insert(Completion{ + .completion = completion, + .description = description + }); +} + +bool Completion::operator<(const Completion & other) const +{ return completion < other.completion || (completion == other.completion && description < other.description); } + bool pathCompletions = false; -std::shared_ptr> completions; +std::shared_ptr completions; std::string completionMarker = "___COMPLETE___"; @@ -148,7 +159,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) - completions->insert("--" + name); + completions->add("--" + name, flag->description); } } auto i = longFlags.find(string(*pos, 2)); @@ -165,9 +176,9 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) if (auto prefix = needsCompletion(*pos)) { if (prefix == "-") { - completions->insert("--"); - for (auto & [flag, _] : shortFlags) - completions->insert(std::string("-") + flag); + completions->add("--"); + for (auto & [flagName, flag] : shortFlags) + completions->add(std::string("-") + flagName, flag->description); } } @@ -244,11 +255,11 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) - completions->insert(type); + completions->add(type); } Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) @@ -292,7 +303,7 @@ static void _completePath(std::string_view prefix, bool onlyDirs) auto st = lstat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } - completions->insert(globbuf.gl_pathv[i]); + completions->add(globbuf.gl_pathv[i]); } globfree(&globbuf); } @@ -385,7 +396,7 @@ MultiCommand::MultiCommand(const Commands & commands) if (auto prefix = needsCompletion(s)) { for (auto & [name, command] : commands) if (hasPrefix(name, *prefix)) - completions->insert(name); + completions->add(name); } auto i = commands.find(s); if (i == commands.end()) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index f41242e17..26f1bc11b 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -283,7 +283,17 @@ typedef std::vector> Table2; void printTable(std::ostream & out, const Table2 & table); -extern std::shared_ptr> completions; +struct Completion { + std::string completion; + std::string description; + + bool operator<(const Completion & other) const; +}; +class Completions : public std::set { +public: + void add(std::string completion, std::string description = ""); +}; +extern std::shared_ptr completions; extern bool pathCompletions; std::optional needsCompletion(std::string_view s); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 9bf6b7caa..7473c9758 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -26,7 +26,7 @@ void completeFlakeInputPath( auto flake = flake::getFlake(*evalState, flakeRef, true); for (auto & input : flake.inputs) if (hasPrefix(input.first, prefix)) - completions->insert(input.first); + completions->add(input.first); } MixFlakeOptions::MixFlakeOptions() @@ -211,7 +211,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = attr->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->insert(flakeRefS + "#" + concatStringsSep(".", attrPath2)); + completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2)); } } } @@ -222,7 +222,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->insert(flakeRefS + "#"); + completions->add(flakeRefS + "#"); } } } @@ -243,7 +243,7 @@ ref EvalCommand::getEvalState() void completeFlakeRef(ref store, std::string_view prefix) { if (prefix == "") - completions->insert("."); + completions->add("."); completeDir(0, prefix); @@ -254,10 +254,10 @@ void completeFlakeRef(ref store, std::string_view prefix) if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) - completions->insert(from2); + completions->add(from2); } else { if (hasPrefix(from, prefix)) - completions->insert(from); + completions->add(from); } } } diff --git a/src/nix/main.cc b/src/nix/main.cc index 1e9e07bc0..5056ceb78 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -208,7 +208,7 @@ void mainWrapped(int argc, char * * argv) if (completions) { std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n"); for (auto & s : *completions) - std::cout << s << "\n"; + std::cout << s.completion << "\t" << s.description << "\n"; } }); From eea310b2418e30280ae5fd9dac54ffe7f3c5dafa Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 9 Oct 2020 09:41:55 +0200 Subject: [PATCH 114/123] Add a zsh completion script Based on @clhodapp's suggestion in https://github.com/spwhitt/nix-zsh-completions/issues/32#issuecomment-705315356 and adapted to use the description of the completions --- misc/zsh/completion.zsh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 misc/zsh/completion.zsh diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh new file mode 100644 index 000000000..d4df6447e --- /dev/null +++ b/misc/zsh/completion.zsh @@ -0,0 +1,21 @@ +function _nix() { + local ifs_bk="$IFS" + local input=("${(Q)words[@]}") + IFS=$'\n' + local res=($(NIX_GET_COMPLETIONS=$((CURRENT - 1)) "$input[@]")) + IFS="$ifs_bk" + local tpe="${${res[1]}%%> *}" + local -a suggestions + declare -a suggestions + for suggestion in ${res:1}; do + # FIXME: This doesn't work properly if the suggestion word contains a `:` + # itself + suggestions+="${suggestion/ /:}" + done + if [[ "$tpe" == filenames ]]; then + compadd -f + fi + _describe 'nix' suggestions +} + +compdef _nix nix From 636ec171391660e986623cb75c400a99d652c991 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 15:41:24 +0200 Subject: [PATCH 115/123] Remove stray DerivationOutputsAndPaths type --- src/libstore/derivations.hh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d48266774..6d292b2e5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -61,8 +61,6 @@ typedef std::map DerivationOutputs; also contains, for each output, the (optional) store path in which it would be written. To calculate values of these types, see the corresponding functions in BasicDerivation */ -typedef std::map> - DerivationOutputsAndPaths; typedef std::map>> DerivationOutputsAndOptPaths; From 87157b2bd3224b8329ffcb73e92c64bdc36cff16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 16:02:53 +0200 Subject: [PATCH 116/123] writeFile(): Add error context to writeFull() failure Issue #4092. --- src/libutil/util.cc | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f07e99885..1a8873136 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -326,7 +326,12 @@ void writeFile(const Path & path, const string & s, mode_t mode) AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) throw SysError("opening file '%1%'", path); - writeFull(fd.get(), s); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } } @@ -338,11 +343,16 @@ void writeFile(const Path & path, Source & source, mode_t mode) std::vector buf(64 * 1024); - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), (unsigned char *) buf.data(), n); - } catch (EndOfFile &) { break; } + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), (unsigned char *) buf.data(), n); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; } } From e845d19ae368cb9ee6371c4b2fdbdc86a110d893 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 17:54:59 +0200 Subject: [PATCH 117/123] Remove Lazy This fixes a crash during startup when compiling Nix as a single compilation unit. --- src/libutil/lazy.hh | 48 --------------------------------------------- src/libutil/util.cc | 34 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 64 deletions(-) delete mode 100644 src/libutil/lazy.hh diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh deleted file mode 100644 index d073e486c..000000000 --- a/src/libutil/lazy.hh +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#include - -namespace nix { - -/* A helper class for lazily-initialized variables. - - Lazy var([]() { return value; }); - - declares a variable of type T that is initialized to 'value' (in a - thread-safe way) on first use, that is, when var() is first - called. If the initialiser code throws an exception, then all - subsequent calls to var() will rethrow that exception. */ -template -class Lazy -{ - - typedef std::function Init; - - Init init; - - std::once_flag done; - - T value; - - std::exception_ptr ex; - -public: - - Lazy(Init init) : init(init) - { } - - const T & operator () () - { - std::call_once(done, [&]() { - try { - value = init(); - } catch (...) { - ex = std::current_exception(); - } - }); - if (ex) std::rethrow_exception(ex); - return value; - } -}; - -} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1a8873136..9804e9a51 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,4 +1,3 @@ -#include "lazy.hh" #include "util.hh" #include "affinity.hh" #include "sync.hh" @@ -522,21 +521,24 @@ std::string getUserName() } -static Lazy getHome2([]() { - auto homeDir = getEnv("HOME"); - if (!homeDir) { - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; - } - return *homeDir; -}); - -Path getHome() { return getHome2(); } +Path getHome() +{ + static Path homeDir = []() + { + auto homeDir = getEnv("HOME"); + if (!homeDir) { + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + homeDir = pw->pw_dir; + } + return *homeDir; + }(); + return homeDir; +} Path getCacheDir() From 59bd6e87a44fab86c511f78ff50f3e3c4cca4b59 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 21:55:59 +0200 Subject: [PATCH 118/123] Completions::add(): Guard against newlines --- src/libutil/args.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 66d8df085..8bd9c8aeb 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -19,10 +19,11 @@ void Args::addFlag(Flag && flag_) void Completions::add(std::string completion, std::string description) { - insert(Completion{ - .completion = completion, - .description = description - }); + assert(description.find('\n') == std::string::npos); + insert(Completion { + .completion = completion, + .description = description + }); } bool Completion::operator<(const Completion & other) const From 44349064f75616fd47194425e27f5cb1d6ef5ecb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 22:02:00 +0200 Subject: [PATCH 119/123] nix develop: Source ~/.bashrc Fixes #4104. --- src/nix/develop.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index a46ea39b6..f763e8ba0 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -377,6 +377,10 @@ struct CmdDevelop : Common, MixEnvironment script += fmt("exec %s\n", concatStringsSep(" ", args)); } + else { + script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n"; + } + writeFull(rcFileFd.get(), script); stopProgressBar(); From 725488b8922c731042a557608b8d833b05e77243 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 22:03:18 +0200 Subject: [PATCH 120/123] nix develop: Unset $HOSTNAME This is set to "localhost" by stdenv which is probably not what you want. --- src/nix/develop.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index f763e8ba0..9372f43de 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -164,6 +164,7 @@ struct Common : InstallableCommand, MixProfile "BASHOPTS", "EUID", "HOME", // FIXME: don't ignore in pure mode? + "HOSTNAME", "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "NIX_LOG_FD", From 6cc1541782084111a8fa0a1e34d685a2e8c58459 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 9 Oct 2020 20:18:08 +0000 Subject: [PATCH 121/123] Split out `local-fs-store.hh` This matches the already-existing `local-fs-store.cc`. --- src/libfetchers/registry.cc | 1 + src/libstore/gc.cc | 1 + src/libstore/local-fs-store.cc | 1 + src/libstore/local-fs-store.hh | 48 ++++++++++++++++++++++++++ src/libstore/local-store.hh | 1 + src/libstore/profiles.cc | 1 + src/libstore/remote-store.hh | 1 + src/libstore/store-api.hh | 41 ---------------------- src/nix-build/nix-build.cc | 1 + src/nix-env/nix-env.cc | 1 + src/nix-env/user-env.cc | 1 + src/nix-instantiate/nix-instantiate.cc | 1 + src/nix/build.cc | 1 + src/nix/bundle.cc | 1 + src/nix/command.cc | 1 + src/nix/doctor.cc | 1 + 16 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 src/libstore/local-fs-store.hh diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 4367ee810..2426882ca 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -3,6 +3,7 @@ #include "util.hh" #include "globals.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 518a357ef..7a04b2f89 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,6 +1,7 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" +#include "local-fs-store.hh" #include "finally.hh" #include diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 2f1d9663a..e7c3dae92 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,6 +1,7 @@ #include "archive.hh" #include "fs-accessor.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "globals.hh" #include "compression.hh" #include "derivations.hh" diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh new file mode 100644 index 000000000..8eccd8236 --- /dev/null +++ b/src/libstore/local-fs-store.hh @@ -0,0 +1,48 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +struct LocalFSStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes + // it to omit the call to the Setting constructor. Clang works fine + // either way. + const PathSetting rootDir{(StoreConfig*) this, true, "", + "root", "directory prefixed to all other paths"}; + const PathSetting stateDir{(StoreConfig*) this, false, + rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + "state", "directory where Nix will store state"}; + const PathSetting logDir{(StoreConfig*) this, false, + rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + "log", "directory where Nix will store state"}; +}; + +class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +{ +public: + + const static string drvsLogDir; + + LocalFSStore(const Params & params); + + void narFromPath(const StorePath & path, Sink & sink) override; + ref getFSAccessor() override; + + /* Register a permanent GC root. */ + Path addPermRoot(const StorePath & storePath, const Path & gcRoot); + + virtual Path getRealStoreDir() { return storeDir; } + + Path toRealPath(const Path & storePath) override + { + assert(isInStore(storePath)); + return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); + } + + std::shared_ptr getBuildLog(const StorePath & path) override; +}; + +} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e7c9d1605..f1e2ab7f9 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -4,6 +4,7 @@ #include "pathlocks.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "sync.hh" #include "util.hh" diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index c3809bad7..ed10dd519 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,5 +1,6 @@ #include "profiles.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "util.hh" #include diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index ec04be985..554fb6bed 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -4,6 +4,7 @@ #include #include "store-api.hh" +#include "local-fs-store.hh" namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 450c0f554..f77bc21d1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -715,47 +715,6 @@ protected: }; -struct LocalFSStoreConfig : virtual StoreConfig -{ - using StoreConfig::StoreConfig; - // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes - // it to omit the call to the Setting constructor. Clang works fine - // either way. - const PathSetting rootDir{(StoreConfig*) this, true, "", - "root", "directory prefixed to all other paths"}; - const PathSetting stateDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, - "state", "directory where Nix will store state"}; - const PathSetting logDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, - "log", "directory where Nix will store state"}; -}; - -class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig -{ -public: - - const static string drvsLogDir; - - LocalFSStore(const Params & params); - - void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; - - /* Register a permanent GC root. */ - Path addPermRoot(const StorePath & storePath, const Path & gcRoot); - - virtual Path getRealStoreDir() { return storeDir; } - - Path toRealPath(const Path & storePath) override - { - assert(isInStore(storePath)); - return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); - } - - std::shared_ptr getBuildLog(const StorePath & path) override; -}; - /* Copy a path from one store to another. */ void copyStorePath(ref srcStore, ref dstStore, diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index a79b1086b..f60e0706c 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -6,6 +6,7 @@ #include #include "store-api.hh" +#include "local-fs-store.hh" #include "globals.hh" #include "derivations.hh" #include "affinity.hh" diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e6667e7f5..a4b5c9e2c 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -8,6 +8,7 @@ #include "profiles.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "user-env.hh" #include "util.hh" #include "json.hh" diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 8c6c8af05..87387e794 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "derivations.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "globals.hh" #include "shared.hh" #include "eval.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 18a0049a6..3956fef6d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -8,6 +8,7 @@ #include "value-to-json.hh" #include "util.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "common-eval-args.hh" #include "../nix/legacy.hh" diff --git a/src/nix/build.cc b/src/nix/build.cc index d85a482db..65708e98b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -3,6 +3,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" using namespace nix; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fc41da9e4..2d0a0b6ea 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "fs-accessor.hh" using namespace nix; diff --git a/src/nix/command.cc b/src/nix/command.cc index ba7de9fdd..9a38c77f1 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "derivations.hh" #include "nixexpr.hh" #include "profiles.hh" diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 4588ac05e..4f3003448 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -5,6 +5,7 @@ #include "serve-protocol.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "util.hh" #include "worker-protocol.hh" From 38e3897162825e8b3a7005df140a04b9acf0a2fb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 17:18:26 +0000 Subject: [PATCH 122/123] Copy {uds-,}remote-store.{cc,hh} This prepares for the splitting that happens in the next commit. --- src/libstore/uds-remote-store.cc | 1017 ++++++++++++++++++++++++++++++ src/libstore/uds-remote-store.hh | 204 ++++++ 2 files changed, 1221 insertions(+) create mode 100644 src/libstore/uds-remote-store.cc create mode 100644 src/libstore/uds-remote-store.hh diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc new file mode 100644 index 000000000..23b1942ce --- /dev/null +++ b/src/libstore/uds-remote-store.cc @@ -0,0 +1,1017 @@ +#include "serialise.hh" +#include "util.hh" +#include "remote-fs-accessor.hh" +#include "remote-store.hh" +#include "worker-protocol.hh" +#include "archive.hh" +#include "affinity.hh" +#include "globals.hh" +#include "derivations.hh" +#include "pool.hh" +#include "finally.hh" +#include "logging.hh" +#include "callback.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace nix { + +namespace worker_proto { + +std::string read(const Store & store, Source & from, Phantom _) +{ + return readString(from); +} + +void write(const Store & store, Sink & out, const std::string & str) +{ + out << str; +} + + +StorePath read(const Store & store, Source & from, Phantom _) +{ + return store.parseStorePath(readString(from)); +} + +void write(const Store & store, Sink & out, const StorePath & storePath) +{ + out << store.printStorePath(storePath); +} + + +ContentAddress read(const Store & store, Source & from, Phantom _) +{ + return parseContentAddress(readString(from)); +} + +void write(const Store & store, Sink & out, const ContentAddress & ca) +{ + out << renderContentAddress(ca); +} + + +std::optional read(const Store & store, Source & from, Phantom> _) +{ + auto s = readString(from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void write(const Store & store, Sink & out, const std::optional & storePathOpt) +{ + out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional read(const Store & store, Source & from, Phantom> _) +{ + return parseContentAddressOpt(readString(from)); +} + +void write(const Store & store, Sink & out, const std::optional & caOpt) +{ + out << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} + + +/* TODO: Separate these store impls into different files, give them better names */ +RemoteStore::RemoteStore(const Params & params) + : Store(params) + , RemoteStoreConfig(params) + , connections(make_ref>( + std::max(1, (int) maxConnections), + [this]() { + auto conn = openConnectionWrapper(); + try { + initConnection(*conn); + } catch (...) { + failed = true; + throw; + } + return conn; + }, + [this](const ref & r) { + return + r->to.good() + && r->from.good() + && std::chrono::duration_cast( + std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; + } + )) +{ +} + + +ref RemoteStore::openConnectionWrapper() +{ + if (failed) + throw Error("opening a connection to remote store '%s' previously failed", getUri()); + try { + return openConnection(); + } catch (...) { + failed = true; + throw; + } +} + + +UDSRemoteStore::UDSRemoteStore(const Params & params) + : StoreConfig(params) + , Store(params) + , LocalFSStore(params) + , RemoteStore(params) +{ +} + + +UDSRemoteStore::UDSRemoteStore( + const std::string scheme, + std::string socket_path, + const Params & params) + : UDSRemoteStore(params) +{ + path.emplace(socket_path); +} + + +std::string UDSRemoteStore::getUri() +{ + if (path) { + return std::string("unix://") + *path; + } else { + return "daemon"; + } +} + + +ref UDSRemoteStore::openConnection() +{ + auto conn = make_ref(); + + /* Connect to a daemon that does the privileged work for us. */ + conn->fd = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); + if (!conn->fd) + throw SysError("cannot create Unix domain socket"); + closeOnExec(conn->fd.get()); + + string socketPath = path ? *path : settings.nixDaemonSocketFile; + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPath.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%1%' is too long", socketPath); + strcpy(addr.sun_path, socketPath.c_str()); + + if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to daemon at '%1%'", socketPath); + + conn->from.fd = conn->fd.get(); + conn->to.fd = conn->fd.get(); + + conn->startTime = std::chrono::steady_clock::now(); + + return conn; +} + + +void RemoteStore::initConnection(Connection & conn) +{ + /* Send the magic greeting, check for the reply. */ + try { + conn.to << WORKER_MAGIC_1; + conn.to.flush(); + unsigned int magic = readInt(conn.from); + if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); + + conn.from >> conn.daemonVersion; + if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + throw Error("Nix daemon protocol version not supported"); + if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) + throw Error("the Nix daemon version is too old"); + conn.to << PROTOCOL_VERSION; + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { + int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; + if (cpu != -1) + conn.to << 1 << cpu; + else + conn.to << 0; + } + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) + conn.to << false; + + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); + } + catch (Error & e) { + throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); + } + + setOptions(conn); +} + + +void RemoteStore::setOptions(Connection & conn) +{ + conn.to << wopSetOptions + << settings.keepFailed + << settings.keepGoing + << settings.tryFallback + << verbosity + << settings.maxBuildJobs + << settings.maxSilentTime + << true + << (settings.verboseBuild ? lvlError : lvlVomit) + << 0 // obsolete log type + << 0 /* obsolete print build trace */ + << settings.buildCores + << settings.useSubstitutes; + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + std::map overrides; + globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); + overrides.erase(loggerSettings.showTrace.name); + conn.to << overrides.size(); + for (auto & i : overrides) + conn.to << i.first << i.second.value; + } + + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); +} + + +/* A wrapper around Pool::Handle that marks + the connection as bad (causing it to be closed) if a non-daemon + exception is thrown before the handle is closed. Such an exception + causes a deviation from the expected protocol and therefore a + desynchronization between the client and daemon. */ +struct ConnectionHandle +{ + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle() + { + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); + } + } + + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) + { + auto ex = handle->processStderr(sink, source, flush); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); + } + } + + void withFramedSink(std::function fun); +}; + + +ConnectionHandle RemoteStore::getConnection() +{ + return ConnectionHandle(connections->get()); +} + + +bool RemoteStore::isValidPathUncached(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopIsValidPath << printStorePath(path); + conn.processStderr(); + return readInt(conn->from); +} + + +StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + StorePathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; + } else { + conn->to << wopQueryValidPaths; + worker_proto::write(*this, conn->to, paths); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); + } +} + + +StorePathSet RemoteStore::queryAllValidPaths() +{ + auto conn(getConnection()); + conn->to << wopQueryAllValidPaths; + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); +} + + +StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) +{ + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + StorePathSet res; + for (auto & i : paths) { + conn->to << wopHasSubstitutes << printStorePath(i); + conn.processStderr(); + if (readInt(conn->from)) res.insert(i); + } + return res; + } else { + conn->to << wopQuerySubstitutablePaths; + worker_proto::write(*this, conn->to, paths); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); + } +} + + +void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) +{ + if (pathsMap.empty()) return; + + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + + for (auto & i : pathsMap) { + SubstitutablePathInfo info; + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); + conn.processStderr(); + unsigned int reply = readInt(conn->from); + if (reply == 0) continue; + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); + infos.insert_or_assign(i.first, std::move(info)); + } + + } else { + + conn->to << wopQuerySubstitutablePathInfos; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); + worker_proto::write(*this, conn->to, paths); + } else + worker_proto::write(*this, conn->to, pathsMap); + conn.processStderr(); + size_t count = readNum(conn->from); + for (size_t n = 0; n < count; n++) { + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); + } + + } +} + + +ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) +{ + auto deriver = readString(conn->from); + auto narHash = Hash::parseAny(readString(conn->from), htSHA256); + auto info = make_ref(path, narHash); + if (deriver != "") info->deriver = parseStorePath(deriver); + info->references = worker_proto::read(*this, conn->from, Phantom {}); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings(conn->from); + info->ca = parseContentAddressOpt(readString(conn->from)); + } + return info; +} + + +void RemoteStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + try { + std::shared_ptr info; + { + auto conn(getConnection()); + conn->to << wopQueryPathInfo << printStorePath(path); + try { + conn.processStderr(); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(e.info()); + throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid; conn->from >> valid; + if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); + } + info = readValidPathInfo(conn, path); + } + callback(std::move(info)); + } catch (...) { callback.rethrow(); } +} + + +void RemoteStore::queryReferrers(const StorePath & path, + StorePathSet & referrers) +{ + auto conn(getConnection()); + conn->to << wopQueryReferrers << printStorePath(path); + conn.processStderr(); + for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) + referrers.insert(i); +} + + +StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopQueryValidDerivers << printStorePath(path); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); +} + + +StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) +{ + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { + return Store::queryDerivationOutputs(path); + } + conn->to << wopQueryDerivationOutputs << printStorePath(path); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); +} + + +std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) +{ + if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom>> {}); + } else { + // Fallback for old daemon versions. + // For floating-CA derivations (and their co-dependencies) this is an + // under-approximation as it only returns the paths that can be inferred + // from the derivation itself (and not the ones that are known because + // the have been built), but as old stores don't handle floating-CA + // derivations this shouldn't matter + auto derivation = readDerivation(path); + auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); + std::map> ret; + for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { + ret.emplace(outputName, outputAndPath.second); + } + return ret; + } +} + +std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) +{ + auto conn(getConnection()); + conn->to << wopQueryPathFromHashPart << hashPart; + conn.processStderr(); + Path path = readString(conn->from); + if (path.empty()) return {}; + return parseStorePath(path); +} + + +ref RemoteStore::addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair) +{ + std::optional conn_(getConnection()); + auto & conn = *conn_; + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + + conn->to + << wopAddToStore + << name + << renderContentAddressMethod(caMethod); + worker_proto::write(*this, conn->to, references); + conn->to << repair; + + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + + auto path = parseStorePath(readString(conn->from)); + return readValidPathInfo(conn, path); + } + else { + if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); + + std::visit(overloaded { + [&](TextHashMethod thm) -> void { + std::string s = dump.drain(); + conn->to << wopAddTextToStore << name << s; + worker_proto::write(*this, conn->to, references); + conn.processStderr(); + }, + [&](FixedOutputHashMethod fohm) -> void { + conn->to + << wopAddToStore + << name + << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(fohm.hashType); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + dump.drainInto(conn->to); + } else { + std::string contents = dump.drain(); + dumpString(contents, conn->to); + } + } + conn->to.warn = false; + conn.processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + + } + }, caMethod); + auto path = parseStorePath(readString(conn->from)); + // Release our connection to prevent a deadlock in queryPathInfo(). + conn_.reset(); + return queryPathInfo(path); + } +} + + +StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashType, RepairFlag repair) +{ + StorePathSet references; + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; +} + + +void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { + conn->to << wopImportPaths; + + auto source2 = sinkToSource([&](Sink & sink) { + sink << 1 // == path follows + ; + copyNAR(source, sink); + sink + << exportMagic + << printStorePath(info.path); + worker_proto::write(*this, sink, info.references); + sink + << (info.deriver ? printStorePath(*info.deriver) : "") + << 0 // == no legacy signature + << 0 // == no path follows + ; + }); + + conn.processStderr(0, source2.get()); + + auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); + assert(importedPaths.size() <= 1); + } + + else { + conn->to << wopAddToStoreNar + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(Base16, false); + worker_proto::write(*this, conn->to, info.references); + conn->to << info.registrationTime << info.narSize + << info.ultimate << info.sigs << renderContentAddress(info.ca) + << repair << !checkSigs; + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + conn.withFramedSink([&](Sink & sink) { + copyNAR(source, sink); + }); + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); + } + } +} + + +StorePath RemoteStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) +{ + StringSource source(s); + return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; +} + + +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +{ + auto conn(getConnection()); + conn->to << wopBuildPaths; + assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); + Strings ss; + for (auto & p : drvPaths) + ss.push_back(p.to_string(*this)); + conn->to << ss; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << buildMode; + else + /* Old daemons did not take a 'buildMode' parameter, so we + need to validate it here on the client side. */ + if (buildMode != bmNormal) + throw Error("repairing or checking is not supported when building through the Nix daemon"); + conn.processStderr(); + readInt(conn->from); +} + + +BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + auto conn(getConnection()); + conn->to << wopBuildDerivation << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + conn->to << buildMode; + conn.processStderr(); + BuildResult res; + unsigned int status; + conn->from >> status >> res.errorMsg; + res.status = (BuildResult::Status) status; + return res; +} + + +void RemoteStore::ensurePath(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopEnsurePath << printStorePath(path); + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::addTempRoot(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopAddTempRoot << printStorePath(path); + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::addIndirectRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << wopAddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::syncWithGC() +{ + auto conn(getConnection()); + conn->to << wopSyncWithGC; + conn.processStderr(); + readInt(conn->from); +} + + +Roots RemoteStore::findRoots(bool censor) +{ + auto conn(getConnection()); + conn->to << wopFindRoots; + conn.processStderr(); + size_t count = readNum(conn->from); + Roots result; + while (count--) { + Path link = readString(conn->from); + auto target = parseStorePath(readString(conn->from)); + result[std::move(target)].emplace(link); + } + return result; +} + + +void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + auto conn(getConnection()); + + conn->to + << wopCollectGarbage << options.action; + worker_proto::write(*this, conn->to, options.pathsToDelete); + conn->to << options.ignoreLiveness + << options.maxFreed + /* removed options */ + << 0 << 0 << 0; + + conn.processStderr(); + + results.paths = readStrings(conn->from); + results.bytesFreed = readLongLong(conn->from); + readLongLong(conn->from); // obsolete + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.clear(); + } +} + + +void RemoteStore::optimiseStore() +{ + auto conn(getConnection()); + conn->to << wopOptimiseStore; + conn.processStderr(); + readInt(conn->from); +} + + +bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) +{ + auto conn(getConnection()); + conn->to << wopVerifyStore << checkContents << repair; + conn.processStderr(); + return readInt(conn->from); +} + + +void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) +{ + auto conn(getConnection()); + conn->to << wopAddSignatures << printStorePath(storePath) << sigs; + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) +{ + { + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + // Don't hold the connection handle in the fallback case + // to prevent a deadlock. + goto fallback; + conn->to << wopQueryMissing; + Strings ss; + for (auto & p : targets) + ss.push_back(p.to_string(*this)); + conn->to << ss; + conn.processStderr(); + willBuild = worker_proto::read(*this, conn->from, Phantom {}); + willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); + unknown = worker_proto::read(*this, conn->from, Phantom {}); + conn->from >> downloadSize >> narSize; + return; + } + + fallback: + return Store::queryMissing(targets, willBuild, willSubstitute, + unknown, downloadSize, narSize); +} + + +void RemoteStore::connect() +{ + auto conn(getConnection()); +} + + +unsigned int RemoteStore::getProtocol() +{ + auto conn(connections->get()); + return conn->daemonVersion; +} + + +void RemoteStore::flushBadConnections() +{ + connections->flushBad(); +} + + +RemoteStore::Connection::~Connection() +{ + try { + to.flush(); + } catch (...) { + ignoreException(); + } +} + +void RemoteStore::narFromPath(const StorePath & path, Sink & sink) +{ + auto conn(connections->get()); + conn->to << wopNarFromPath << printStorePath(path); + conn->processStderr(); + copyNAR(conn->from, sink); +} + +ref RemoteStore::getFSAccessor() +{ + return make_ref(ref(shared_from_this())); +} + +static Logger::Fields readFields(Source & from) +{ + Logger::Fields fields; + size_t size = readInt(from); + for (size_t n = 0; n < size; n++) { + auto type = (decltype(Logger::Field::type)) readInt(from); + if (type == Logger::Field::tInt) + fields.push_back(readNum(from)); + else if (type == Logger::Field::tString) + fields.push_back(readString(from)); + else + throw Error("got unsupported field type %x from Nix daemon", (int) type); + } + return fields; +} + + +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush) +{ + if (flush) + to.flush(); + + while (true) { + + auto msg = readNum(from); + + if (msg == STDERR_WRITE) { + string s = readString(from); + if (!sink) throw Error("no sink"); + (*sink)(s); + } + + else if (msg == STDERR_READ) { + if (!source) throw Error("no source"); + size_t len = readNum(from); + auto buf = std::make_unique(len); + writeString(buf.get(), source->read(buf.get(), len), to); + to.flush(); + } + + else if (msg == STDERR_ERROR) { + if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + return std::make_exception_ptr(readError(from)); + } else { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } + } + + else if (msg == STDERR_NEXT) + printError(chomp(readString(from))); + + else if (msg == STDERR_START_ACTIVITY) { + auto act = readNum(from); + auto lvl = (Verbosity) readInt(from); + auto type = (ActivityType) readInt(from); + auto s = readString(from); + auto fields = readFields(from); + auto parent = readNum(from); + logger->startActivity(act, lvl, type, s, fields, parent); + } + + else if (msg == STDERR_STOP_ACTIVITY) { + auto act = readNum(from); + logger->stopActivity(act); + } + + else if (msg == STDERR_RESULT) { + auto act = readNum(from); + auto type = (ResultType) readInt(from); + auto fields = readFields(from); + logger->result(act, type, fields); + } + + else if (msg == STDERR_LAST) + break; + + else + throw Error("got unknown message type %x from Nix daemon", msg); + } + + return nullptr; +} + +void ConnectionHandle::withFramedSink(std::function fun) +{ + (*this)->to.flush(); + + std::exception_ptr ex; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + processStderr(nullptr, nullptr, false); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink((*this)->to, ex); + fun(sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + +} + +static RegisterStoreImplementation regUDSRemoteStore; + +} diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh new file mode 100644 index 000000000..554fb6bed --- /dev/null +++ b/src/libstore/uds-remote-store.hh @@ -0,0 +1,204 @@ +#pragma once + +#include +#include + +#include "store-api.hh" +#include "local-fs-store.hh" + + +namespace nix { + + +class Pipe; +class Pid; +struct FdSink; +struct FdSource; +template class Pool; +struct ConnectionHandle; + +struct RemoteStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + + const Setting maxConnections{(StoreConfig*) this, 1, + "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + + const Setting maxConnectionAge{(StoreConfig*) this, std::numeric_limits::max(), + "max-connection-age", "number of seconds to reuse a connection"}; +}; + +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +{ +public: + + virtual bool sameMachine() = 0; + + RemoteStore(const Params & params); + + /* Implementations of abstract store API methods. */ + + bool isValidPathUncached(const StorePath & path) override; + + StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + + StorePathSet queryAllValidPaths() override; + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; + + StorePathSet queryValidDerivers(const StorePath & path) override; + + StorePathSet queryDerivationOutputs(const StorePath & path) override; + + std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::optional queryPathFromHashPart(const std::string & hashPart) override; + + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; + + void querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) override; + + /* Add a content-addressable store path. `dump` will be drained. */ + ref addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair); + + /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + + void addToStore(const ValidPathInfo & info, Source & nar, + RepairFlag repair, CheckSigsFlag checkSigs) override; + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; + + void buildPaths(const std::vector & paths, BuildMode buildMode) override; + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; + + void ensurePath(const StorePath & path) override; + + void addTempRoot(const StorePath & path) override; + + void addIndirectRoot(const Path & path) override; + + void syncWithGC() override; + + Roots findRoots(bool censor) override; + + void collectGarbage(const GCOptions & options, GCResults & results) override; + + void optimiseStore() override; + + bool verifyStore(bool checkContents, RepairFlag repair) override; + + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; + + void queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) override; + + void connect() override; + + unsigned int getProtocol() override; + + void flushBadConnections(); + + struct Connection + { + AutoCloseFD fd; + FdSink to; + FdSource from; + unsigned int daemonVersion; + std::chrono::time_point startTime; + + virtual ~Connection(); + + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + }; + + ref openConnectionWrapper(); + +protected: + + virtual ref openConnection() = 0; + + void initConnection(Connection & conn); + + ref> connections; + + virtual void setOptions(Connection & conn); + + ConnectionHandle getConnection(); + + friend struct ConnectionHandle; + + virtual ref getFSAccessor() override; + + virtual void narFromPath(const StorePath & path, Sink & sink) override; + + ref readValidPathInfo(ConnectionHandle & conn, const StorePath & path); + +private: + + std::atomic_bool failed{false}; + +}; + +struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +{ + UDSRemoteStoreConfig(const Store::Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + { + } + + UDSRemoteStoreConfig() + : UDSRemoteStoreConfig(Store::Params({})) + { + } + + const std::string name() override { return "Local Daemon Store"; } +}; + +class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig +{ +public: + + UDSRemoteStore(const Params & params); + UDSRemoteStore(const std::string scheme, std::string path, const Params & params); + + std::string getUri() override; + + static std::set uriSchemes() + { return {"unix"}; } + + bool sameMachine() override + { return true; } + + ref getFSAccessor() override + { return LocalFSStore::getFSAccessor(); } + + void narFromPath(const StorePath & path, Sink & sink) override + { LocalFSStore::narFromPath(path, sink); } + +private: + + ref openConnection() override; + std::optional path; +}; + + +} From 15fdb7cc6bb9e86283a67cce646981821dca1558 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:01:53 +0000 Subject: [PATCH 123/123] Split out uds-remote-store.{cc.hh} --- src/libstore/remote-store.cc | 75 --- src/libstore/remote-store.hh | 45 -- src/libstore/store-api.cc | 2 +- src/libstore/uds-remote-store.cc | 940 +------------------------------ src/libstore/uds-remote-store.hh | 154 +---- 5 files changed, 4 insertions(+), 1212 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 23b1942ce..488270f48 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -12,16 +12,6 @@ #include "logging.hh" #include "callback.hh" -#include -#include -#include -#include -#include -#include -#include - -#include - namespace nix { namespace worker_proto { @@ -125,69 +115,6 @@ ref RemoteStore::openConnectionWrapper() } -UDSRemoteStore::UDSRemoteStore(const Params & params) - : StoreConfig(params) - , Store(params) - , LocalFSStore(params) - , RemoteStore(params) -{ -} - - -UDSRemoteStore::UDSRemoteStore( - const std::string scheme, - std::string socket_path, - const Params & params) - : UDSRemoteStore(params) -{ - path.emplace(socket_path); -} - - -std::string UDSRemoteStore::getUri() -{ - if (path) { - return std::string("unix://") + *path; - } else { - return "daemon"; - } -} - - -ref UDSRemoteStore::openConnection() -{ - auto conn = make_ref(); - - /* Connect to a daemon that does the privileged work for us. */ - conn->fd = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!conn->fd) - throw SysError("cannot create Unix domain socket"); - closeOnExec(conn->fd.get()); - - string socketPath = path ? *path : settings.nixDaemonSocketFile; - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%1%' is too long", socketPath); - strcpy(addr.sun_path, socketPath.c_str()); - - if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to daemon at '%1%'", socketPath); - - conn->from.fd = conn->fd.get(); - conn->to.fd = conn->fd.get(); - - conn->startTime = std::chrono::steady_clock::now(); - - return conn; -} - - void RemoteStore::initConnection(Connection & conn) { /* Send the magic greeting, check for the reply. */ @@ -1012,6 +939,4 @@ void ConnectionHandle::withFramedSink(std::function fun) } -static RegisterStoreImplementation regUDSRemoteStore; - } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 554fb6bed..9f78fcb02 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -4,7 +4,6 @@ #include #include "store-api.hh" -#include "local-fs-store.hh" namespace nix { @@ -156,49 +155,5 @@ private: }; -struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig -{ - UDSRemoteStoreConfig(const Store::Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - { - } - - UDSRemoteStoreConfig() - : UDSRemoteStoreConfig(Store::Params({})) - { - } - - const std::string name() override { return "Local Daemon Store"; } -}; - -class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig -{ -public: - - UDSRemoteStore(const Params & params); - UDSRemoteStore(const std::string scheme, std::string path, const Params & params); - - std::string getUri() override; - - static std::set uriSchemes() - { return {"unix"}; } - - bool sameMachine() override - { return true; } - - ref getFSAccessor() override - { return LocalFSStore::getFSAccessor(); } - - void narFromPath(const StorePath & path, Sink & sink) override - { LocalFSStore::narFromPath(path, sink); } - -private: - - ref openConnection() override; - std::optional path; -}; - } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1bbc74db8..9f21f0434 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1011,7 +1011,7 @@ Derivation Store::readDerivation(const StorePath & drvPath) #include "local-store.hh" -#include "remote-store.hh" +#include "uds-remote-store.hh" namespace nix { diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 23b1942ce..24f3e9c6d 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,16 +1,4 @@ -#include "serialise.hh" -#include "util.hh" -#include "remote-fs-accessor.hh" -#include "remote-store.hh" -#include "worker-protocol.hh" -#include "archive.hh" -#include "affinity.hh" -#include "globals.hh" -#include "derivations.hh" -#include "pool.hh" -#include "finally.hh" -#include "logging.hh" -#include "callback.hh" +#include "uds-remote-store.hh" #include #include @@ -22,109 +10,9 @@ #include + namespace nix { -namespace worker_proto { - -std::string read(const Store & store, Source & from, Phantom _) -{ - return readString(from); -} - -void write(const Store & store, Sink & out, const std::string & str) -{ - out << str; -} - - -StorePath read(const Store & store, Source & from, Phantom _) -{ - return store.parseStorePath(readString(from)); -} - -void write(const Store & store, Sink & out, const StorePath & storePath) -{ - out << store.printStorePath(storePath); -} - - -ContentAddress read(const Store & store, Source & from, Phantom _) -{ - return parseContentAddress(readString(from)); -} - -void write(const Store & store, Sink & out, const ContentAddress & ca) -{ - out << renderContentAddress(ca); -} - - -std::optional read(const Store & store, Source & from, Phantom> _) -{ - auto s = readString(from); - return s == "" ? std::optional {} : store.parseStorePath(s); -} - -void write(const Store & store, Sink & out, const std::optional & storePathOpt) -{ - out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); -} - - -std::optional read(const Store & store, Source & from, Phantom> _) -{ - return parseContentAddressOpt(readString(from)); -} - -void write(const Store & store, Sink & out, const std::optional & caOpt) -{ - out << (caOpt ? renderContentAddress(*caOpt) : ""); -} - -} - - -/* TODO: Separate these store impls into different files, give them better names */ -RemoteStore::RemoteStore(const Params & params) - : Store(params) - , RemoteStoreConfig(params) - , connections(make_ref>( - std::max(1, (int) maxConnections), - [this]() { - auto conn = openConnectionWrapper(); - try { - initConnection(*conn); - } catch (...) { - failed = true; - throw; - } - return conn; - }, - [this](const ref & r) { - return - r->to.good() - && r->from.good() - && std::chrono::duration_cast( - std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; - } - )) -{ -} - - -ref RemoteStore::openConnectionWrapper() -{ - if (failed) - throw Error("opening a connection to remote store '%s' previously failed", getUri()); - try { - return openConnection(); - } catch (...) { - failed = true; - throw; - } -} - - UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) , Store(params) @@ -188,830 +76,6 @@ ref UDSRemoteStore::openConnection() } -void RemoteStore::initConnection(Connection & conn) -{ - /* Send the magic greeting, check for the reply. */ - try { - conn.to << WORKER_MAGIC_1; - conn.to.flush(); - unsigned int magic = readInt(conn.from); - if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - - conn.from >> conn.daemonVersion; - if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) - throw Error("Nix daemon protocol version not supported"); - if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) - throw Error("the Nix daemon version is too old"); - conn.to << PROTOCOL_VERSION; - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; - if (cpu != -1) - conn.to << 1 << cpu; - else - conn.to << 0; - } - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) - conn.to << false; - - auto ex = conn.processStderr(); - if (ex) std::rethrow_exception(ex); - } - catch (Error & e) { - throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); - } - - setOptions(conn); -} - - -void RemoteStore::setOptions(Connection & conn) -{ - conn.to << wopSetOptions - << settings.keepFailed - << settings.keepGoing - << settings.tryFallback - << verbosity - << settings.maxBuildJobs - << settings.maxSilentTime - << true - << (settings.verboseBuild ? lvlError : lvlVomit) - << 0 // obsolete log type - << 0 /* obsolete print build trace */ - << settings.buildCores - << settings.useSubstitutes; - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - std::map overrides; - globalConfig.getSettings(overrides, true); - overrides.erase(settings.keepFailed.name); - overrides.erase(settings.keepGoing.name); - overrides.erase(settings.tryFallback.name); - overrides.erase(settings.maxBuildJobs.name); - overrides.erase(settings.maxSilentTime.name); - overrides.erase(settings.buildCores.name); - overrides.erase(settings.useSubstitutes.name); - overrides.erase(loggerSettings.showTrace.name); - conn.to << overrides.size(); - for (auto & i : overrides) - conn.to << i.first << i.second.value; - } - - auto ex = conn.processStderr(); - if (ex) std::rethrow_exception(ex); -} - - -/* A wrapper around Pool::Handle that marks - the connection as bad (causing it to be closed) if a non-daemon - exception is thrown before the handle is closed. Such an exception - causes a deviation from the expected protocol and therefore a - desynchronization between the client and daemon. */ -struct ConnectionHandle -{ - Pool::Handle handle; - bool daemonException = false; - - ConnectionHandle(Pool::Handle && handle) - : handle(std::move(handle)) - { } - - ConnectionHandle(ConnectionHandle && h) - : handle(std::move(h.handle)) - { } - - ~ConnectionHandle() - { - if (!daemonException && std::uncaught_exceptions()) { - handle.markBad(); - debug("closing daemon connection because of an exception"); - } - } - - RemoteStore::Connection * operator -> () { return &*handle; } - - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) - { - auto ex = handle->processStderr(sink, source, flush); - if (ex) { - daemonException = true; - std::rethrow_exception(ex); - } - } - - void withFramedSink(std::function fun); -}; - - -ConnectionHandle RemoteStore::getConnection() -{ - return ConnectionHandle(connections->get()); -} - - -bool RemoteStore::isValidPathUncached(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopIsValidPath << printStorePath(path); - conn.processStderr(); - return readInt(conn->from); -} - - -StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) -{ - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - StorePathSet res; - for (auto & i : paths) - if (isValidPath(i)) res.insert(i); - return res; - } else { - conn->to << wopQueryValidPaths; - worker_proto::write(*this, conn->to, paths); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); - } -} - - -StorePathSet RemoteStore::queryAllValidPaths() -{ - auto conn(getConnection()); - conn->to << wopQueryAllValidPaths; - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); -} - - -StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) -{ - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - StorePathSet res; - for (auto & i : paths) { - conn->to << wopHasSubstitutes << printStorePath(i); - conn.processStderr(); - if (readInt(conn->from)) res.insert(i); - } - return res; - } else { - conn->to << wopQuerySubstitutablePaths; - worker_proto::write(*this, conn->to, paths); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); - } -} - - -void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) -{ - if (pathsMap.empty()) return; - - auto conn(getConnection()); - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - - for (auto & i : pathsMap) { - SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); - conn.processStderr(); - unsigned int reply = readInt(conn->from); - if (reply == 0) continue; - auto deriver = readString(conn->from); - if (deriver != "") - info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); - info.downloadSize = readLongLong(conn->from); - info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.first, std::move(info)); - } - - } else { - - conn->to << wopQuerySubstitutablePathInfos; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); - worker_proto::write(*this, conn->to, paths); - } else - worker_proto::write(*this, conn->to, pathsMap); - conn.processStderr(); - size_t count = readNum(conn->from); - for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); - auto deriver = readString(conn->from); - if (deriver != "") - info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); - info.downloadSize = readLongLong(conn->from); - info.narSize = readLongLong(conn->from); - } - - } -} - - -ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) -{ - auto deriver = readString(conn->from); - auto narHash = Hash::parseAny(readString(conn->from), htSHA256); - auto info = make_ref(path, narHash); - if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom {}); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); - } - return info; -} - - -void RemoteStore::queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept -{ - try { - std::shared_ptr info; - { - auto conn(getConnection()); - conn->to << wopQueryPathInfo << printStorePath(path); - try { - conn.processStderr(); - } catch (Error & e) { - // Ugly backwards compatibility hack. - if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.info()); - throw; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { - bool valid; conn->from >> valid; - if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); - } - info = readValidPathInfo(conn, path); - } - callback(std::move(info)); - } catch (...) { callback.rethrow(); } -} - - -void RemoteStore::queryReferrers(const StorePath & path, - StorePathSet & referrers) -{ - auto conn(getConnection()); - conn->to << wopQueryReferrers << printStorePath(path); - conn.processStderr(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) - referrers.insert(i); -} - - -StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopQueryValidDerivers << printStorePath(path); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); -} - - -StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) -{ - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { - return Store::queryDerivationOutputs(path); - } - conn->to << wopQueryDerivationOutputs << printStorePath(path); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); -} - - -std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) -{ - if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom>> {}); - } else { - // Fallback for old daemon versions. - // For floating-CA derivations (and their co-dependencies) this is an - // under-approximation as it only returns the paths that can be inferred - // from the derivation itself (and not the ones that are known because - // the have been built), but as old stores don't handle floating-CA - // derivations this shouldn't matter - auto derivation = readDerivation(path); - auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); - std::map> ret; - for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { - ret.emplace(outputName, outputAndPath.second); - } - return ret; - } -} - -std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) -{ - auto conn(getConnection()); - conn->to << wopQueryPathFromHashPart << hashPart; - conn.processStderr(); - Path path = readString(conn->from); - if (path.empty()) return {}; - return parseStorePath(path); -} - - -ref RemoteStore::addCAToStore( - Source & dump, - const string & name, - ContentAddressMethod caMethod, - const StorePathSet & references, - RepairFlag repair) -{ - std::optional conn_(getConnection()); - auto & conn = *conn_; - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { - - conn->to - << wopAddToStore - << name - << renderContentAddressMethod(caMethod); - worker_proto::write(*this, conn->to, references); - conn->to << repair; - - conn.withFramedSink([&](Sink & sink) { - dump.drainInto(sink); - }); - - auto path = parseStorePath(readString(conn->from)); - return readValidPathInfo(conn, path); - } - else { - if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); - - std::visit(overloaded { - [&](TextHashMethod thm) -> void { - std::string s = dump.drain(); - conn->to << wopAddTextToStore << name << s; - worker_proto::write(*this, conn->to, references); - conn.processStderr(); - }, - [&](FixedOutputHashMethod fohm) -> void { - conn->to - << wopAddToStore - << name - << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(fohm.hashType); - - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { - dump.drainInto(conn->to); - } else { - std::string contents = dump.drain(); - dumpString(contents, conn->to); - } - } - conn->to.warn = false; - conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) - try { - conn.processStderr(); - } catch (EndOfFile & e) { } - throw; - } - - } - }, caMethod); - auto path = parseStorePath(readString(conn->from)); - // Release our connection to prevent a deadlock in queryPathInfo(). - conn_.reset(); - return queryPathInfo(path); - } -} - - -StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair) -{ - StorePathSet references; - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; -} - - -void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) -{ - auto conn(getConnection()); - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { - conn->to << wopImportPaths; - - auto source2 = sinkToSource([&](Sink & sink) { - sink << 1 // == path follows - ; - copyNAR(source, sink); - sink - << exportMagic - << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); - sink - << (info.deriver ? printStorePath(*info.deriver) : "") - << 0 // == no legacy signature - << 0 // == no path follows - ; - }); - - conn.processStderr(0, source2.get()); - - auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); - assert(importedPaths.size() <= 1); - } - - else { - conn->to << wopAddToStoreNar - << printStorePath(info.path) - << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); - conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderContentAddress(info.ca) - << repair << !checkSigs; - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { - conn.withFramedSink([&](Sink & sink) { - copyNAR(source, sink); - }); - } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { - conn.processStderr(0, &source); - } else { - copyNAR(source, conn->to); - conn.processStderr(0, nullptr); - } - } -} - - -StorePath RemoteStore::addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) -{ - StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; -} - - -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - auto conn(getConnection()); - conn->to << wopBuildPaths; - assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); - conn->to << ss; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) - conn->to << buildMode; - else - /* Old daemons did not take a 'buildMode' parameter, so we - need to validate it here on the client side. */ - if (buildMode != bmNormal) - throw Error("repairing or checking is not supported when building through the Nix daemon"); - conn.processStderr(); - readInt(conn->from); -} - - -BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - auto conn(getConnection()); - conn->to << wopBuildDerivation << printStorePath(drvPath); - writeDerivation(conn->to, *this, drv); - conn->to << buildMode; - conn.processStderr(); - BuildResult res; - unsigned int status; - conn->from >> status >> res.errorMsg; - res.status = (BuildResult::Status) status; - return res; -} - - -void RemoteStore::ensurePath(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopEnsurePath << printStorePath(path); - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::addTempRoot(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopAddTempRoot << printStorePath(path); - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::addIndirectRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << wopAddIndirectRoot << path; - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::syncWithGC() -{ - auto conn(getConnection()); - conn->to << wopSyncWithGC; - conn.processStderr(); - readInt(conn->from); -} - - -Roots RemoteStore::findRoots(bool censor) -{ - auto conn(getConnection()); - conn->to << wopFindRoots; - conn.processStderr(); - size_t count = readNum(conn->from); - Roots result; - while (count--) { - Path link = readString(conn->from); - auto target = parseStorePath(readString(conn->from)); - result[std::move(target)].emplace(link); - } - return result; -} - - -void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) -{ - auto conn(getConnection()); - - conn->to - << wopCollectGarbage << options.action; - worker_proto::write(*this, conn->to, options.pathsToDelete); - conn->to << options.ignoreLiveness - << options.maxFreed - /* removed options */ - << 0 << 0 << 0; - - conn.processStderr(); - - results.paths = readStrings(conn->from); - results.bytesFreed = readLongLong(conn->from); - readLongLong(conn->from); // obsolete - - { - auto state_(Store::state.lock()); - state_->pathInfoCache.clear(); - } -} - - -void RemoteStore::optimiseStore() -{ - auto conn(getConnection()); - conn->to << wopOptimiseStore; - conn.processStderr(); - readInt(conn->from); -} - - -bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) -{ - auto conn(getConnection()); - conn->to << wopVerifyStore << checkContents << repair; - conn.processStderr(); - return readInt(conn->from); -} - - -void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) -{ - auto conn(getConnection()); - conn->to << wopAddSignatures << printStorePath(storePath) << sigs; - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) -{ - { - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) - // Don't hold the connection handle in the fallback case - // to prevent a deadlock. - goto fallback; - conn->to << wopQueryMissing; - Strings ss; - for (auto & p : targets) - ss.push_back(p.to_string(*this)); - conn->to << ss; - conn.processStderr(); - willBuild = worker_proto::read(*this, conn->from, Phantom {}); - willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); - unknown = worker_proto::read(*this, conn->from, Phantom {}); - conn->from >> downloadSize >> narSize; - return; - } - - fallback: - return Store::queryMissing(targets, willBuild, willSubstitute, - unknown, downloadSize, narSize); -} - - -void RemoteStore::connect() -{ - auto conn(getConnection()); -} - - -unsigned int RemoteStore::getProtocol() -{ - auto conn(connections->get()); - return conn->daemonVersion; -} - - -void RemoteStore::flushBadConnections() -{ - connections->flushBad(); -} - - -RemoteStore::Connection::~Connection() -{ - try { - to.flush(); - } catch (...) { - ignoreException(); - } -} - -void RemoteStore::narFromPath(const StorePath & path, Sink & sink) -{ - auto conn(connections->get()); - conn->to << wopNarFromPath << printStorePath(path); - conn->processStderr(); - copyNAR(conn->from, sink); -} - -ref RemoteStore::getFSAccessor() -{ - return make_ref(ref(shared_from_this())); -} - -static Logger::Fields readFields(Source & from) -{ - Logger::Fields fields; - size_t size = readInt(from); - for (size_t n = 0; n < size; n++) { - auto type = (decltype(Logger::Field::type)) readInt(from); - if (type == Logger::Field::tInt) - fields.push_back(readNum(from)); - else if (type == Logger::Field::tString) - fields.push_back(readString(from)); - else - throw Error("got unsupported field type %x from Nix daemon", (int) type); - } - return fields; -} - - -std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush) -{ - if (flush) - to.flush(); - - while (true) { - - auto msg = readNum(from); - - if (msg == STDERR_WRITE) { - string s = readString(from); - if (!sink) throw Error("no sink"); - (*sink)(s); - } - - else if (msg == STDERR_READ) { - if (!source) throw Error("no source"); - size_t len = readNum(from); - auto buf = std::make_unique(len); - writeString(buf.get(), source->read(buf.get(), len), to); - to.flush(); - } - - else if (msg == STDERR_ERROR) { - if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { - return std::make_exception_ptr(readError(from)); - } else { - string error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); - } - } - - else if (msg == STDERR_NEXT) - printError(chomp(readString(from))); - - else if (msg == STDERR_START_ACTIVITY) { - auto act = readNum(from); - auto lvl = (Verbosity) readInt(from); - auto type = (ActivityType) readInt(from); - auto s = readString(from); - auto fields = readFields(from); - auto parent = readNum(from); - logger->startActivity(act, lvl, type, s, fields, parent); - } - - else if (msg == STDERR_STOP_ACTIVITY) { - auto act = readNum(from); - logger->stopActivity(act); - } - - else if (msg == STDERR_RESULT) { - auto act = readNum(from); - auto type = (ResultType) readInt(from); - auto fields = readFields(from); - logger->result(act, type, fields); - } - - else if (msg == STDERR_LAST) - break; - - else - throw Error("got unknown message type %x from Nix daemon", msg); - } - - return nullptr; -} - -void ConnectionHandle::withFramedSink(std::function fun) -{ - (*this)->to.flush(); - - std::exception_ptr ex; - - /* Handle log messages / exceptions from the remote on a - separate thread. */ - std::thread stderrThread([&]() - { - try { - processStderr(nullptr, nullptr, false); - } catch (...) { - ex = std::current_exception(); - } - }); - - Finally joinStderrThread([&]() - { - if (stderrThread.joinable()) { - stderrThread.join(); - if (ex) { - try { - std::rethrow_exception(ex); - } catch (...) { - ignoreException(); - } - } - } - }); - - { - FramedSink sink((*this)->to, ex); - fun(sink); - sink.flush(); - } - - stderrThread.join(); - if (ex) - std::rethrow_exception(ex); - -} - static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 554fb6bed..e5de104c9 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -1,161 +1,10 @@ #pragma once -#include -#include - -#include "store-api.hh" +#include "remote-store.hh" #include "local-fs-store.hh" - namespace nix { - -class Pipe; -class Pid; -struct FdSink; -struct FdSource; -template class Pool; -struct ConnectionHandle; - -struct RemoteStoreConfig : virtual StoreConfig -{ - using StoreConfig::StoreConfig; - - const Setting maxConnections{(StoreConfig*) this, 1, - "max-connections", "maximum number of concurrent connections to the Nix daemon"}; - - const Setting maxConnectionAge{(StoreConfig*) this, std::numeric_limits::max(), - "max-connection-age", "number of seconds to reuse a connection"}; -}; - -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ -class RemoteStore : public virtual Store, public virtual RemoteStoreConfig -{ -public: - - virtual bool sameMachine() = 0; - - RemoteStore(const Params & params); - - /* Implementations of abstract store API methods. */ - - bool isValidPathUncached(const StorePath & path) override; - - StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override; - - StorePathSet queryAllValidPaths() override; - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override; - - StorePathSet queryValidDerivers(const StorePath & path) override; - - StorePathSet queryDerivationOutputs(const StorePath & path) override; - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; - std::optional queryPathFromHashPart(const std::string & hashPart) override; - - StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - - void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) override; - - /* Add a content-addressable store path. `dump` will be drained. */ - ref addCAToStore( - Source & dump, - const string & name, - ContentAddressMethod caMethod, - const StorePathSet & references, - RepairFlag repair); - - /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; - - void addToStore(const ValidPathInfo & info, Source & nar, - RepairFlag repair, CheckSigsFlag checkSigs) override; - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) override; - - void buildPaths(const std::vector & paths, BuildMode buildMode) override; - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override; - - void ensurePath(const StorePath & path) override; - - void addTempRoot(const StorePath & path) override; - - void addIndirectRoot(const Path & path) override; - - void syncWithGC() override; - - Roots findRoots(bool censor) override; - - void collectGarbage(const GCOptions & options, GCResults & results) override; - - void optimiseStore() override; - - bool verifyStore(bool checkContents, RepairFlag repair) override; - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override; - - void connect() override; - - unsigned int getProtocol() override; - - void flushBadConnections(); - - struct Connection - { - AutoCloseFD fd; - FdSink to; - FdSource from; - unsigned int daemonVersion; - std::chrono::time_point startTime; - - virtual ~Connection(); - - std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); - }; - - ref openConnectionWrapper(); - -protected: - - virtual ref openConnection() = 0; - - void initConnection(Connection & conn); - - ref> connections; - - virtual void setOptions(Connection & conn); - - ConnectionHandle getConnection(); - - friend struct ConnectionHandle; - - virtual ref getFSAccessor() override; - - virtual void narFromPath(const StorePath & path, Sink & sink) override; - - ref readValidPathInfo(ConnectionHandle & conn, const StorePath & path); - -private: - - std::atomic_bool failed{false}; - -}; - struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { UDSRemoteStoreConfig(const Store::Params & params) @@ -200,5 +49,4 @@ private: std::optional path; }; - }