From 4abd5554ad0be209b103378a26fa8784aee3a0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Fri, 2 Oct 2020 15:40:55 +0200 Subject: [PATCH 01/31] Fix macOS sandbox build --- doc/manual/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 3b8e7e2df..d308c7cf6 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -40,7 +40,7 @@ $(d)/nix.json: $(bindir)/nix @mv $@.tmp $@ $(d)/conf-file.json: $(bindir)/nix - $(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp + $(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix From d0bb544128140d51b5a5a17d36eeb9fd13838586 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 10:18:44 +0200 Subject: [PATCH 02/31] Add missing #pragma once --- src/libexpr/primops.hh | 2 ++ src/libstore/daemon.hh | 2 ++ src/libstore/parsed-derivations.hh | 2 ++ src/libutil/topo-sort.hh | 2 ++ 4 files changed, 8 insertions(+) diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index ed5e2ea58..9d42d6539 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -1,3 +1,5 @@ +#pragma once + #include "eval.hh" #include diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 841ace316..67755d54e 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,3 +1,5 @@ +#pragma once + #include "serialise.hh" #include "store-api.hh" diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 3fa09f34f..c9fbe68c4 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,3 +1,5 @@ +#pragma once + #include "store-api.hh" #include diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh index 7a68ff169..7418be5e0 100644 --- a/src/libutil/topo-sort.hh +++ b/src/libutil/topo-sort.hh @@ -1,3 +1,5 @@ +#pragma once + #include "error.hh" namespace nix { From 6691256e790b2917ccd06f16472b667426d0f413 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 10:40:49 +0200 Subject: [PATCH 03/31] Factor out common showBytes() --- src/libmain/shared.cc | 12 +++--------- src/libstore/optimise-store.cc | 12 +++--------- src/libutil/util.cc | 7 +++++++ src/libutil/util.hh | 3 +++ 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 22ae51e47..2247aeca4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -386,18 +386,12 @@ RunPager::~RunPager() } -string showBytes(uint64_t bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); -} - - PrintFreed::~PrintFreed() { if (show) - std::cout << format("%1% store paths deleted, %2% freed\n") - % results.paths.size() - % showBytes(results.bytesFreed); + std::cout << fmt("%d store paths deleted, %s freed\n", + results.paths.size(), + showBytes(results.bytesFreed)); } Exit::~Exit() { } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index c032a5e22..a0d482ddf 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -276,21 +276,15 @@ void LocalStore::optimiseStore(OptimiseStats & stats) } } -static string showBytes(uint64_t bytes) -{ - return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); -} - void LocalStore::optimiseStore() { OptimiseStats stats; optimiseStore(stats); - printInfo( - format("%1% freed by hard-linking %2% files") - % showBytes(stats.bytesFreed) - % stats.filesLinked); + printInfo("%s freed by hard-linking %d files", + showBytes(stats.bytesFreed), + stats.filesLinked); } void LocalStore::optimisePath(const Path & path) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9e7142e01..f07e99885 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1641,4 +1641,11 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) return fdSocket; } + +string showBytes(uint64_t bytes) +{ + return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); +} + + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b8e201203..129d59a97 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -573,4 +573,7 @@ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; +std::string showBytes(uint64_t bytes); + + } From 636455c471bfde89dbdf33e10308962b79d50f07 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 11:16:32 +0200 Subject: [PATCH 04/31] Remove 'using namespace fetchers' --- src/libexpr/flake/flake.cc | 2 +- src/libexpr/flake/lockfile.cc | 4 ++-- src/libexpr/flake/lockfile.hh | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index b4ede542c..bae4d65e5 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -12,7 +12,7 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +typedef std::pair FetchedFlake; typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 78431f000..bb46e1bb4 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -13,12 +13,12 @@ FlakeRef getFlakeRef( { auto i = json.find(attr); if (i != json.end()) { - auto attrs = jsonToAttrs(*i); + auto attrs = fetchers::jsonToAttrs(*i); // FIXME: remove when we drop support for version 5. if (info) { auto j = json.find(info); if (j != json.end()) { - for (auto k : jsonToAttrs(*j)) + for (auto k : fetchers::jsonToAttrs(*j)) attrs.insert_or_assign(k.first, k.second); } } diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 9ec8b39c3..627794d8c 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -11,8 +11,6 @@ class StorePath; namespace nix::flake { -using namespace fetchers; - typedef std::vector InputPath; struct LockedNode; From b4db315a56860b10a7d19339f2310bc1149219cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 13:26:00 +0200 Subject: [PATCH 05/31] mk/precompiled-headers.mk: Fix clang test "clang++" includes the string "g++" so this test didn't work properly. However the separate handling of clang might not be needed anymore... --- mk/precompiled-headers.mk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 1fdb4b3a4..bfee35a2c 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -21,18 +21,18 @@ clean-files += $(GCH) $(PCH) ifeq ($(PRECOMPILE_HEADERS), 1) - ifeq ($(findstring g++,$(CXX)), g++) - - GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch - - GLOBAL_ORDER_AFTER += $(GCH) - - else ifeq ($(findstring clang++,$(CXX)), clang++) + ifeq ($(findstring clang++,$(CXX)), clang++) GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch GLOBAL_ORDER_AFTER += $(PCH) + else ifeq ($(findstring g++,$(CXX)), g++) + + GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch + + GLOBAL_ORDER_AFTER += $(GCH) + else $(error Don't know how to precompile headers on $(CXX)) From 0856c0a0b4ff9b72b4e786e700785bb331298582 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 13:27:09 +0200 Subject: [PATCH 06/31] mk/precompiled-headers.mk: Remove special handling for clang --- .gitignore | 1 - local.mk | 2 +- mk/precompiled-headers.mk | 27 +++------------------------ 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 4711fa7fa..b087cd8d5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ perl/Makefile.config /aclocal.m4 /autom4te.cache /precompiled-headers.h.gch -/precompiled-headers.h.pch /config.* /configure /stamp-h1 diff --git a/local.mk b/local.mk index b1ca832a6..fe314b902 100644 --- a/local.mk +++ b/local.mk @@ -11,6 +11,6 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations $(foreach i, config.h $(wildcard src/lib*/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) -$(GCH) $(PCH): src/libutil/util.hh config.h +$(GCH): src/libutil/util.hh config.h GCH_CXXFLAGS = -I src/libutil diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index bfee35a2c..cdd3daecd 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -10,33 +10,12 @@ $(GCH): precompiled-headers.h @mkdir -p "$(dir $@)" $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) -PCH = $(buildprefix)precompiled-headers.h.pch - -$(PCH): precompiled-headers.h - @rm -f $@ - @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) - -clean-files += $(GCH) $(PCH) +clean-files += $(GCH) ifeq ($(PRECOMPILE_HEADERS), 1) - ifeq ($(findstring clang++,$(CXX)), clang++) + GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch - GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch - - GLOBAL_ORDER_AFTER += $(PCH) - - else ifeq ($(findstring g++,$(CXX)), g++) - - GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch - - GLOBAL_ORDER_AFTER += $(GCH) - - else - - $(error Don't know how to precompile headers on $(CXX)) - - endif + GLOBAL_ORDER_AFTER += $(GCH) endif From 0419cd26954d9d7c5b5ace96501b41cc81d66eed Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 13:34:58 +0200 Subject: [PATCH 07/31] Remove unneeded -lboost_* flags --- src/nix/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/local.mk b/src/nix/local.mk index ab4e9121b..f37b73384 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -19,7 +19,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr nix_LIBS = libexpr libmain libfetchers libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -llowdown +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ From 85c8be6286e8f5464813a8a29b0b78d892498745 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 13:36:55 +0200 Subject: [PATCH 08/31] Remove static variable name clashes This was useful for an experiment with building Nix as a single compilation unit. It's not very useful otherwise but also doesn't hurt... --- src/build-remote/build-remote.cc | 4 ++-- src/libexpr/eval.cc | 2 +- src/libexpr/primops/context.cc | 10 +++++----- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libexpr/primops/fromTOML.cc | 2 +- src/libfetchers/git.cc | 2 +- src/libfetchers/github.cc | 4 ++-- src/libfetchers/indirect.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/path.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/dummy-store.cc | 2 +- src/libstore/filetransfer.cc | 2 +- src/libstore/globals.cc | 2 +- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/ssh-store.cc | 2 +- src/libutil/archive.cc | 2 +- src/libutil/args.cc | 6 +++--- src/libutil/logging.cc | 2 +- src/nix-build/nix-build.cc | 6 +++--- src/nix-channel/nix-channel.cc | 4 ++-- src/nix-collect-garbage/nix-collect-garbage.cc | 4 ++-- src/nix-copy-closure/nix-copy-closure.cc | 4 ++-- src/nix-daemon/nix-daemon.cc | 4 ++-- src/nix-env/nix-env.cc | 4 ++-- src/nix-instantiate/nix-instantiate.cc | 4 ++-- src/nix-prefetch-url/nix-prefetch-url.cc | 4 ++-- src/nix-store/nix-store.cc | 9 +++++++-- src/nix/add-to-store.cc | 2 +- src/nix/build.cc | 2 +- src/nix/cat.cc | 4 ++-- src/nix/copy.cc | 2 +- src/nix/describe-stores.cc | 2 +- src/nix/develop.cc | 4 ++-- src/nix/diff-closures.cc | 2 +- src/nix/doctor.cc | 2 +- src/nix/dump-path.cc | 2 +- src/nix/edit.cc | 2 +- src/nix/eval.cc | 2 +- src/nix/flake.cc | 2 +- src/nix/hash.cc | 14 +++++++------- src/nix/log.cc | 2 +- src/nix/ls.cc | 4 ++-- src/nix/make-content-addressable.cc | 2 +- src/nix/optimise-store.cc | 2 +- src/nix/path-info.cc | 2 +- src/nix/ping-store.cc | 2 +- src/nix/profile.cc | 2 +- src/nix/registry.cc | 2 +- src/nix/repl.cc | 2 +- src/nix/run.cc | 4 ++-- src/nix/search.cc | 2 +- src/nix/show-config.cc | 2 +- src/nix/show-derivation.cc | 2 +- src/nix/sigs.cc | 4 ++-- src/nix/upgrade-nix.cc | 2 +- src/nix/verify.cc | 2 +- src/nix/why-depends.cc | 2 +- 63 files changed, 97 insertions(+), 92 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index ce5127113..9a0e9f08d 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -44,7 +44,7 @@ static bool allSupportedLocally(Store & store, const std::set& requ return true; } -static int _main(int argc, char * * argv) +static int main_build_remote(int argc, char * * argv) { { logger = makeJSONLogger(*logger); @@ -297,4 +297,4 @@ connected: } } -static RegisterLegacyCommand s1("build-remote", _main); +static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 883fc27a7..d6366050c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2081,7 +2081,7 @@ Strings EvalSettings::getDefaultNixPath() EvalSettings evalSettings; -static GlobalConfig::Register r1(&evalSettings); +static GlobalConfig::Register rEvalSettings(&evalSettings); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index dbb93bae6..b570fca31 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, mkString(v, s, PathSet()); } -static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -21,7 +21,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, mkBool(v, !context.empty()); } -static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a @@ -42,7 +42,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po mkString(v, s, context2); } -static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); /* Extract the context of a string as a structured Nix value. @@ -127,7 +127,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, v.attrs->sort(); } -static RegisterPrimOp r4("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); /* Append the given context to a given string. @@ -191,6 +191,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg mkString(v, orig, context); } -static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 1a064ed5c..a77035c16 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -87,6 +87,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar state.allowedPaths->insert(tree.actualPath); } -static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 06e8304b8..7cd4d0fbf 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -152,7 +152,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V fetchTree(state, pos, args, v, std::nullopt); } -static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); +static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, std::string name) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index b00827a4b..77bff44ae 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -88,6 +88,6 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va } } -static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ad7638d73..a6411b02b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -452,6 +452,6 @@ struct GitInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8610fe447..90893075e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -334,7 +334,7 @@ struct GitLabInputScheme : GitArchiveInputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); -static auto r2 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 74332ae3d..10e59919a 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -100,6 +100,6 @@ struct IndirectInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index d80c2ea7a..7d3d52751 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -293,6 +293,6 @@ struct MercurialInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 99d4b4e8f..bcb904c0d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -102,6 +102,6 @@ struct PathInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ca49482a9..8c0f20475 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -231,6 +231,6 @@ struct TarballInputScheme : InputScheme } }; -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 49641c2ac..736be8413 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -63,6 +63,6 @@ struct DummyStore : public Store, public virtual DummyStoreConfig { unsupported("buildDerivation"); } }; -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regDummyStore; } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index cd619672f..c2c65af05 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -31,7 +31,7 @@ namespace nix { FileTransferSettings fileTransferSettings; -static GlobalConfig::Register r1(&fileTransferSettings); +static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); std::string resolveUri(const std::string & uri) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index c5734852d..1238dc530 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -25,7 +25,7 @@ namespace nix { Settings settings; -static GlobalConfig::Register r1(&settings); +static GlobalConfig::Register rSettings(&settings); Settings::Settings() : nixPrefix(NIX_PREFIX) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3d3d91e5e..9d2a89f96 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -181,6 +181,6 @@ protected: }; -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regHttpBinaryCacheStore; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 6db44047d..467169ce8 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -335,6 +335,6 @@ public: } }; -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b5744448e..7d979c5c2 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -105,6 +105,6 @@ std::set LocalBinaryCacheStore::uriSchemes() return {"file"}; } -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 049d4e954..40ef72293 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1008,6 +1008,6 @@ void ConnectionHandle::withFramedSink(std::function fun) } -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d43f267e0..552c4aac7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -439,7 +439,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache }; -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 6d6eca98d..08d0bd565 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -83,6 +83,6 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regSSHStore; } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6ad9fa238..f1479329f 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -33,7 +33,7 @@ struct ArchiveSettings : Config static ArchiveSettings archiveSettings; -static GlobalConfig::Register r1(&archiveSettings); +static GlobalConfig::Register rArchiveSettings(&archiveSettings); const std::string narVersionMagic1 = "nix-archive-1"; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 147602415..2760b830b 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -277,7 +277,7 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< }; } -static void completePath(std::string_view prefix, bool onlyDirs) +static void _completePath(std::string_view prefix, bool onlyDirs) { pathCompletions = true; glob_t globbuf; @@ -300,12 +300,12 @@ static void completePath(std::string_view prefix, bool onlyDirs) void completePath(size_t, std::string_view prefix) { - completePath(prefix, false); + _completePath(prefix, false); } void completeDir(size_t, std::string_view prefix) { - completePath(prefix, true); + _completePath(prefix, true); } Strings argvToStrings(int argc, char * * argv) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index cbbf64395..8a6752e22 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -10,7 +10,7 @@ namespace nix { LoggerSettings loggerSettings; -static GlobalConfig::Register r1(&loggerSettings); +static GlobalConfig::Register rLoggerSettings(&loggerSettings); static thread_local ActivityId curActivity = 0; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 3a8d67f21..a79b1086b 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -67,7 +67,7 @@ std::vector shellwords(const string & s) return res; } -static void _main(int argc, char * * argv) +static void main_nix_build(int argc, char * * argv) { auto dryRun = false; auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); @@ -540,5 +540,5 @@ static void _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-build", _main); -static RegisterLegacyCommand s2("nix-shell", _main); +static RegisterLegacyCommand r_nix_build("nix-build", main_nix_build); +static RegisterLegacyCommand r_nix_shell("nix-shell", main_nix_build); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index e48f7af9a..309970df6 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -153,7 +153,7 @@ static void update(const StringSet & channelNames) replaceSymlink(profile, channelLink); } -static int _main(int argc, char ** argv) +static int main_nix_channel(int argc, char ** argv) { { // Figure out the name of the `.nix-channels' file to use @@ -250,4 +250,4 @@ static int _main(int argc, char ** argv) } } -static RegisterLegacyCommand s1("nix-channel", _main); +static RegisterLegacyCommand r_nix_channel("nix-channel", main_nix_channel); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index bcf1d8518..57092b887 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -49,7 +49,7 @@ void removeOldGenerations(std::string dir) } } -static int _main(int argc, char * * argv) +static int main_nix_collect_garbage(int argc, char * * argv) { { bool removeOld = false; @@ -92,4 +92,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-collect-garbage", _main); +static RegisterLegacyCommand r_nix_collect_garbage("nix-collect-garbage", main_nix_collect_garbage); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index b10184718..10990f7b5 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -4,7 +4,7 @@ using namespace nix; -static int _main(int argc, char ** argv) +static int main_nix_copy_closure(int argc, char ** argv) { { auto gzip = false; @@ -65,4 +65,4 @@ static int _main(int argc, char ** argv) } } -static RegisterLegacyCommand s1("nix-copy-closure", _main); +static RegisterLegacyCommand r_nix_copy_closure("nix-copy-closure", main_nix_copy_closure); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 6e652ccbf..fc6195cf0 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -265,7 +265,7 @@ static void daemonLoop(char * * argv) } -static int _main(int argc, char * * argv) +static int main_nix_daemon(int argc, char * * argv) { { auto stdio = false; @@ -330,4 +330,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-daemon", _main); +static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 3e7c453fb..e6667e7f5 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1330,7 +1330,7 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) } -static int _main(int argc, char * * argv) +static int main_nix_env(int argc, char * * argv) { { Strings opFlags, opArgs; @@ -1460,4 +1460,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-env", _main); +static RegisterLegacyCommand r_nix_env("nix-env", main_nix_env); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 539092cbe..18a0049a6 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -83,7 +83,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, } -static int _main(int argc, char * * argv) +static int main_nix_instantiate(int argc, char * * argv) { { Strings files; @@ -192,4 +192,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-instantiate", _main); +static RegisterLegacyCommand r_nix_instantiate("nix-instantiate", main_nix_instantiate); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 377ae03a8..3bdee55a7 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -48,7 +48,7 @@ string resolveMirrorUri(EvalState & state, string uri) } -static int _main(int argc, char * * argv) +static int main_nix_prefetch_url(int argc, char * * argv) { { HashType ht = htSHA256; @@ -229,4 +229,4 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-prefetch-url", _main); +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index da91caef1..14baabc36 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -24,6 +24,9 @@ #endif +namespace nix_store { + + using namespace nix; using std::cin; using std::cout; @@ -1025,7 +1028,7 @@ static void opVersion(Strings opFlags, Strings opArgs) /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ -static int _main(int argc, char * * argv) +static int main_nix_store(int argc, char * * argv) { { Strings opFlags, opArgs; @@ -1121,4 +1124,6 @@ static int _main(int argc, char * * argv) } } -static RegisterLegacyCommand s1("nix-store", _main); +static RegisterLegacyCommand r_nix_store("nix-store", main_nix_store); + +} diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 023ffa4ed..7fe87d757 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -87,4 +87,4 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; -static auto r1 = registerCommand("add-to-store"); +static auto rCmdAddToStore = registerCommand("add-to-store"); diff --git a/src/nix/build.cc b/src/nix/build.cc index 4605eb13e..d85a482db 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -88,4 +88,4 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile } }; -static auto r1 = registerCommand("build"); +static auto rCmdBuild = registerCommand("build"); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 97306107c..eef172cfc 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -72,5 +72,5 @@ struct CmdCatNar : StoreCommand, MixCat } }; -static auto r1 = registerCommand("cat-store"); -static auto r2 = registerCommand("cat-nar"); +static auto rCmdCatStore = registerCommand("cat-store"); +static auto rCmdCatNar = registerCommand("cat-nar"); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 811c8ab34..cb31aac8f 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -106,4 +106,4 @@ struct CmdCopy : StorePathsCommand } }; -static auto r1 = registerCommand("copy"); +static auto rCmdCopy = registerCommand("copy"); diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc index 0cc2d9337..1dd384c0e 100644 --- a/src/nix/describe-stores.cc +++ b/src/nix/describe-stores.cc @@ -41,4 +41,4 @@ struct CmdDescribeStores : Command, MixJSON } }; -static auto r1 = registerCommand("describe-stores"); +static auto rDescribeStore = registerCommand("describe-stores"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index f29fa71d2..a46ea39b6 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -443,5 +443,5 @@ struct CmdPrintDevEnv : Common } }; -static auto r1 = registerCommand("print-dev-env"); -static auto r2 = registerCommand("develop"); +static auto rCmdPrintDevEnv = registerCommand("print-dev-env"); +static auto rCmdDevelop = registerCommand("develop"); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 0dc99d05e..30e7b20e1 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -143,4 +143,4 @@ struct CmdDiffClosures : SourceExprCommand } }; -static auto r1 = registerCommand("diff-closures"); +static auto rCmdDiffClosures = registerCommand("diff-closures"); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 683e91446..4588ac05e 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -131,4 +131,4 @@ struct CmdDoctor : StoreCommand } }; -static auto r1 = registerCommand("doctor"); +static auto rCmdDoctor = registerCommand("doctor"); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index e1de71bf8..6fd197531 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -30,4 +30,4 @@ struct CmdDumpPath : StorePathCommand } }; -static auto r1 = registerCommand("dump-path"); +static auto rDumpPath = registerCommand("dump-path"); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 378a3739c..51c16f5a9 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -54,4 +54,4 @@ struct CmdEdit : InstallableCommand } }; -static auto r1 = registerCommand("edit"); +static auto rCmdEdit = registerCommand("edit"); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index a8ca446be..43ce46546 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -90,4 +90,4 @@ struct CmdEval : MixJSON, InstallableCommand } }; -static auto r1 = registerCommand("eval"); +static auto rCmdEval = registerCommand("eval"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ae6f4c5f9..d45f13029 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -965,4 +965,4 @@ struct CmdFlake : NixMultiCommand } }; -static auto r1 = registerCommand("flake"); +static auto rCmdFlake = registerCommand("flake"); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 494f00a20..1d23bb0e2 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -79,8 +79,8 @@ struct CmdHash : Command } }; -static RegisterCommand r1("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); -static RegisterCommand r2("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); +static RegisterCommand rCmdHashFile("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); +static RegisterCommand rCmdHashPath("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); struct CmdToBase : Command { @@ -112,10 +112,10 @@ struct CmdToBase : Command } }; -static RegisterCommand r3("to-base16", [](){ return make_ref(Base16); }); -static RegisterCommand r4("to-base32", [](){ return make_ref(Base32); }); -static RegisterCommand r5("to-base64", [](){ return make_ref(Base64); }); -static RegisterCommand r6("to-sri", [](){ return make_ref(SRI); }); +static RegisterCommand rCmdToBase16("to-base16", [](){ return make_ref(Base16); }); +static RegisterCommand rCmdToBase32("to-base32", [](){ return make_ref(Base32); }); +static RegisterCommand rCmdToBase64("to-base64", [](){ return make_ref(Base64); }); +static RegisterCommand rCmdToSRI("to-sri", [](){ return make_ref(SRI); }); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) @@ -167,4 +167,4 @@ static int compatNixHash(int argc, char * * argv) return 0; } -static RegisterLegacyCommand s1("nix-hash", compatNixHash); +static RegisterLegacyCommand r_nix_hash("nix-hash", compatNixHash); diff --git a/src/nix/log.cc b/src/nix/log.cc index 33380dcf5..33a3053f5 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -64,4 +64,4 @@ struct CmdLog : InstallableCommand } }; -static auto r1 = registerCommand("log"); +static auto rCmdLog = registerCommand("log"); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 76c8bc9a3..baca54431 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -152,5 +152,5 @@ struct CmdLsNar : Command, MixLs } }; -static auto r1 = registerCommand("ls-store"); -static auto r2 = registerCommand("ls-nar"); +static auto rCmdLsStore = registerCommand("ls-store"); +static auto rCmdLsNar = registerCommand("ls-nar"); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 38b60fc38..df3ec5194 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -108,4 +108,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand("make-content-addressable"); +static auto rCmdMakeContentAddressable = registerCommand("make-content-addressable"); diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index b45951879..51a7a9756 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -31,4 +31,4 @@ struct CmdOptimiseStore : StoreCommand } }; -static auto r1 = registerCommand("optimise-store"); +static auto rCmdOptimiseStore = registerCommand("optimise-store"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 0c12efaf0..63cf885f9 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -127,4 +127,4 @@ struct CmdPathInfo : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand("path-info"); +static auto rCmdPathInfo = registerCommand("path-info"); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 127397a29..8db78d591 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -29,4 +29,4 @@ struct CmdPingStore : StoreCommand } }; -static auto r1 = registerCommand("ping-store"); +static auto rCmdPingStore = registerCommand("ping-store"); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 7ce4dfe4c..01aef2f9b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -467,4 +467,4 @@ struct CmdProfile : NixMultiCommand } }; -static auto r1 = registerCommand("profile"); +static auto rCmdProfile = registerCommand("profile"); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index cb11ec195..8e8983ad0 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -143,4 +143,4 @@ struct CmdRegistry : virtual NixMultiCommand } }; -static auto r1 = registerCommand("registry"); +static auto rCmdRegistry = registerCommand("registry"); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 329999475..9ff386b1d 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -825,6 +825,6 @@ struct CmdRepl : StoreCommand, MixEvalArgs } }; -static auto r1 = registerCommand("repl"); +static auto rCmdRepl = registerCommand("repl"); } diff --git a/src/nix/run.cc b/src/nix/run.cc index e6584346e..790784382 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -140,7 +140,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment } }; -static auto r1 = registerCommand("shell"); +static auto rCmdShell = registerCommand("shell"); struct CmdRun : InstallableCommand, RunCommon { @@ -209,7 +209,7 @@ struct CmdRun : InstallableCommand, RunCommon } }; -static auto r2 = registerCommand("run"); +static auto rCmdRun = registerCommand("run"); void chrootHelper(int argc, char * * argv) { diff --git a/src/nix/search.cc b/src/nix/search.cc index 430979274..d4326dc84 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -185,4 +185,4 @@ struct CmdSearch : InstallableCommand, MixJSON } }; -static auto r1 = registerCommand("search"); +static auto rCmdSearch = registerCommand("search"); diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 3ed1ad2aa..1ef54a33a 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -30,4 +30,4 @@ struct CmdShowConfig : Command, MixJSON } }; -static auto r1 = registerCommand("show-config"); +static auto rShowConfig = registerCommand("show-config"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index b9f33499b..2542537d3 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -123,4 +123,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static auto r1 = registerCommand("show-derivation"); +static auto rCmdShowDerivation = registerCommand("show-derivation"); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 7821a5432..44916c77f 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -92,7 +92,7 @@ struct CmdCopySigs : StorePathsCommand } }; -static auto r1 = registerCommand("copy-sigs"); +static auto rCmdCopySigs = registerCommand("copy-sigs"); struct CmdSignPaths : StorePathsCommand { @@ -144,4 +144,4 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto r2 = registerCommand("sign-paths"); +static auto rCmdSignPaths = registerCommand("sign-paths"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a880bdae0..66ecc5b34 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -158,4 +158,4 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand } }; -static auto r1 = registerCommand("upgrade-nix"); +static auto rCmdUpgradeNix = registerCommand("upgrade-nix"); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 26f755fd9..ec7333d03 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -189,4 +189,4 @@ struct CmdVerify : StorePathsCommand } }; -static auto r1 = registerCommand("verify"); +static auto rCmdVerify = registerCommand("verify"); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 7e630b745..63bf087e6 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -263,4 +263,4 @@ struct CmdWhyDepends : SourceExprCommand } }; -static auto r1 = registerCommand("why-depends"); +static auto rCmdWhyDepends = registerCommand("why-depends"); From ad143c5b3b7e713b89f0437ce17c20ac642ca530 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 14:00:42 +0200 Subject: [PATCH 09/31] Shut up some clang warnings --- src/libexpr/eval-cache.hh | 2 +- src/libfetchers/fetchers.hh | 3 +++ src/libfetchers/github.cc | 6 +++--- src/libstore/dummy-store.cc | 3 +-- src/libstore/store-api.hh | 2 ++ 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 8ffffc0ed..e23e45c94 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -11,7 +11,7 @@ namespace nix::eval_cache { MakeError(CachedEvalError, EvalError); -class AttrDb; +struct AttrDb; class AttrCursor; class EvalCache : public std::enable_shared_from_this diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index cc31a31b9..e8ae59143 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -86,6 +86,9 @@ public: struct InputScheme { + virtual ~InputScheme() + { } + virtual std::optional inputFromURL(const ParsedURL & url) = 0; virtual std::optional inputFromAttrs(const Attrs & attrs) = 0; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 90893075e..92ff224f7 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -25,7 +25,7 @@ struct GitArchiveInputScheme : InputScheme { virtual std::string type() = 0; - virtual std::optional > accessHeaderFromToken(const std::string & token) const = 0; + virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; std::optional inputFromURL(const ParsedURL & url) override { @@ -215,7 +215,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string type() override { return "github"; } - std::optional > accessHeaderFromToken(const std::string & token) const + std::optional> accessHeaderFromToken(const std::string & token) const override { // Github supports PAT/OAuth2 tokens and HTTP Basic // Authentication. The former simply specifies the token, the @@ -270,7 +270,7 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string type() override { return "gitlab"; } - std::optional > accessHeaderFromToken(const std::string & token) const + std::optional> accessHeaderFromToken(const std::string & token) const override { // Gitlab supports 4 kinds of authorization, two of which are // relevant here: OAuth2 and PAT (Private Access Token). The diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 736be8413..98b745c3a 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -18,8 +18,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig DummyStore(const Params & params) : StoreConfig(params) , Store(params) - { - } + { } string getUri() override { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 854446987..450c0f554 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -194,6 +194,8 @@ struct StoreConfig : public Config */ StoreConfig() { assert(false); } + virtual ~StoreConfig() { } + virtual const std::string name() = 0; const PathSetting storeDir_{this, false, settings.nixStore, From d761485010cc9d071da17a5d90cf4126a4bf499d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 18:52:46 +0200 Subject: [PATCH 10/31] Prevent a deadlock when user namespace setup fails Observed on Centos 7 when user namespaces are disabled: DerivationGoal::startBuilder() throws an exception, ~DerivationGoal() waits for the child process to exit, but the child process hangs forever in drainFD(userNamespaceSync.readSide.get()) in DerivationGoal::runChild(). Not sure why the SIGKILL doesn't get through. Issue #4092. --- src/libstore/build.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 87e01a378..9497e7b3e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2686,6 +2686,12 @@ void DerivationGoal::startBuilder() userNamespaceSync.readSide = -1; + /* Close the write side to prevent runChild() from hanging + reading from this. */ + Finally cleanup([&]() { + userNamespaceSync.writeSide = -1; + }); + pid_t tmp; if (!string2Int(readLine(builderOut.readSide.get()), tmp)) abort(); pid = tmp; @@ -2712,7 +2718,6 @@ void DerivationGoal::startBuilder() /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); - userNamespaceSync.writeSide = -1; } else #endif From 57d960dcd174f0910c7fca02ee3afe80cb2b7844 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 7 Oct 2020 12:48:35 +0000 Subject: [PATCH 11/31] Remove generic std::optional suppport from worker proto See comment for rational; I think it's good to leave a comment lest anyone is tempted to add such a sum-type instance again. Fixes #4113 --- src/libstore/worker-protocol.hh | 48 +++++++++++---------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 2934c1d67..564fb978a 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -83,7 +83,6 @@ MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); MAKE_WORKER_PROTO(template, std::set); -MAKE_WORKER_PROTO(template, std::optional); #define X_ template #define Y_ std::map @@ -91,6 +90,22 @@ MAKE_WORKER_PROTO(X_, Y_); #undef X_ #undef Y_ +/* These use the empty string for the null case, relying on the fact + that the underlying types never serialize to the empty string. + + We do this instead of a generic std::optional instance because + ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + the same reason, we don't have a std::variant instances (ordinal + tags 0...n). + + We could the generic instances and then these as specializations for + compatability, but that's proven a bit finnicky, and also makes the + worker protocol harder to implement in other languages where such + specializations may not be allowed. + */ +MAKE_WORKER_PROTO(, std::optional); +MAKE_WORKER_PROTO(, std::optional); + template std::set read(const Store & store, Source & from, Phantom> _) { @@ -134,37 +149,6 @@ void write(const Store & store, Sink & out, const std::map & resMap) } } -template -std::optional read(const Store & store, Source & from, Phantom> _) -{ - auto tag = readNum(from); - switch (tag) { - case 0: - return std::nullopt; - case 1: - return read(store, from, Phantom {}); - default: - throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag); - } -} - -template -void write(const Store & store, Sink & out, const std::optional & optVal) -{ - out << (uint64_t) (optVal ? 1 : 0); - if (optVal) - worker_proto::write(store, out, *optVal); -} - -/* Specialization which uses and empty string for the empty case, taking - advantage of the fact these types always serialize to non-empty strings. - This is done primarily for backwards compatability, so that T <= - std::optional, where <= is the compatability partial order, T is one of - the types below. - */ -MAKE_WORKER_PROTO(, std::optional); -MAKE_WORKER_PROTO(, std::optional); - } } From 27ca87c46a57e1e6603028c1902189b53635d576 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 16:33:19 +0200 Subject: [PATCH 12/31] Formatting --- src/libutil/error.cc | 26 +++++++++++++------------- src/libutil/error.hh | 4 ++-- src/libutil/fmt.hh | 30 +++++++++++++++++------------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index fd6f69b7f..803a72953 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -11,13 +11,13 @@ const std::string nativeSystem = SYSTEM; BaseError & BaseError::addTrace(std::optional e, hintformat hint) { - err.traces.push_front(Trace { .pos = e, .hint = hint}); + err.traces.push_front(Trace { .pos = e, .hint = hint }); return *this; } // c++ std::exception descendants must have a 'const char* what()' function. // This stringifies the error and caches it for use by what(), or similarly by msg(). -const string& BaseError::calcWhat() const +const string & BaseError::calcWhat() const { if (what_.has_value()) return *what_; @@ -34,12 +34,12 @@ const string& BaseError::calcWhat() const std::optional ErrorInfo::programName = std::nullopt; -std::ostream& operator<<(std::ostream &os, const hintformat &hf) +std::ostream & operator<<(std::ostream & os, const hintformat & hf) { return os << hf.str(); } -string showErrPos(const ErrPos &errPos) +string showErrPos(const ErrPos & errPos) { if (errPos.line > 0) { if (errPos.column > 0) { @@ -53,7 +53,7 @@ string showErrPos(const ErrPos &errPos) } } -std::optional getCodeLines(const ErrPos &errPos) +std::optional getCodeLines(const ErrPos & errPos) { if (errPos.line <= 0) return std::nullopt; @@ -92,13 +92,13 @@ std::optional getCodeLines(const ErrPos &errPos) return loc; } } - catch (EndOfFile &eof) { + catch (EndOfFile & eof) { if (loc.errLineOfCode.has_value()) return loc; else return std::nullopt; } - catch (std::exception &e) { + catch (std::exception & e) { printError("error reading nix file: %s\n%s", errPos.file, e.what()); return std::nullopt; } @@ -137,10 +137,10 @@ std::optional getCodeLines(const ErrPos &errPos) } // print lines of code to the ostream, indicating the error column. -void printCodeLines(std::ostream &out, - const string &prefix, - const ErrPos &errPos, - const LinesOfCode &loc) +void printCodeLines(std::ostream & out, + const string & prefix, + const ErrPos & errPos, + const LinesOfCode & loc) { // previous line of code. if (loc.prevLineOfCode.has_value()) { @@ -186,7 +186,7 @@ void printCodeLines(std::ostream &out, } } -void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out) +void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out) { if (pos) { @@ -212,7 +212,7 @@ void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out) } } -std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace) +std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { auto errwidth = std::max(getWindowSize().second, 20); string prefix = ""; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f3babcbde..d1b6d82bb 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -107,7 +107,7 @@ struct Trace { struct ErrorInfo { Verbosity level; string name; - string description; + string description; // FIXME: remove? it seems to be barely used std::optional hint; std::optional errPos; std::list traces; @@ -169,7 +169,7 @@ public: #endif const string & msg() const { return calcWhat(); } - const ErrorInfo & info() { calcWhat(); return err; } + const ErrorInfo & info() const { calcWhat(); return err; } template BaseError & addTrace(std::optional e, const string &fs, const Args & ... args) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 6e69bdce2..85c0e9429 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -76,11 +76,11 @@ template struct yellowtxt { yellowtxt(const T &s) : value(s) {} - const T &value; + const T & value; }; template -std::ostream& operator<<(std::ostream &out, const yellowtxt &y) +std::ostream & operator<<(std::ostream & out, const yellowtxt & y) { return out << ANSI_YELLOW << y.value << ANSI_NORMAL; } @@ -88,12 +88,12 @@ std::ostream& operator<<(std::ostream &out, const yellowtxt &y) template struct normaltxt { - normaltxt(const T &s) : value(s) {} - const T &value; + normaltxt(const T & s) : value(s) {} + const T & value; }; template -std::ostream& operator<<(std::ostream &out, const normaltxt &y) +std::ostream & operator<<(std::ostream & out, const normaltxt & y) { return out << ANSI_NORMAL << y.value; } @@ -101,26 +101,30 @@ std::ostream& operator<<(std::ostream &out, const normaltxt &y) class hintformat { public: - hintformat(const string &format) :fmt(format) + hintformat(const string & format) : fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } - hintformat(const hintformat &hf) - : fmt(hf.fmt) - {} + hintformat(const hintformat & hf) + : fmt(hf.fmt) + { } + + hintformat(format && fmt) + : fmt(std::move(fmt)) + { } template - hintformat& operator%(const T &value) + hintformat & operator%(const T & value) { fmt % yellowtxt(value); return *this; } template - hintformat& operator%(const normaltxt &value) + hintformat & operator%(const normaltxt & value) { fmt % value.value; return *this; @@ -135,7 +139,7 @@ private: format fmt; }; -std::ostream& operator<<(std::ostream &os, const hintformat &hf); +std::ostream & operator<<(std::ostream & os, const hintformat & hf); template inline hintformat hintfmt(const std::string & fs, const Args & ... args) From be149acfdaae06667bc58df467c04c41827b69d0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 16:34:03 +0200 Subject: [PATCH 13/31] Serialize exceptions from the sandbox process to the parent Fixes #4118. --- src/libstore/build.cc | 16 ++++++++++----- src/libutil/serialise.cc | 42 ++++++++++++++++++++++++++++++++++++++++ src/libutil/serialise.hh | 3 +++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9497e7b3e..fa7283588 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2737,9 +2737,12 @@ void DerivationGoal::startBuilder() /* Check if setting up the build environment failed. */ while (true) { string msg = readLine(builderOut.readSide.get()); + if (string(msg, 0, 1) == "\2") break; if (string(msg, 0, 1) == "\1") { - if (msg.size() == 1) break; - throw Error(string(msg, 1)); + FdSource source(builderOut.readSide.get()); + auto ex = readError(source); + ex.addTrace({}, "while setting up the build environment"); + throw ex; } debug(msg); } @@ -3785,7 +3788,7 @@ void DerivationGoal::runChild() args.push_back(rewriteStrings(i, inputRewrites)); /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\1\n")); + writeFull(STDERR_FILENO, string("\2\n")); /* Execute the program. This should not return. */ if (drv->isBuiltin()) { @@ -3815,8 +3818,11 @@ void DerivationGoal::runChild() throw SysError("executing '%1%'", drv->builder); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); + } catch (Error & e) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); _exit(1); } } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index a469a1e73..5c9f6f901 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -266,6 +266,24 @@ Sink & operator << (Sink & sink, const StringSet & s) return sink; } +Sink & operator << (Sink & sink, const Error & ex) +{ + auto info = ex.info(); + sink + << "Error" + << info.level + << info.name + << info.description + << (info.hint ? info.hint->str() : "") + << 0 // FIXME: info.errPos + << info.traces.size(); + for (auto & trace : info.traces) { + sink << 0; // FIXME: trace.pos + sink << trace.hint.str(); + } + return sink; +} + void readPadding(size_t len, Source & source) { @@ -319,6 +337,30 @@ template Paths readStrings(Source & source); template PathSet readStrings(Source & source); +Error readError(Source & source) +{ + auto type = readString(source); + assert(type == "Error"); + ErrorInfo info; + info.level = (Verbosity) readInt(source); + info.name = readString(source); + info.description = readString(source); + auto hint = readString(source); + if (hint != "") info.hint = hintformat(std::move(format("%s") % hint)); + auto havePos = readNum(source); + assert(havePos == 0); + auto nrTraces = readNum(source); + for (size_t i = 0; i < nrTraces; ++i) { + havePos = readNum(source); + assert(havePos == 0); + info.traces.push_back(Trace { + .hint = hintformat(std::move(format("%s") % readString(source))) + }); + } + return Error(std::move(info)); +} + + void StringSink::operator () (const unsigned char * data, size_t len) { static bool warned = false; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index b41e58f33..d7fe0b81e 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -321,6 +321,7 @@ inline Sink & operator << (Sink & sink, uint64_t n) Sink & operator << (Sink & sink, const string & s); Sink & operator << (Sink & sink, const Strings & s); Sink & operator << (Sink & sink, const StringSet & s); +Sink & operator << (Sink & in, const Error & ex); MakeError(SerialisationError, Error); @@ -382,6 +383,8 @@ Source & operator >> (Source & in, bool & b) return in; } +Error readError(Source & source); + /* An adapter that converts a std::basic_istream into a source. */ struct StreamToSourceAdapter : Source From c43e882f54b5f09278dfb0d315239531c0676b4e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 17:13:54 +0200 Subject: [PATCH 14/31] Serialize exceptions from the daemon to the client --- src/libstore/daemon.cc | 23 ++++++++++++++++------- src/libstore/remote-store.cc | 10 +++++++--- src/libstore/worker-protocol.hh | 2 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ae2fbec35..99d8add92 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -101,17 +101,20 @@ struct TunnelLogger : public Logger /* stopWork() means that we're done; stop sending stderr to the client. */ - void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) + void stopWork(const Error * ex = nullptr) { auto state(state_.lock()); state->canSendStderr = false; - if (success) + if (!ex) to << STDERR_LAST; else { - to << STDERR_ERROR << msg; - if (status != 0) to << status; + if (GET_PROTOCOL_MINOR(clientVersion) >= 26) { + to << STDERR_ERROR << *ex; + } else { + to << STDERR_ERROR << ex->what() << ex->status; + } } } @@ -935,10 +938,11 @@ void processConnection( during addTextToStore() / importPath(). If that happens, just send the error message and exit. */ bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; - tunnelLogger->stopWork(false, e.msg(), e.status); + tunnelLogger->stopWork(&e); if (!errorAllowed) throw; } catch (std::bad_alloc & e) { - tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + auto ex = Error("Nix daemon out of memory"); + tunnelLogger->stopWork(&ex); throw; } @@ -947,8 +951,13 @@ void processConnection( assert(!tunnelLogger->state_.lock()->canSendStderr); }; + } catch (Error & e) { + tunnelLogger->stopWork(&e); + to.flush(); + return; } catch (std::exception & e) { - tunnelLogger->stopWork(false, e.what(), 1); + auto ex = Error(e.what()); + tunnelLogger->stopWork(&ex); to.flush(); return; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 40ef72293..23b1942ce 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -925,9 +925,13 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * } else if (msg == STDERR_ERROR) { - string error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + return std::make_exception_ptr(readError(from)); + } else { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } } else if (msg == STDERR_NEXT) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 564fb978a..b3705578e 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x119 +#define PROTOCOL_VERSION 0x11a #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From e705c2429445a4a05076fc067bb349bdd6752bd9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 17:28:43 +0200 Subject: [PATCH 15/31] Tweak error messages --- src/libstore/build.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index fa7283588..0585905e0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2355,7 +2355,8 @@ void DerivationGoal::startBuilder() worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); } catch (InvalidPath & e) { } catch (Error & e) { - throw Error("while processing 'sandbox-paths': %s", e.what()); + e.addTrace({}, "while processing 'sandbox-paths'"); + throw; } for (auto & i : closure) { auto p = worker.store.printStorePath(i); @@ -3809,7 +3810,7 @@ void DerivationGoal::runChild() throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { - writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + writeFull(STDERR_FILENO, e.what() + "\n"); _exit(1); } } From f66bbd8c7bb1472facf8917e58e3cd4f6ddfa1b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 21:25:06 +0200 Subject: [PATCH 16/31] Doh --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0585905e0..e1774237b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3810,7 +3810,7 @@ void DerivationGoal::runChild() throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + "\n"); + writeFull(STDERR_FILENO, e.what() + std::string("\n")); _exit(1); } } From 6aa64627c8e431c3b187f7bb44c943d06e39b929 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 22:02:36 +0200 Subject: [PATCH 17/31] Support user namespaces being disabled If max_user_namespaces is set to 0, then don't run the build in a user namespace. Fixes #4092. --- src/libstore/build.cc | 54 ++++++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e1774237b..a645c429d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -831,6 +831,10 @@ private: paths to the sandbox as a result of recursive Nix calls. */ AutoCloseFD sandboxMountNamespace; + /* On Linux, whether we're doing the build in its own user + namespace. */ + bool usingUserNamespace = true; + /* The build hook. */ std::unique_ptr hook; @@ -920,8 +924,8 @@ private: result. */ std::map prevInfos; - const uid_t sandboxUid = 1000; - const gid_t sandboxGid = 100; + uid_t sandboxUid = 1000; + gid_t sandboxGid = 100; const static Path homeDir; @@ -2629,6 +2633,24 @@ void DerivationGoal::startBuilder() options.allowVfork = false; + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + static bool userNamespacesEnabled = + pathExists(maxUserNamespaces) + && trim(readFile(maxUserNamespaces)) != "0"; + + usingUserNamespace = userNamespacesEnabled; + + if (usingUserNamespace) { + sandboxUid = 1000; + sandboxGid = 100; + } else { + debug("note: not using a user namespace"); + if (!buildUser) + throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); + sandboxUid = buildUser->getUID(); + sandboxGid = buildUser->getGID(); + } + Pid helper = startProcess([&]() { /* Drop additional groups here because we can't do it @@ -2647,9 +2669,11 @@ void DerivationGoal::startBuilder() PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); if (stack == MAP_FAILED) throw SysError("allocating stack"); - int flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (privateNetwork) flags |= CLONE_NEWNET; + if (usingUserNamespace) + flags |= CLONE_NEWUSER; pid_t child = clone(childEntry, stack + stackSize, flags, this); if (child == -1 && errno == EINVAL) { @@ -2697,19 +2721,21 @@ void DerivationGoal::startBuilder() if (!string2Int(readLine(builderOut.readSide.get()), tmp)) abort(); pid = tmp; - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + if (usingUserNamespace) { + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - (format("%d %d 1") % sandboxUid % hostUid).str()); + writeFile("/proc/" + std::to_string(pid) + "/uid_map", + (format("%d %d 1") % sandboxUid % hostUid).str()); - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - (format("%d %d 1") % sandboxGid % hostGid).str()); + writeFile("/proc/" + std::to_string(pid) + "/gid_map", + (format("%d %d 1") % sandboxGid % hostGid).str()); + } /* Save the mount namespace of the child. We have to do this *before* the child does a chroot. */ @@ -2745,7 +2771,7 @@ void DerivationGoal::startBuilder() ex.addTrace({}, "while setting up the build environment"); throw ex; } - debug(msg); + debug("sandbox setup: " + msg); } } From 97ffc1e0139e124b7e36b5d1a62b90300f231118 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 22:46:01 +0200 Subject: [PATCH 18/31] Dynamically disable user namespaces if CLONE_NEWUSER fails This makes builds work inside nixos-enter. Fixes #3145. --- src/libstore/build.cc | 45 ++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index a645c429d..05a9ef088 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -924,8 +924,8 @@ private: result. */ std::map prevInfos; - uid_t sandboxUid = 1000; - gid_t sandboxGid = 100; + uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } const static Path homeDir; @@ -2428,15 +2428,14 @@ void DerivationGoal::startBuilder() "root:x:0:0:Nix build user:%3%:/noshell\n" "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid, sandboxGid, settings.sandboxBuildDir)); + sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ writeFile(chrootRootDir + "/etc/group", - (format( - "root:x:0:\n" + fmt("root:x:0:\n" "nixbld:!:%1%:\n" - "nogroup:x:65534:\n") % sandboxGid).str()); + "nogroup:x:65534:\n", sandboxGid())); /* Create /etc/hosts with localhost entry. */ if (!(derivationIsImpure(derivationType))) @@ -2640,17 +2639,6 @@ void DerivationGoal::startBuilder() usingUserNamespace = userNamespacesEnabled; - if (usingUserNamespace) { - sandboxUid = 1000; - sandboxGid = 100; - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - sandboxUid = buildUser->getUID(); - sandboxGid = buildUser->getGID(); - } - Pid helper = startProcess([&]() { /* Drop additional groups here because we can't do it @@ -2682,11 +2670,12 @@ void DerivationGoal::startBuilder() flags &= ~CLONE_NEWPID; child = clone(childEntry, stack + stackSize, flags, this); } - if (child == -1 && (errno == EPERM || errno == EINVAL)) { + if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { /* Some distros patch Linux to not allow unprivileged * user namespaces. If we get EPERM or EINVAL, try * without CLONE_NEWUSER and see if that works. */ + usingUserNamespace = false; flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } @@ -2697,7 +2686,8 @@ void DerivationGoal::startBuilder() _exit(1); if (child == -1) throw SysError("cloning builder process"); - writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); + writeFull(builderOut.writeSide.get(), + fmt("%d %d\n", usingUserNamespace, child)); _exit(0); }, options); @@ -2718,7 +2708,10 @@ void DerivationGoal::startBuilder() }); pid_t tmp; - if (!string2Int(readLine(builderOut.readSide.get()), tmp)) abort(); + auto ss = tokenizeString>(readLine(builderOut.readSide.get())); + assert(ss.size() == 2); + usingUserNamespace = ss[0] == "1"; + if (!string2Int(ss[1], tmp)) abort(); pid = tmp; if (usingUserNamespace) { @@ -2729,12 +2722,16 @@ void DerivationGoal::startBuilder() uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); writeFile("/proc/" + std::to_string(pid) + "/uid_map", - (format("%d %d 1") % sandboxUid % hostUid).str()); + fmt("%d %d 1", sandboxUid(), hostUid)); writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", - (format("%d %d 1") % sandboxGid % hostGid).str()); + fmt("%d %d 1", sandboxGid(), hostGid)); + } else { + debug("note: not using a user namespace"); + if (!buildUser) + throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); } /* Save the mount namespace of the child. We have to do this @@ -3595,9 +3592,9 @@ void DerivationGoal::runChild() /* Switch to the sandbox uid/gid in the user namespace, which corresponds to the build user or calling user in the parent namespace. */ - if (setgid(sandboxGid) == -1) + if (setgid(sandboxGid()) == -1) throw SysError("setgid failed"); - if (setuid(sandboxUid) == -1) + if (setuid(sandboxUid()) == -1) throw SysError("setuid failed"); setUser = false; From eaef251b2ba416fcde8f0f0b72ce68c548ce35c3 Mon Sep 17 00:00:00 2001 From: Horki Date: Thu, 8 Oct 2020 13:40:47 +0200 Subject: [PATCH 19/31] rust: small patches --- nix-rust/src/lib.rs | 1 + nix-rust/src/store/path.rs | 35 +++++++++++++++++------------------ nix-rust/src/util/base32.rs | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 27ea69fbd..101de106f 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -1,3 +1,4 @@ +#[allow(improper_ctypes_definitions)] #[cfg(not(test))] mod c; mod error; diff --git a/nix-rust/src/store/path.rs b/nix-rust/src/store/path.rs index 47b5975c0..99f7a1f18 100644 --- a/nix-rust/src/store/path.rs +++ b/nix-rust/src/store/path.rs @@ -19,9 +19,9 @@ impl StorePath { } Self::new_from_base_name( path.file_name() - .ok_or(Error::BadStorePath(path.into()))? + .ok_or_else(|| Error::BadStorePath(path.into()))? .to_str() - .ok_or(Error::BadStorePath(path.into()))?, + .ok_or_else(|| Error::BadStorePath(path.into()))?, ) } @@ -34,7 +34,7 @@ impl StorePath { pub fn new_from_base_name(base_name: &str) -> Result { if base_name.len() < STORE_PATH_HASH_CHARS + 1 - || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8 + || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != b'-' { return Err(Error::BadStorePath(base_name.into())); } @@ -65,7 +65,7 @@ impl StorePathHash { Ok(Self(bytes)) } - pub fn hash<'a>(&'a self) -> &'a [u8; STORE_PATH_HASH_BYTES] { + pub fn hash(&self) -> &[u8; STORE_PATH_HASH_BYTES] { &self.0 } } @@ -98,7 +98,7 @@ pub struct StorePathName(String); impl StorePathName { pub fn new(s: &str) -> Result { - if s.len() == 0 { + if s.is_empty() { return Err(Error::StorePathNameEmpty); } @@ -106,25 +106,24 @@ impl StorePathName { return Err(Error::StorePathNameTooLong); } - if s.starts_with('.') - || !s.chars().all(|c| { - c.is_ascii_alphabetic() - || c.is_ascii_digit() - || c == '+' - || c == '-' - || c == '.' - || c == '_' - || c == '?' - || c == '=' - }) - { + let is_good_path_name = s.chars().all(|c| { + c.is_ascii_alphabetic() + || c.is_ascii_digit() + || c == '+' + || c == '-' + || c == '.' + || c == '_' + || c == '?' + || c == '=' + }); + if s.starts_with('.') || !is_good_path_name { return Err(Error::BadStorePathName); } Ok(Self(s.to_string())) } - pub fn name<'a>(&'a self) -> &'a str { + pub fn name(&self) -> &str { &self.0 } } diff --git a/nix-rust/src/util/base32.rs b/nix-rust/src/util/base32.rs index efd4a2901..7e71dc920 100644 --- a/nix-rust/src/util/base32.rs +++ b/nix-rust/src/util/base32.rs @@ -13,7 +13,7 @@ pub fn decoded_len(input_len: usize) -> usize { input_len * 5 / 8 } -static BASE32_CHARS: &'static [u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz"; +static BASE32_CHARS: &[u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz"; lazy_static! { static ref BASE32_CHARS_REVERSE: Box<[u8; 256]> = { From 58dadf295499588e492dab6bdc5934dc27ad3e64 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Oct 2020 17:30:40 +0200 Subject: [PATCH 20/31] Remove stray 'Title:' from the manual Closes #4096. --- doc/manual/generate-manpage.nix | 3 +-- doc/manual/local.mk | 15 ++++++++++++--- doc/manual/src/command-ref/conf-file-prefix.md | 2 -- doc/manual/src/command-ref/nix-build.md | 2 -- doc/manual/src/command-ref/nix-channel.md | 2 -- doc/manual/src/command-ref/nix-collect-garbage.md | 2 -- doc/manual/src/command-ref/nix-copy-closure.md | 2 -- doc/manual/src/command-ref/nix-daemon.md | 2 -- doc/manual/src/command-ref/nix-env.md | 2 -- doc/manual/src/command-ref/nix-hash.md | 2 -- doc/manual/src/command-ref/nix-instantiate.md | 2 -- doc/manual/src/command-ref/nix-prefetch-url.md | 2 -- doc/manual/src/command-ref/nix-shell.md | 2 -- doc/manual/src/command-ref/nix-store.md | 2 -- 14 files changed, 13 insertions(+), 29 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 4709c0e2c..db266750a 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -52,5 +52,4 @@ in command: -"Title: nix\n\n" -+ showCommand { command = "nix"; section = "#"; def = command; } +showCommand { command = "nix"; section = "#"; def = command; } diff --git a/doc/manual/local.mk b/doc/manual/local.mk index d308c7cf6..7d9a1a3e8 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -18,13 +18,22 @@ dist-files += $(man-pages) nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr $(d)/%.1: $(d)/src/command-ref/%.md - $(trace-gen) lowdown -sT man $^ -o $@ + @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp + @cat $^ >> $^.tmp + $(trace-gen) lowdown -sT man $^.tmp -o $@ + @rm $^.tmp $(d)/%.8: $(d)/src/command-ref/%.md - $(trace-gen) lowdown -sT man $^ -o $@ + @printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp + @cat $^ >> $^.tmp + $(trace-gen) lowdown -sT man $^.tmp -o $@ + @rm $^.tmp $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md - $(trace-gen) lowdown -sT man $^ -o $@ + @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp + @cat $^ >> $^.tmp + $(trace-gen) lowdown -sT man $^.tmp -o $@ + @rm $^.tmp $(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix $(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 04c6cd859..9987393d2 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -1,5 +1,3 @@ -Title: nix.conf - # Name `nix.conf` - Nix configuration file diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index 4bcb8db40..4565bfbc2 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -1,5 +1,3 @@ -Title: nix-build - # Name `nix-build` - build a Nix expression diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index f0e205967..4ca12d2cc 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -1,5 +1,3 @@ -Title: nix-channel - # Name `nix-channel` - manage Nix channels diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 62a6b7ca0..296165993 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -1,5 +1,3 @@ -Title: nix-collect-garbage - # Name `nix-collect-garbage` - delete unreachable store paths diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index 5ce320af7..dcb844a72 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -1,5 +1,3 @@ -Title: nix-copy-closure - # Name `nix-copy-closure` - copy a closure to or from a remote machine via SSH diff --git a/doc/manual/src/command-ref/nix-daemon.md b/doc/manual/src/command-ref/nix-daemon.md index bd5d25026..e91cb01dd 100644 --- a/doc/manual/src/command-ref/nix-daemon.md +++ b/doc/manual/src/command-ref/nix-daemon.md @@ -1,5 +1,3 @@ -Title: nix-daemon - # Name `nix-daemon` - Nix multi-user support daemon diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index ee838581b..1c23bb0ad 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -1,5 +1,3 @@ -Title: nix-env - # Name `nix-env` - manipulate or query Nix user environments diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md index d3f91f8e9..7ed82cdfc 100644 --- a/doc/manual/src/command-ref/nix-hash.md +++ b/doc/manual/src/command-ref/nix-hash.md @@ -1,5 +1,3 @@ -Title: nix-hash - # Name `nix-hash` - compute the cryptographic hash of a path diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index d09f5ed6a..c369397b6 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -1,5 +1,3 @@ -Title: nix-instantiate - # Name `nix-instantiate` - instantiate store derivations from Nix expressions diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 1307c7c37..78c612cd4 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -1,5 +1,3 @@ -Title: nix-prefetch-url - # Name `nix-prefetch-url` - copy a file from a URL into the store and print its hash diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index fc42a202a..45a5ff08c 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -1,5 +1,3 @@ -Title: nix-shell - # Name `nix-shell` - start an interactive shell based on a Nix expression diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 4680339e4..827adbd05 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -1,5 +1,3 @@ -Title: nix-store - # Name `nix-store` - manipulate or query the Nix store From 04e5d0e7040fdfbbc084634c0694ae7da89765d9 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 9 Oct 2020 09:39:51 +0200 Subject: [PATCH 21/31] Add a description in the completion outputs Make nix output completions in the form `completion\tdescription`. This can't be used by bash (afaik), but other shells like zsh or fish can display it along the completion choices --- misc/bash/completion.sh | 5 +++-- src/libmain/common-args.cc | 2 +- src/libutil/args.cc | 29 ++++++++++++++++++++--------- src/libutil/args.hh | 12 +++++++++++- src/nix/installables.cc | 12 ++++++------ src/nix/main.cc | 2 +- 6 files changed, 42 insertions(+), 20 deletions(-) diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index bc184edd6..bea2a40bc 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -4,13 +4,14 @@ function _complete_nix { _get_comp_words_by_ref -n ':=&' words cword cur local have_type while IFS= read -r line; do + local completion=${line%% *} if [[ -z $have_type ]]; then have_type=1 - if [[ $line = filenames ]]; then + if [[ $completion = filenames ]]; then compopt -o filenames fi else - COMPREPLY+=("$line") + COMPREPLY+=("$completion") fi done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}") __ltrim_colon_completions "$cur" diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 3411e2d7a..9151a0344 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -44,7 +44,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->insert(s.first); + completions->add(s.first, s.second.description); } } }); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 2760b830b..66d8df085 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -17,8 +17,19 @@ void Args::addFlag(Flag && flag_) if (flag->shortName) shortFlags[flag->shortName] = flag; } +void Completions::add(std::string completion, std::string description) +{ + insert(Completion{ + .completion = completion, + .description = description + }); +} + +bool Completion::operator<(const Completion & other) const +{ return completion < other.completion || (completion == other.completion && description < other.description); } + bool pathCompletions = false; -std::shared_ptr> completions; +std::shared_ptr completions; std::string completionMarker = "___COMPLETE___"; @@ -148,7 +159,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) - completions->insert("--" + name); + completions->add("--" + name, flag->description); } } auto i = longFlags.find(string(*pos, 2)); @@ -165,9 +176,9 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) if (auto prefix = needsCompletion(*pos)) { if (prefix == "-") { - completions->insert("--"); - for (auto & [flag, _] : shortFlags) - completions->insert(std::string("-") + flag); + completions->add("--"); + for (auto & [flagName, flag] : shortFlags) + completions->add(std::string("-") + flagName, flag->description); } } @@ -244,11 +255,11 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) - completions->insert(type); + completions->add(type); } Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) @@ -292,7 +303,7 @@ static void _completePath(std::string_view prefix, bool onlyDirs) auto st = lstat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } - completions->insert(globbuf.gl_pathv[i]); + completions->add(globbuf.gl_pathv[i]); } globfree(&globbuf); } @@ -385,7 +396,7 @@ MultiCommand::MultiCommand(const Commands & commands) if (auto prefix = needsCompletion(s)) { for (auto & [name, command] : commands) if (hasPrefix(name, *prefix)) - completions->insert(name); + completions->add(name); } auto i = commands.find(s); if (i == commands.end()) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index f41242e17..26f1bc11b 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -283,7 +283,17 @@ typedef std::vector> Table2; void printTable(std::ostream & out, const Table2 & table); -extern std::shared_ptr> completions; +struct Completion { + std::string completion; + std::string description; + + bool operator<(const Completion & other) const; +}; +class Completions : public std::set { +public: + void add(std::string completion, std::string description = ""); +}; +extern std::shared_ptr completions; extern bool pathCompletions; std::optional needsCompletion(std::string_view s); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 9bf6b7caa..7473c9758 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -26,7 +26,7 @@ void completeFlakeInputPath( auto flake = flake::getFlake(*evalState, flakeRef, true); for (auto & input : flake.inputs) if (hasPrefix(input.first, prefix)) - completions->insert(input.first); + completions->add(input.first); } MixFlakeOptions::MixFlakeOptions() @@ -211,7 +211,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = attr->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->insert(flakeRefS + "#" + concatStringsSep(".", attrPath2)); + completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2)); } } } @@ -222,7 +222,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->insert(flakeRefS + "#"); + completions->add(flakeRefS + "#"); } } } @@ -243,7 +243,7 @@ ref EvalCommand::getEvalState() void completeFlakeRef(ref store, std::string_view prefix) { if (prefix == "") - completions->insert("."); + completions->add("."); completeDir(0, prefix); @@ -254,10 +254,10 @@ void completeFlakeRef(ref store, std::string_view prefix) if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) - completions->insert(from2); + completions->add(from2); } else { if (hasPrefix(from, prefix)) - completions->insert(from); + completions->add(from); } } } diff --git a/src/nix/main.cc b/src/nix/main.cc index 1e9e07bc0..5056ceb78 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -208,7 +208,7 @@ void mainWrapped(int argc, char * * argv) if (completions) { std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n"); for (auto & s : *completions) - std::cout << s << "\n"; + std::cout << s.completion << "\t" << s.description << "\n"; } }); From eea310b2418e30280ae5fd9dac54ffe7f3c5dafa Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 9 Oct 2020 09:41:55 +0200 Subject: [PATCH 22/31] Add a zsh completion script Based on @clhodapp's suggestion in https://github.com/spwhitt/nix-zsh-completions/issues/32#issuecomment-705315356 and adapted to use the description of the completions --- misc/zsh/completion.zsh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 misc/zsh/completion.zsh diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh new file mode 100644 index 000000000..d4df6447e --- /dev/null +++ b/misc/zsh/completion.zsh @@ -0,0 +1,21 @@ +function _nix() { + local ifs_bk="$IFS" + local input=("${(Q)words[@]}") + IFS=$'\n' + local res=($(NIX_GET_COMPLETIONS=$((CURRENT - 1)) "$input[@]")) + IFS="$ifs_bk" + local tpe="${${res[1]}%%> *}" + local -a suggestions + declare -a suggestions + for suggestion in ${res:1}; do + # FIXME: This doesn't work properly if the suggestion word contains a `:` + # itself + suggestions+="${suggestion/ /:}" + done + if [[ "$tpe" == filenames ]]; then + compadd -f + fi + _describe 'nix' suggestions +} + +compdef _nix nix From 636ec171391660e986623cb75c400a99d652c991 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 15:41:24 +0200 Subject: [PATCH 23/31] Remove stray DerivationOutputsAndPaths type --- src/libstore/derivations.hh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d48266774..6d292b2e5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -61,8 +61,6 @@ typedef std::map DerivationOutputs; also contains, for each output, the (optional) store path in which it would be written. To calculate values of these types, see the corresponding functions in BasicDerivation */ -typedef std::map> - DerivationOutputsAndPaths; typedef std::map>> DerivationOutputsAndOptPaths; From 87157b2bd3224b8329ffcb73e92c64bdc36cff16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 16:02:53 +0200 Subject: [PATCH 24/31] writeFile(): Add error context to writeFull() failure Issue #4092. --- src/libutil/util.cc | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f07e99885..1a8873136 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -326,7 +326,12 @@ void writeFile(const Path & path, const string & s, mode_t mode) AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) throw SysError("opening file '%1%'", path); - writeFull(fd.get(), s); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } } @@ -338,11 +343,16 @@ void writeFile(const Path & path, Source & source, mode_t mode) std::vector buf(64 * 1024); - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), (unsigned char *) buf.data(), n); - } catch (EndOfFile &) { break; } + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), (unsigned char *) buf.data(), n); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; } } From e845d19ae368cb9ee6371c4b2fdbdc86a110d893 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 17:54:59 +0200 Subject: [PATCH 25/31] Remove Lazy This fixes a crash during startup when compiling Nix as a single compilation unit. --- src/libutil/lazy.hh | 48 --------------------------------------------- src/libutil/util.cc | 34 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 64 deletions(-) delete mode 100644 src/libutil/lazy.hh diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh deleted file mode 100644 index d073e486c..000000000 --- a/src/libutil/lazy.hh +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#include - -namespace nix { - -/* A helper class for lazily-initialized variables. - - Lazy var([]() { return value; }); - - declares a variable of type T that is initialized to 'value' (in a - thread-safe way) on first use, that is, when var() is first - called. If the initialiser code throws an exception, then all - subsequent calls to var() will rethrow that exception. */ -template -class Lazy -{ - - typedef std::function Init; - - Init init; - - std::once_flag done; - - T value; - - std::exception_ptr ex; - -public: - - Lazy(Init init) : init(init) - { } - - const T & operator () () - { - std::call_once(done, [&]() { - try { - value = init(); - } catch (...) { - ex = std::current_exception(); - } - }); - if (ex) std::rethrow_exception(ex); - return value; - } -}; - -} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1a8873136..9804e9a51 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,4 +1,3 @@ -#include "lazy.hh" #include "util.hh" #include "affinity.hh" #include "sync.hh" @@ -522,21 +521,24 @@ std::string getUserName() } -static Lazy getHome2([]() { - auto homeDir = getEnv("HOME"); - if (!homeDir) { - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; - } - return *homeDir; -}); - -Path getHome() { return getHome2(); } +Path getHome() +{ + static Path homeDir = []() + { + auto homeDir = getEnv("HOME"); + if (!homeDir) { + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + homeDir = pw->pw_dir; + } + return *homeDir; + }(); + return homeDir; +} Path getCacheDir() From 59bd6e87a44fab86c511f78ff50f3e3c4cca4b59 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 21:55:59 +0200 Subject: [PATCH 26/31] Completions::add(): Guard against newlines --- src/libutil/args.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 66d8df085..8bd9c8aeb 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -19,10 +19,11 @@ void Args::addFlag(Flag && flag_) void Completions::add(std::string completion, std::string description) { - insert(Completion{ - .completion = completion, - .description = description - }); + assert(description.find('\n') == std::string::npos); + insert(Completion { + .completion = completion, + .description = description + }); } bool Completion::operator<(const Completion & other) const From 44349064f75616fd47194425e27f5cb1d6ef5ecb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 22:02:00 +0200 Subject: [PATCH 27/31] nix develop: Source ~/.bashrc Fixes #4104. --- src/nix/develop.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index a46ea39b6..f763e8ba0 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -377,6 +377,10 @@ struct CmdDevelop : Common, MixEnvironment script += fmt("exec %s\n", concatStringsSep(" ", args)); } + else { + script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n"; + } + writeFull(rcFileFd.get(), script); stopProgressBar(); From 725488b8922c731042a557608b8d833b05e77243 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Oct 2020 22:03:18 +0200 Subject: [PATCH 28/31] nix develop: Unset $HOSTNAME This is set to "localhost" by stdenv which is probably not what you want. --- src/nix/develop.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index f763e8ba0..9372f43de 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -164,6 +164,7 @@ struct Common : InstallableCommand, MixProfile "BASHOPTS", "EUID", "HOME", // FIXME: don't ignore in pure mode? + "HOSTNAME", "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "NIX_LOG_FD", From 6cc1541782084111a8fa0a1e34d685a2e8c58459 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 9 Oct 2020 20:18:08 +0000 Subject: [PATCH 29/31] Split out `local-fs-store.hh` This matches the already-existing `local-fs-store.cc`. --- src/libfetchers/registry.cc | 1 + src/libstore/gc.cc | 1 + src/libstore/local-fs-store.cc | 1 + src/libstore/local-fs-store.hh | 48 ++++++++++++++++++++++++++ src/libstore/local-store.hh | 1 + src/libstore/profiles.cc | 1 + src/libstore/remote-store.hh | 1 + src/libstore/store-api.hh | 41 ---------------------- src/nix-build/nix-build.cc | 1 + src/nix-env/nix-env.cc | 1 + src/nix-env/user-env.cc | 1 + src/nix-instantiate/nix-instantiate.cc | 1 + src/nix/build.cc | 1 + src/nix/bundle.cc | 1 + src/nix/command.cc | 1 + src/nix/doctor.cc | 1 + 16 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 src/libstore/local-fs-store.hh diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 4367ee810..2426882ca 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -3,6 +3,7 @@ #include "util.hh" #include "globals.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 518a357ef..7a04b2f89 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,6 +1,7 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" +#include "local-fs-store.hh" #include "finally.hh" #include diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 2f1d9663a..e7c3dae92 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,6 +1,7 @@ #include "archive.hh" #include "fs-accessor.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "globals.hh" #include "compression.hh" #include "derivations.hh" diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh new file mode 100644 index 000000000..8eccd8236 --- /dev/null +++ b/src/libstore/local-fs-store.hh @@ -0,0 +1,48 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +struct LocalFSStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes + // it to omit the call to the Setting constructor. Clang works fine + // either way. + const PathSetting rootDir{(StoreConfig*) this, true, "", + "root", "directory prefixed to all other paths"}; + const PathSetting stateDir{(StoreConfig*) this, false, + rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + "state", "directory where Nix will store state"}; + const PathSetting logDir{(StoreConfig*) this, false, + rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + "log", "directory where Nix will store state"}; +}; + +class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +{ +public: + + const static string drvsLogDir; + + LocalFSStore(const Params & params); + + void narFromPath(const StorePath & path, Sink & sink) override; + ref getFSAccessor() override; + + /* Register a permanent GC root. */ + Path addPermRoot(const StorePath & storePath, const Path & gcRoot); + + virtual Path getRealStoreDir() { return storeDir; } + + Path toRealPath(const Path & storePath) override + { + assert(isInStore(storePath)); + return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); + } + + std::shared_ptr getBuildLog(const StorePath & path) override; +}; + +} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e7c9d1605..f1e2ab7f9 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -4,6 +4,7 @@ #include "pathlocks.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "sync.hh" #include "util.hh" diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index c3809bad7..ed10dd519 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,5 +1,6 @@ #include "profiles.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "util.hh" #include diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index ec04be985..554fb6bed 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -4,6 +4,7 @@ #include #include "store-api.hh" +#include "local-fs-store.hh" namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 450c0f554..f77bc21d1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -715,47 +715,6 @@ protected: }; -struct LocalFSStoreConfig : virtual StoreConfig -{ - using StoreConfig::StoreConfig; - // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes - // it to omit the call to the Setting constructor. Clang works fine - // either way. - const PathSetting rootDir{(StoreConfig*) this, true, "", - "root", "directory prefixed to all other paths"}; - const PathSetting stateDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, - "state", "directory where Nix will store state"}; - const PathSetting logDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, - "log", "directory where Nix will store state"}; -}; - -class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig -{ -public: - - const static string drvsLogDir; - - LocalFSStore(const Params & params); - - void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; - - /* Register a permanent GC root. */ - Path addPermRoot(const StorePath & storePath, const Path & gcRoot); - - virtual Path getRealStoreDir() { return storeDir; } - - Path toRealPath(const Path & storePath) override - { - assert(isInStore(storePath)); - return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); - } - - std::shared_ptr getBuildLog(const StorePath & path) override; -}; - /* Copy a path from one store to another. */ void copyStorePath(ref srcStore, ref dstStore, diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index a79b1086b..f60e0706c 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -6,6 +6,7 @@ #include #include "store-api.hh" +#include "local-fs-store.hh" #include "globals.hh" #include "derivations.hh" #include "affinity.hh" diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e6667e7f5..a4b5c9e2c 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -8,6 +8,7 @@ #include "profiles.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "user-env.hh" #include "util.hh" #include "json.hh" diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 8c6c8af05..87387e794 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "derivations.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "globals.hh" #include "shared.hh" #include "eval.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 18a0049a6..3956fef6d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -8,6 +8,7 @@ #include "value-to-json.hh" #include "util.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "common-eval-args.hh" #include "../nix/legacy.hh" diff --git a/src/nix/build.cc b/src/nix/build.cc index d85a482db..65708e98b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -3,6 +3,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" using namespace nix; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fc41da9e4..2d0a0b6ea 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "fs-accessor.hh" using namespace nix; diff --git a/src/nix/command.cc b/src/nix/command.cc index ba7de9fdd..9a38c77f1 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "derivations.hh" #include "nixexpr.hh" #include "profiles.hh" diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 4588ac05e..4f3003448 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -5,6 +5,7 @@ #include "serve-protocol.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "util.hh" #include "worker-protocol.hh" From 38e3897162825e8b3a7005df140a04b9acf0a2fb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 17:18:26 +0000 Subject: [PATCH 30/31] Copy {uds-,}remote-store.{cc,hh} This prepares for the splitting that happens in the next commit. --- src/libstore/uds-remote-store.cc | 1017 ++++++++++++++++++++++++++++++ src/libstore/uds-remote-store.hh | 204 ++++++ 2 files changed, 1221 insertions(+) create mode 100644 src/libstore/uds-remote-store.cc create mode 100644 src/libstore/uds-remote-store.hh diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc new file mode 100644 index 000000000..23b1942ce --- /dev/null +++ b/src/libstore/uds-remote-store.cc @@ -0,0 +1,1017 @@ +#include "serialise.hh" +#include "util.hh" +#include "remote-fs-accessor.hh" +#include "remote-store.hh" +#include "worker-protocol.hh" +#include "archive.hh" +#include "affinity.hh" +#include "globals.hh" +#include "derivations.hh" +#include "pool.hh" +#include "finally.hh" +#include "logging.hh" +#include "callback.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace nix { + +namespace worker_proto { + +std::string read(const Store & store, Source & from, Phantom _) +{ + return readString(from); +} + +void write(const Store & store, Sink & out, const std::string & str) +{ + out << str; +} + + +StorePath read(const Store & store, Source & from, Phantom _) +{ + return store.parseStorePath(readString(from)); +} + +void write(const Store & store, Sink & out, const StorePath & storePath) +{ + out << store.printStorePath(storePath); +} + + +ContentAddress read(const Store & store, Source & from, Phantom _) +{ + return parseContentAddress(readString(from)); +} + +void write(const Store & store, Sink & out, const ContentAddress & ca) +{ + out << renderContentAddress(ca); +} + + +std::optional read(const Store & store, Source & from, Phantom> _) +{ + auto s = readString(from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void write(const Store & store, Sink & out, const std::optional & storePathOpt) +{ + out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional read(const Store & store, Source & from, Phantom> _) +{ + return parseContentAddressOpt(readString(from)); +} + +void write(const Store & store, Sink & out, const std::optional & caOpt) +{ + out << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} + + +/* TODO: Separate these store impls into different files, give them better names */ +RemoteStore::RemoteStore(const Params & params) + : Store(params) + , RemoteStoreConfig(params) + , connections(make_ref>( + std::max(1, (int) maxConnections), + [this]() { + auto conn = openConnectionWrapper(); + try { + initConnection(*conn); + } catch (...) { + failed = true; + throw; + } + return conn; + }, + [this](const ref & r) { + return + r->to.good() + && r->from.good() + && std::chrono::duration_cast( + std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; + } + )) +{ +} + + +ref RemoteStore::openConnectionWrapper() +{ + if (failed) + throw Error("opening a connection to remote store '%s' previously failed", getUri()); + try { + return openConnection(); + } catch (...) { + failed = true; + throw; + } +} + + +UDSRemoteStore::UDSRemoteStore(const Params & params) + : StoreConfig(params) + , Store(params) + , LocalFSStore(params) + , RemoteStore(params) +{ +} + + +UDSRemoteStore::UDSRemoteStore( + const std::string scheme, + std::string socket_path, + const Params & params) + : UDSRemoteStore(params) +{ + path.emplace(socket_path); +} + + +std::string UDSRemoteStore::getUri() +{ + if (path) { + return std::string("unix://") + *path; + } else { + return "daemon"; + } +} + + +ref UDSRemoteStore::openConnection() +{ + auto conn = make_ref(); + + /* Connect to a daemon that does the privileged work for us. */ + conn->fd = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); + if (!conn->fd) + throw SysError("cannot create Unix domain socket"); + closeOnExec(conn->fd.get()); + + string socketPath = path ? *path : settings.nixDaemonSocketFile; + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPath.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%1%' is too long", socketPath); + strcpy(addr.sun_path, socketPath.c_str()); + + if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to daemon at '%1%'", socketPath); + + conn->from.fd = conn->fd.get(); + conn->to.fd = conn->fd.get(); + + conn->startTime = std::chrono::steady_clock::now(); + + return conn; +} + + +void RemoteStore::initConnection(Connection & conn) +{ + /* Send the magic greeting, check for the reply. */ + try { + conn.to << WORKER_MAGIC_1; + conn.to.flush(); + unsigned int magic = readInt(conn.from); + if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); + + conn.from >> conn.daemonVersion; + if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + throw Error("Nix daemon protocol version not supported"); + if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) + throw Error("the Nix daemon version is too old"); + conn.to << PROTOCOL_VERSION; + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { + int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; + if (cpu != -1) + conn.to << 1 << cpu; + else + conn.to << 0; + } + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) + conn.to << false; + + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); + } + catch (Error & e) { + throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); + } + + setOptions(conn); +} + + +void RemoteStore::setOptions(Connection & conn) +{ + conn.to << wopSetOptions + << settings.keepFailed + << settings.keepGoing + << settings.tryFallback + << verbosity + << settings.maxBuildJobs + << settings.maxSilentTime + << true + << (settings.verboseBuild ? lvlError : lvlVomit) + << 0 // obsolete log type + << 0 /* obsolete print build trace */ + << settings.buildCores + << settings.useSubstitutes; + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + std::map overrides; + globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); + overrides.erase(loggerSettings.showTrace.name); + conn.to << overrides.size(); + for (auto & i : overrides) + conn.to << i.first << i.second.value; + } + + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); +} + + +/* A wrapper around Pool::Handle that marks + the connection as bad (causing it to be closed) if a non-daemon + exception is thrown before the handle is closed. Such an exception + causes a deviation from the expected protocol and therefore a + desynchronization between the client and daemon. */ +struct ConnectionHandle +{ + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle() + { + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); + } + } + + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) + { + auto ex = handle->processStderr(sink, source, flush); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); + } + } + + void withFramedSink(std::function fun); +}; + + +ConnectionHandle RemoteStore::getConnection() +{ + return ConnectionHandle(connections->get()); +} + + +bool RemoteStore::isValidPathUncached(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopIsValidPath << printStorePath(path); + conn.processStderr(); + return readInt(conn->from); +} + + +StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + StorePathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; + } else { + conn->to << wopQueryValidPaths; + worker_proto::write(*this, conn->to, paths); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); + } +} + + +StorePathSet RemoteStore::queryAllValidPaths() +{ + auto conn(getConnection()); + conn->to << wopQueryAllValidPaths; + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); +} + + +StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) +{ + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + StorePathSet res; + for (auto & i : paths) { + conn->to << wopHasSubstitutes << printStorePath(i); + conn.processStderr(); + if (readInt(conn->from)) res.insert(i); + } + return res; + } else { + conn->to << wopQuerySubstitutablePaths; + worker_proto::write(*this, conn->to, paths); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); + } +} + + +void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) +{ + if (pathsMap.empty()) return; + + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + + for (auto & i : pathsMap) { + SubstitutablePathInfo info; + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); + conn.processStderr(); + unsigned int reply = readInt(conn->from); + if (reply == 0) continue; + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); + infos.insert_or_assign(i.first, std::move(info)); + } + + } else { + + conn->to << wopQuerySubstitutablePathInfos; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); + worker_proto::write(*this, conn->to, paths); + } else + worker_proto::write(*this, conn->to, pathsMap); + conn.processStderr(); + size_t count = readNum(conn->from); + for (size_t n = 0; n < count; n++) { + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); + } + + } +} + + +ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) +{ + auto deriver = readString(conn->from); + auto narHash = Hash::parseAny(readString(conn->from), htSHA256); + auto info = make_ref(path, narHash); + if (deriver != "") info->deriver = parseStorePath(deriver); + info->references = worker_proto::read(*this, conn->from, Phantom {}); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings(conn->from); + info->ca = parseContentAddressOpt(readString(conn->from)); + } + return info; +} + + +void RemoteStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + try { + std::shared_ptr info; + { + auto conn(getConnection()); + conn->to << wopQueryPathInfo << printStorePath(path); + try { + conn.processStderr(); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(e.info()); + throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid; conn->from >> valid; + if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); + } + info = readValidPathInfo(conn, path); + } + callback(std::move(info)); + } catch (...) { callback.rethrow(); } +} + + +void RemoteStore::queryReferrers(const StorePath & path, + StorePathSet & referrers) +{ + auto conn(getConnection()); + conn->to << wopQueryReferrers << printStorePath(path); + conn.processStderr(); + for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) + referrers.insert(i); +} + + +StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopQueryValidDerivers << printStorePath(path); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); +} + + +StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) +{ + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { + return Store::queryDerivationOutputs(path); + } + conn->to << wopQueryDerivationOutputs << printStorePath(path); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom {}); +} + + +std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) +{ + if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom>> {}); + } else { + // Fallback for old daemon versions. + // For floating-CA derivations (and their co-dependencies) this is an + // under-approximation as it only returns the paths that can be inferred + // from the derivation itself (and not the ones that are known because + // the have been built), but as old stores don't handle floating-CA + // derivations this shouldn't matter + auto derivation = readDerivation(path); + auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); + std::map> ret; + for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { + ret.emplace(outputName, outputAndPath.second); + } + return ret; + } +} + +std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) +{ + auto conn(getConnection()); + conn->to << wopQueryPathFromHashPart << hashPart; + conn.processStderr(); + Path path = readString(conn->from); + if (path.empty()) return {}; + return parseStorePath(path); +} + + +ref RemoteStore::addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair) +{ + std::optional conn_(getConnection()); + auto & conn = *conn_; + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + + conn->to + << wopAddToStore + << name + << renderContentAddressMethod(caMethod); + worker_proto::write(*this, conn->to, references); + conn->to << repair; + + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + + auto path = parseStorePath(readString(conn->from)); + return readValidPathInfo(conn, path); + } + else { + if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); + + std::visit(overloaded { + [&](TextHashMethod thm) -> void { + std::string s = dump.drain(); + conn->to << wopAddTextToStore << name << s; + worker_proto::write(*this, conn->to, references); + conn.processStderr(); + }, + [&](FixedOutputHashMethod fohm) -> void { + conn->to + << wopAddToStore + << name + << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(fohm.hashType); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + dump.drainInto(conn->to); + } else { + std::string contents = dump.drain(); + dumpString(contents, conn->to); + } + } + conn->to.warn = false; + conn.processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + + } + }, caMethod); + auto path = parseStorePath(readString(conn->from)); + // Release our connection to prevent a deadlock in queryPathInfo(). + conn_.reset(); + return queryPathInfo(path); + } +} + + +StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashType, RepairFlag repair) +{ + StorePathSet references; + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; +} + + +void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { + conn->to << wopImportPaths; + + auto source2 = sinkToSource([&](Sink & sink) { + sink << 1 // == path follows + ; + copyNAR(source, sink); + sink + << exportMagic + << printStorePath(info.path); + worker_proto::write(*this, sink, info.references); + sink + << (info.deriver ? printStorePath(*info.deriver) : "") + << 0 // == no legacy signature + << 0 // == no path follows + ; + }); + + conn.processStderr(0, source2.get()); + + auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); + assert(importedPaths.size() <= 1); + } + + else { + conn->to << wopAddToStoreNar + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(Base16, false); + worker_proto::write(*this, conn->to, info.references); + conn->to << info.registrationTime << info.narSize + << info.ultimate << info.sigs << renderContentAddress(info.ca) + << repair << !checkSigs; + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + conn.withFramedSink([&](Sink & sink) { + copyNAR(source, sink); + }); + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); + } + } +} + + +StorePath RemoteStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) +{ + StringSource source(s); + return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; +} + + +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +{ + auto conn(getConnection()); + conn->to << wopBuildPaths; + assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); + Strings ss; + for (auto & p : drvPaths) + ss.push_back(p.to_string(*this)); + conn->to << ss; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << buildMode; + else + /* Old daemons did not take a 'buildMode' parameter, so we + need to validate it here on the client side. */ + if (buildMode != bmNormal) + throw Error("repairing or checking is not supported when building through the Nix daemon"); + conn.processStderr(); + readInt(conn->from); +} + + +BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + auto conn(getConnection()); + conn->to << wopBuildDerivation << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + conn->to << buildMode; + conn.processStderr(); + BuildResult res; + unsigned int status; + conn->from >> status >> res.errorMsg; + res.status = (BuildResult::Status) status; + return res; +} + + +void RemoteStore::ensurePath(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopEnsurePath << printStorePath(path); + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::addTempRoot(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopAddTempRoot << printStorePath(path); + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::addIndirectRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << wopAddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::syncWithGC() +{ + auto conn(getConnection()); + conn->to << wopSyncWithGC; + conn.processStderr(); + readInt(conn->from); +} + + +Roots RemoteStore::findRoots(bool censor) +{ + auto conn(getConnection()); + conn->to << wopFindRoots; + conn.processStderr(); + size_t count = readNum(conn->from); + Roots result; + while (count--) { + Path link = readString(conn->from); + auto target = parseStorePath(readString(conn->from)); + result[std::move(target)].emplace(link); + } + return result; +} + + +void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + auto conn(getConnection()); + + conn->to + << wopCollectGarbage << options.action; + worker_proto::write(*this, conn->to, options.pathsToDelete); + conn->to << options.ignoreLiveness + << options.maxFreed + /* removed options */ + << 0 << 0 << 0; + + conn.processStderr(); + + results.paths = readStrings(conn->from); + results.bytesFreed = readLongLong(conn->from); + readLongLong(conn->from); // obsolete + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.clear(); + } +} + + +void RemoteStore::optimiseStore() +{ + auto conn(getConnection()); + conn->to << wopOptimiseStore; + conn.processStderr(); + readInt(conn->from); +} + + +bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) +{ + auto conn(getConnection()); + conn->to << wopVerifyStore << checkContents << repair; + conn.processStderr(); + return readInt(conn->from); +} + + +void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) +{ + auto conn(getConnection()); + conn->to << wopAddSignatures << printStorePath(storePath) << sigs; + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) +{ + { + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + // Don't hold the connection handle in the fallback case + // to prevent a deadlock. + goto fallback; + conn->to << wopQueryMissing; + Strings ss; + for (auto & p : targets) + ss.push_back(p.to_string(*this)); + conn->to << ss; + conn.processStderr(); + willBuild = worker_proto::read(*this, conn->from, Phantom {}); + willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); + unknown = worker_proto::read(*this, conn->from, Phantom {}); + conn->from >> downloadSize >> narSize; + return; + } + + fallback: + return Store::queryMissing(targets, willBuild, willSubstitute, + unknown, downloadSize, narSize); +} + + +void RemoteStore::connect() +{ + auto conn(getConnection()); +} + + +unsigned int RemoteStore::getProtocol() +{ + auto conn(connections->get()); + return conn->daemonVersion; +} + + +void RemoteStore::flushBadConnections() +{ + connections->flushBad(); +} + + +RemoteStore::Connection::~Connection() +{ + try { + to.flush(); + } catch (...) { + ignoreException(); + } +} + +void RemoteStore::narFromPath(const StorePath & path, Sink & sink) +{ + auto conn(connections->get()); + conn->to << wopNarFromPath << printStorePath(path); + conn->processStderr(); + copyNAR(conn->from, sink); +} + +ref RemoteStore::getFSAccessor() +{ + return make_ref(ref(shared_from_this())); +} + +static Logger::Fields readFields(Source & from) +{ + Logger::Fields fields; + size_t size = readInt(from); + for (size_t n = 0; n < size; n++) { + auto type = (decltype(Logger::Field::type)) readInt(from); + if (type == Logger::Field::tInt) + fields.push_back(readNum(from)); + else if (type == Logger::Field::tString) + fields.push_back(readString(from)); + else + throw Error("got unsupported field type %x from Nix daemon", (int) type); + } + return fields; +} + + +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush) +{ + if (flush) + to.flush(); + + while (true) { + + auto msg = readNum(from); + + if (msg == STDERR_WRITE) { + string s = readString(from); + if (!sink) throw Error("no sink"); + (*sink)(s); + } + + else if (msg == STDERR_READ) { + if (!source) throw Error("no source"); + size_t len = readNum(from); + auto buf = std::make_unique(len); + writeString(buf.get(), source->read(buf.get(), len), to); + to.flush(); + } + + else if (msg == STDERR_ERROR) { + if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + return std::make_exception_ptr(readError(from)); + } else { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } + } + + else if (msg == STDERR_NEXT) + printError(chomp(readString(from))); + + else if (msg == STDERR_START_ACTIVITY) { + auto act = readNum(from); + auto lvl = (Verbosity) readInt(from); + auto type = (ActivityType) readInt(from); + auto s = readString(from); + auto fields = readFields(from); + auto parent = readNum(from); + logger->startActivity(act, lvl, type, s, fields, parent); + } + + else if (msg == STDERR_STOP_ACTIVITY) { + auto act = readNum(from); + logger->stopActivity(act); + } + + else if (msg == STDERR_RESULT) { + auto act = readNum(from); + auto type = (ResultType) readInt(from); + auto fields = readFields(from); + logger->result(act, type, fields); + } + + else if (msg == STDERR_LAST) + break; + + else + throw Error("got unknown message type %x from Nix daemon", msg); + } + + return nullptr; +} + +void ConnectionHandle::withFramedSink(std::function fun) +{ + (*this)->to.flush(); + + std::exception_ptr ex; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + processStderr(nullptr, nullptr, false); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink((*this)->to, ex); + fun(sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + +} + +static RegisterStoreImplementation regUDSRemoteStore; + +} diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh new file mode 100644 index 000000000..554fb6bed --- /dev/null +++ b/src/libstore/uds-remote-store.hh @@ -0,0 +1,204 @@ +#pragma once + +#include +#include + +#include "store-api.hh" +#include "local-fs-store.hh" + + +namespace nix { + + +class Pipe; +class Pid; +struct FdSink; +struct FdSource; +template class Pool; +struct ConnectionHandle; + +struct RemoteStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + + const Setting maxConnections{(StoreConfig*) this, 1, + "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + + const Setting maxConnectionAge{(StoreConfig*) this, std::numeric_limits::max(), + "max-connection-age", "number of seconds to reuse a connection"}; +}; + +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +{ +public: + + virtual bool sameMachine() = 0; + + RemoteStore(const Params & params); + + /* Implementations of abstract store API methods. */ + + bool isValidPathUncached(const StorePath & path) override; + + StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + + StorePathSet queryAllValidPaths() override; + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; + + StorePathSet queryValidDerivers(const StorePath & path) override; + + StorePathSet queryDerivationOutputs(const StorePath & path) override; + + std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::optional queryPathFromHashPart(const std::string & hashPart) override; + + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; + + void querySubstitutablePathInfos(const StorePathCAMap & paths, + SubstitutablePathInfos & infos) override; + + /* Add a content-addressable store path. `dump` will be drained. */ + ref addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair); + + /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + + void addToStore(const ValidPathInfo & info, Source & nar, + RepairFlag repair, CheckSigsFlag checkSigs) override; + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; + + void buildPaths(const std::vector & paths, BuildMode buildMode) override; + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; + + void ensurePath(const StorePath & path) override; + + void addTempRoot(const StorePath & path) override; + + void addIndirectRoot(const Path & path) override; + + void syncWithGC() override; + + Roots findRoots(bool censor) override; + + void collectGarbage(const GCOptions & options, GCResults & results) override; + + void optimiseStore() override; + + bool verifyStore(bool checkContents, RepairFlag repair) override; + + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; + + void queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) override; + + void connect() override; + + unsigned int getProtocol() override; + + void flushBadConnections(); + + struct Connection + { + AutoCloseFD fd; + FdSink to; + FdSource from; + unsigned int daemonVersion; + std::chrono::time_point startTime; + + virtual ~Connection(); + + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + }; + + ref openConnectionWrapper(); + +protected: + + virtual ref openConnection() = 0; + + void initConnection(Connection & conn); + + ref> connections; + + virtual void setOptions(Connection & conn); + + ConnectionHandle getConnection(); + + friend struct ConnectionHandle; + + virtual ref getFSAccessor() override; + + virtual void narFromPath(const StorePath & path, Sink & sink) override; + + ref readValidPathInfo(ConnectionHandle & conn, const StorePath & path); + +private: + + std::atomic_bool failed{false}; + +}; + +struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +{ + UDSRemoteStoreConfig(const Store::Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + { + } + + UDSRemoteStoreConfig() + : UDSRemoteStoreConfig(Store::Params({})) + { + } + + const std::string name() override { return "Local Daemon Store"; } +}; + +class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig +{ +public: + + UDSRemoteStore(const Params & params); + UDSRemoteStore(const std::string scheme, std::string path, const Params & params); + + std::string getUri() override; + + static std::set uriSchemes() + { return {"unix"}; } + + bool sameMachine() override + { return true; } + + ref getFSAccessor() override + { return LocalFSStore::getFSAccessor(); } + + void narFromPath(const StorePath & path, Sink & sink) override + { LocalFSStore::narFromPath(path, sink); } + +private: + + ref openConnection() override; + std::optional path; +}; + + +} From 15fdb7cc6bb9e86283a67cce646981821dca1558 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 16:01:53 +0000 Subject: [PATCH 31/31] Split out uds-remote-store.{cc.hh} --- src/libstore/remote-store.cc | 75 --- src/libstore/remote-store.hh | 45 -- src/libstore/store-api.cc | 2 +- src/libstore/uds-remote-store.cc | 940 +------------------------------ src/libstore/uds-remote-store.hh | 154 +---- 5 files changed, 4 insertions(+), 1212 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 23b1942ce..488270f48 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -12,16 +12,6 @@ #include "logging.hh" #include "callback.hh" -#include -#include -#include -#include -#include -#include -#include - -#include - namespace nix { namespace worker_proto { @@ -125,69 +115,6 @@ ref RemoteStore::openConnectionWrapper() } -UDSRemoteStore::UDSRemoteStore(const Params & params) - : StoreConfig(params) - , Store(params) - , LocalFSStore(params) - , RemoteStore(params) -{ -} - - -UDSRemoteStore::UDSRemoteStore( - const std::string scheme, - std::string socket_path, - const Params & params) - : UDSRemoteStore(params) -{ - path.emplace(socket_path); -} - - -std::string UDSRemoteStore::getUri() -{ - if (path) { - return std::string("unix://") + *path; - } else { - return "daemon"; - } -} - - -ref UDSRemoteStore::openConnection() -{ - auto conn = make_ref(); - - /* Connect to a daemon that does the privileged work for us. */ - conn->fd = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!conn->fd) - throw SysError("cannot create Unix domain socket"); - closeOnExec(conn->fd.get()); - - string socketPath = path ? *path : settings.nixDaemonSocketFile; - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%1%' is too long", socketPath); - strcpy(addr.sun_path, socketPath.c_str()); - - if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to daemon at '%1%'", socketPath); - - conn->from.fd = conn->fd.get(); - conn->to.fd = conn->fd.get(); - - conn->startTime = std::chrono::steady_clock::now(); - - return conn; -} - - void RemoteStore::initConnection(Connection & conn) { /* Send the magic greeting, check for the reply. */ @@ -1012,6 +939,4 @@ void ConnectionHandle::withFramedSink(std::function fun) } -static RegisterStoreImplementation regUDSRemoteStore; - } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 554fb6bed..9f78fcb02 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -4,7 +4,6 @@ #include #include "store-api.hh" -#include "local-fs-store.hh" namespace nix { @@ -156,49 +155,5 @@ private: }; -struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig -{ - UDSRemoteStoreConfig(const Store::Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - { - } - - UDSRemoteStoreConfig() - : UDSRemoteStoreConfig(Store::Params({})) - { - } - - const std::string name() override { return "Local Daemon Store"; } -}; - -class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig -{ -public: - - UDSRemoteStore(const Params & params); - UDSRemoteStore(const std::string scheme, std::string path, const Params & params); - - std::string getUri() override; - - static std::set uriSchemes() - { return {"unix"}; } - - bool sameMachine() override - { return true; } - - ref getFSAccessor() override - { return LocalFSStore::getFSAccessor(); } - - void narFromPath(const StorePath & path, Sink & sink) override - { LocalFSStore::narFromPath(path, sink); } - -private: - - ref openConnection() override; - std::optional path; -}; - } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1bbc74db8..9f21f0434 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1011,7 +1011,7 @@ Derivation Store::readDerivation(const StorePath & drvPath) #include "local-store.hh" -#include "remote-store.hh" +#include "uds-remote-store.hh" namespace nix { diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 23b1942ce..24f3e9c6d 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,16 +1,4 @@ -#include "serialise.hh" -#include "util.hh" -#include "remote-fs-accessor.hh" -#include "remote-store.hh" -#include "worker-protocol.hh" -#include "archive.hh" -#include "affinity.hh" -#include "globals.hh" -#include "derivations.hh" -#include "pool.hh" -#include "finally.hh" -#include "logging.hh" -#include "callback.hh" +#include "uds-remote-store.hh" #include #include @@ -22,109 +10,9 @@ #include + namespace nix { -namespace worker_proto { - -std::string read(const Store & store, Source & from, Phantom _) -{ - return readString(from); -} - -void write(const Store & store, Sink & out, const std::string & str) -{ - out << str; -} - - -StorePath read(const Store & store, Source & from, Phantom _) -{ - return store.parseStorePath(readString(from)); -} - -void write(const Store & store, Sink & out, const StorePath & storePath) -{ - out << store.printStorePath(storePath); -} - - -ContentAddress read(const Store & store, Source & from, Phantom _) -{ - return parseContentAddress(readString(from)); -} - -void write(const Store & store, Sink & out, const ContentAddress & ca) -{ - out << renderContentAddress(ca); -} - - -std::optional read(const Store & store, Source & from, Phantom> _) -{ - auto s = readString(from); - return s == "" ? std::optional {} : store.parseStorePath(s); -} - -void write(const Store & store, Sink & out, const std::optional & storePathOpt) -{ - out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); -} - - -std::optional read(const Store & store, Source & from, Phantom> _) -{ - return parseContentAddressOpt(readString(from)); -} - -void write(const Store & store, Sink & out, const std::optional & caOpt) -{ - out << (caOpt ? renderContentAddress(*caOpt) : ""); -} - -} - - -/* TODO: Separate these store impls into different files, give them better names */ -RemoteStore::RemoteStore(const Params & params) - : Store(params) - , RemoteStoreConfig(params) - , connections(make_ref>( - std::max(1, (int) maxConnections), - [this]() { - auto conn = openConnectionWrapper(); - try { - initConnection(*conn); - } catch (...) { - failed = true; - throw; - } - return conn; - }, - [this](const ref & r) { - return - r->to.good() - && r->from.good() - && std::chrono::duration_cast( - std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; - } - )) -{ -} - - -ref RemoteStore::openConnectionWrapper() -{ - if (failed) - throw Error("opening a connection to remote store '%s' previously failed", getUri()); - try { - return openConnection(); - } catch (...) { - failed = true; - throw; - } -} - - UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) , Store(params) @@ -188,830 +76,6 @@ ref UDSRemoteStore::openConnection() } -void RemoteStore::initConnection(Connection & conn) -{ - /* Send the magic greeting, check for the reply. */ - try { - conn.to << WORKER_MAGIC_1; - conn.to.flush(); - unsigned int magic = readInt(conn.from); - if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - - conn.from >> conn.daemonVersion; - if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) - throw Error("Nix daemon protocol version not supported"); - if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) - throw Error("the Nix daemon version is too old"); - conn.to << PROTOCOL_VERSION; - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; - if (cpu != -1) - conn.to << 1 << cpu; - else - conn.to << 0; - } - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) - conn.to << false; - - auto ex = conn.processStderr(); - if (ex) std::rethrow_exception(ex); - } - catch (Error & e) { - throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); - } - - setOptions(conn); -} - - -void RemoteStore::setOptions(Connection & conn) -{ - conn.to << wopSetOptions - << settings.keepFailed - << settings.keepGoing - << settings.tryFallback - << verbosity - << settings.maxBuildJobs - << settings.maxSilentTime - << true - << (settings.verboseBuild ? lvlError : lvlVomit) - << 0 // obsolete log type - << 0 /* obsolete print build trace */ - << settings.buildCores - << settings.useSubstitutes; - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - std::map overrides; - globalConfig.getSettings(overrides, true); - overrides.erase(settings.keepFailed.name); - overrides.erase(settings.keepGoing.name); - overrides.erase(settings.tryFallback.name); - overrides.erase(settings.maxBuildJobs.name); - overrides.erase(settings.maxSilentTime.name); - overrides.erase(settings.buildCores.name); - overrides.erase(settings.useSubstitutes.name); - overrides.erase(loggerSettings.showTrace.name); - conn.to << overrides.size(); - for (auto & i : overrides) - conn.to << i.first << i.second.value; - } - - auto ex = conn.processStderr(); - if (ex) std::rethrow_exception(ex); -} - - -/* A wrapper around Pool::Handle that marks - the connection as bad (causing it to be closed) if a non-daemon - exception is thrown before the handle is closed. Such an exception - causes a deviation from the expected protocol and therefore a - desynchronization between the client and daemon. */ -struct ConnectionHandle -{ - Pool::Handle handle; - bool daemonException = false; - - ConnectionHandle(Pool::Handle && handle) - : handle(std::move(handle)) - { } - - ConnectionHandle(ConnectionHandle && h) - : handle(std::move(h.handle)) - { } - - ~ConnectionHandle() - { - if (!daemonException && std::uncaught_exceptions()) { - handle.markBad(); - debug("closing daemon connection because of an exception"); - } - } - - RemoteStore::Connection * operator -> () { return &*handle; } - - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) - { - auto ex = handle->processStderr(sink, source, flush); - if (ex) { - daemonException = true; - std::rethrow_exception(ex); - } - } - - void withFramedSink(std::function fun); -}; - - -ConnectionHandle RemoteStore::getConnection() -{ - return ConnectionHandle(connections->get()); -} - - -bool RemoteStore::isValidPathUncached(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopIsValidPath << printStorePath(path); - conn.processStderr(); - return readInt(conn->from); -} - - -StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) -{ - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - StorePathSet res; - for (auto & i : paths) - if (isValidPath(i)) res.insert(i); - return res; - } else { - conn->to << wopQueryValidPaths; - worker_proto::write(*this, conn->to, paths); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); - } -} - - -StorePathSet RemoteStore::queryAllValidPaths() -{ - auto conn(getConnection()); - conn->to << wopQueryAllValidPaths; - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); -} - - -StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) -{ - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - StorePathSet res; - for (auto & i : paths) { - conn->to << wopHasSubstitutes << printStorePath(i); - conn.processStderr(); - if (readInt(conn->from)) res.insert(i); - } - return res; - } else { - conn->to << wopQuerySubstitutablePaths; - worker_proto::write(*this, conn->to, paths); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); - } -} - - -void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) -{ - if (pathsMap.empty()) return; - - auto conn(getConnection()); - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - - for (auto & i : pathsMap) { - SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); - conn.processStderr(); - unsigned int reply = readInt(conn->from); - if (reply == 0) continue; - auto deriver = readString(conn->from); - if (deriver != "") - info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); - info.downloadSize = readLongLong(conn->from); - info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.first, std::move(info)); - } - - } else { - - conn->to << wopQuerySubstitutablePathInfos; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); - worker_proto::write(*this, conn->to, paths); - } else - worker_proto::write(*this, conn->to, pathsMap); - conn.processStderr(); - size_t count = readNum(conn->from); - for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); - auto deriver = readString(conn->from); - if (deriver != "") - info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); - info.downloadSize = readLongLong(conn->from); - info.narSize = readLongLong(conn->from); - } - - } -} - - -ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) -{ - auto deriver = readString(conn->from); - auto narHash = Hash::parseAny(readString(conn->from), htSHA256); - auto info = make_ref(path, narHash); - if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom {}); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); - } - return info; -} - - -void RemoteStore::queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept -{ - try { - std::shared_ptr info; - { - auto conn(getConnection()); - conn->to << wopQueryPathInfo << printStorePath(path); - try { - conn.processStderr(); - } catch (Error & e) { - // Ugly backwards compatibility hack. - if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.info()); - throw; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { - bool valid; conn->from >> valid; - if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); - } - info = readValidPathInfo(conn, path); - } - callback(std::move(info)); - } catch (...) { callback.rethrow(); } -} - - -void RemoteStore::queryReferrers(const StorePath & path, - StorePathSet & referrers) -{ - auto conn(getConnection()); - conn->to << wopQueryReferrers << printStorePath(path); - conn.processStderr(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) - referrers.insert(i); -} - - -StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopQueryValidDerivers << printStorePath(path); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); -} - - -StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) -{ - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { - return Store::queryDerivationOutputs(path); - } - conn->to << wopQueryDerivationOutputs << printStorePath(path); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); -} - - -std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) -{ - if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom>> {}); - } else { - // Fallback for old daemon versions. - // For floating-CA derivations (and their co-dependencies) this is an - // under-approximation as it only returns the paths that can be inferred - // from the derivation itself (and not the ones that are known because - // the have been built), but as old stores don't handle floating-CA - // derivations this shouldn't matter - auto derivation = readDerivation(path); - auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); - std::map> ret; - for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { - ret.emplace(outputName, outputAndPath.second); - } - return ret; - } -} - -std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) -{ - auto conn(getConnection()); - conn->to << wopQueryPathFromHashPart << hashPart; - conn.processStderr(); - Path path = readString(conn->from); - if (path.empty()) return {}; - return parseStorePath(path); -} - - -ref RemoteStore::addCAToStore( - Source & dump, - const string & name, - ContentAddressMethod caMethod, - const StorePathSet & references, - RepairFlag repair) -{ - std::optional conn_(getConnection()); - auto & conn = *conn_; - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { - - conn->to - << wopAddToStore - << name - << renderContentAddressMethod(caMethod); - worker_proto::write(*this, conn->to, references); - conn->to << repair; - - conn.withFramedSink([&](Sink & sink) { - dump.drainInto(sink); - }); - - auto path = parseStorePath(readString(conn->from)); - return readValidPathInfo(conn, path); - } - else { - if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); - - std::visit(overloaded { - [&](TextHashMethod thm) -> void { - std::string s = dump.drain(); - conn->to << wopAddTextToStore << name << s; - worker_proto::write(*this, conn->to, references); - conn.processStderr(); - }, - [&](FixedOutputHashMethod fohm) -> void { - conn->to - << wopAddToStore - << name - << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(fohm.hashType); - - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { - dump.drainInto(conn->to); - } else { - std::string contents = dump.drain(); - dumpString(contents, conn->to); - } - } - conn->to.warn = false; - conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) - try { - conn.processStderr(); - } catch (EndOfFile & e) { } - throw; - } - - } - }, caMethod); - auto path = parseStorePath(readString(conn->from)); - // Release our connection to prevent a deadlock in queryPathInfo(). - conn_.reset(); - return queryPathInfo(path); - } -} - - -StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair) -{ - StorePathSet references; - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; -} - - -void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) -{ - auto conn(getConnection()); - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { - conn->to << wopImportPaths; - - auto source2 = sinkToSource([&](Sink & sink) { - sink << 1 // == path follows - ; - copyNAR(source, sink); - sink - << exportMagic - << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); - sink - << (info.deriver ? printStorePath(*info.deriver) : "") - << 0 // == no legacy signature - << 0 // == no path follows - ; - }); - - conn.processStderr(0, source2.get()); - - auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); - assert(importedPaths.size() <= 1); - } - - else { - conn->to << wopAddToStoreNar - << printStorePath(info.path) - << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); - conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << renderContentAddress(info.ca) - << repair << !checkSigs; - - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { - conn.withFramedSink([&](Sink & sink) { - copyNAR(source, sink); - }); - } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { - conn.processStderr(0, &source); - } else { - copyNAR(source, conn->to); - conn.processStderr(0, nullptr); - } - } -} - - -StorePath RemoteStore::addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) -{ - StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; -} - - -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - auto conn(getConnection()); - conn->to << wopBuildPaths; - assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); - conn->to << ss; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) - conn->to << buildMode; - else - /* Old daemons did not take a 'buildMode' parameter, so we - need to validate it here on the client side. */ - if (buildMode != bmNormal) - throw Error("repairing or checking is not supported when building through the Nix daemon"); - conn.processStderr(); - readInt(conn->from); -} - - -BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - auto conn(getConnection()); - conn->to << wopBuildDerivation << printStorePath(drvPath); - writeDerivation(conn->to, *this, drv); - conn->to << buildMode; - conn.processStderr(); - BuildResult res; - unsigned int status; - conn->from >> status >> res.errorMsg; - res.status = (BuildResult::Status) status; - return res; -} - - -void RemoteStore::ensurePath(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopEnsurePath << printStorePath(path); - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::addTempRoot(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopAddTempRoot << printStorePath(path); - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::addIndirectRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << wopAddIndirectRoot << path; - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::syncWithGC() -{ - auto conn(getConnection()); - conn->to << wopSyncWithGC; - conn.processStderr(); - readInt(conn->from); -} - - -Roots RemoteStore::findRoots(bool censor) -{ - auto conn(getConnection()); - conn->to << wopFindRoots; - conn.processStderr(); - size_t count = readNum(conn->from); - Roots result; - while (count--) { - Path link = readString(conn->from); - auto target = parseStorePath(readString(conn->from)); - result[std::move(target)].emplace(link); - } - return result; -} - - -void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) -{ - auto conn(getConnection()); - - conn->to - << wopCollectGarbage << options.action; - worker_proto::write(*this, conn->to, options.pathsToDelete); - conn->to << options.ignoreLiveness - << options.maxFreed - /* removed options */ - << 0 << 0 << 0; - - conn.processStderr(); - - results.paths = readStrings(conn->from); - results.bytesFreed = readLongLong(conn->from); - readLongLong(conn->from); // obsolete - - { - auto state_(Store::state.lock()); - state_->pathInfoCache.clear(); - } -} - - -void RemoteStore::optimiseStore() -{ - auto conn(getConnection()); - conn->to << wopOptimiseStore; - conn.processStderr(); - readInt(conn->from); -} - - -bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) -{ - auto conn(getConnection()); - conn->to << wopVerifyStore << checkContents << repair; - conn.processStderr(); - return readInt(conn->from); -} - - -void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) -{ - auto conn(getConnection()); - conn->to << wopAddSignatures << printStorePath(storePath) << sigs; - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) -{ - { - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) - // Don't hold the connection handle in the fallback case - // to prevent a deadlock. - goto fallback; - conn->to << wopQueryMissing; - Strings ss; - for (auto & p : targets) - ss.push_back(p.to_string(*this)); - conn->to << ss; - conn.processStderr(); - willBuild = worker_proto::read(*this, conn->from, Phantom {}); - willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); - unknown = worker_proto::read(*this, conn->from, Phantom {}); - conn->from >> downloadSize >> narSize; - return; - } - - fallback: - return Store::queryMissing(targets, willBuild, willSubstitute, - unknown, downloadSize, narSize); -} - - -void RemoteStore::connect() -{ - auto conn(getConnection()); -} - - -unsigned int RemoteStore::getProtocol() -{ - auto conn(connections->get()); - return conn->daemonVersion; -} - - -void RemoteStore::flushBadConnections() -{ - connections->flushBad(); -} - - -RemoteStore::Connection::~Connection() -{ - try { - to.flush(); - } catch (...) { - ignoreException(); - } -} - -void RemoteStore::narFromPath(const StorePath & path, Sink & sink) -{ - auto conn(connections->get()); - conn->to << wopNarFromPath << printStorePath(path); - conn->processStderr(); - copyNAR(conn->from, sink); -} - -ref RemoteStore::getFSAccessor() -{ - return make_ref(ref(shared_from_this())); -} - -static Logger::Fields readFields(Source & from) -{ - Logger::Fields fields; - size_t size = readInt(from); - for (size_t n = 0; n < size; n++) { - auto type = (decltype(Logger::Field::type)) readInt(from); - if (type == Logger::Field::tInt) - fields.push_back(readNum(from)); - else if (type == Logger::Field::tString) - fields.push_back(readString(from)); - else - throw Error("got unsupported field type %x from Nix daemon", (int) type); - } - return fields; -} - - -std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush) -{ - if (flush) - to.flush(); - - while (true) { - - auto msg = readNum(from); - - if (msg == STDERR_WRITE) { - string s = readString(from); - if (!sink) throw Error("no sink"); - (*sink)(s); - } - - else if (msg == STDERR_READ) { - if (!source) throw Error("no source"); - size_t len = readNum(from); - auto buf = std::make_unique(len); - writeString(buf.get(), source->read(buf.get(), len), to); - to.flush(); - } - - else if (msg == STDERR_ERROR) { - if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { - return std::make_exception_ptr(readError(from)); - } else { - string error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); - } - } - - else if (msg == STDERR_NEXT) - printError(chomp(readString(from))); - - else if (msg == STDERR_START_ACTIVITY) { - auto act = readNum(from); - auto lvl = (Verbosity) readInt(from); - auto type = (ActivityType) readInt(from); - auto s = readString(from); - auto fields = readFields(from); - auto parent = readNum(from); - logger->startActivity(act, lvl, type, s, fields, parent); - } - - else if (msg == STDERR_STOP_ACTIVITY) { - auto act = readNum(from); - logger->stopActivity(act); - } - - else if (msg == STDERR_RESULT) { - auto act = readNum(from); - auto type = (ResultType) readInt(from); - auto fields = readFields(from); - logger->result(act, type, fields); - } - - else if (msg == STDERR_LAST) - break; - - else - throw Error("got unknown message type %x from Nix daemon", msg); - } - - return nullptr; -} - -void ConnectionHandle::withFramedSink(std::function fun) -{ - (*this)->to.flush(); - - std::exception_ptr ex; - - /* Handle log messages / exceptions from the remote on a - separate thread. */ - std::thread stderrThread([&]() - { - try { - processStderr(nullptr, nullptr, false); - } catch (...) { - ex = std::current_exception(); - } - }); - - Finally joinStderrThread([&]() - { - if (stderrThread.joinable()) { - stderrThread.join(); - if (ex) { - try { - std::rethrow_exception(ex); - } catch (...) { - ignoreException(); - } - } - } - }); - - { - FramedSink sink((*this)->to, ex); - fun(sink); - sink.flush(); - } - - stderrThread.join(); - if (ex) - std::rethrow_exception(ex); - -} - static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 554fb6bed..e5de104c9 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -1,161 +1,10 @@ #pragma once -#include -#include - -#include "store-api.hh" +#include "remote-store.hh" #include "local-fs-store.hh" - namespace nix { - -class Pipe; -class Pid; -struct FdSink; -struct FdSource; -template class Pool; -struct ConnectionHandle; - -struct RemoteStoreConfig : virtual StoreConfig -{ - using StoreConfig::StoreConfig; - - const Setting maxConnections{(StoreConfig*) this, 1, - "max-connections", "maximum number of concurrent connections to the Nix daemon"}; - - const Setting maxConnectionAge{(StoreConfig*) this, std::numeric_limits::max(), - "max-connection-age", "number of seconds to reuse a connection"}; -}; - -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ -class RemoteStore : public virtual Store, public virtual RemoteStoreConfig -{ -public: - - virtual bool sameMachine() = 0; - - RemoteStore(const Params & params); - - /* Implementations of abstract store API methods. */ - - bool isValidPathUncached(const StorePath & path) override; - - StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override; - - StorePathSet queryAllValidPaths() override; - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override; - - StorePathSet queryValidDerivers(const StorePath & path) override; - - StorePathSet queryDerivationOutputs(const StorePath & path) override; - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; - std::optional queryPathFromHashPart(const std::string & hashPart) override; - - StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - - void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) override; - - /* Add a content-addressable store path. `dump` will be drained. */ - ref addCAToStore( - Source & dump, - const string & name, - ContentAddressMethod caMethod, - const StorePathSet & references, - RepairFlag repair); - - /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; - - void addToStore(const ValidPathInfo & info, Source & nar, - RepairFlag repair, CheckSigsFlag checkSigs) override; - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) override; - - void buildPaths(const std::vector & paths, BuildMode buildMode) override; - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override; - - void ensurePath(const StorePath & path) override; - - void addTempRoot(const StorePath & path) override; - - void addIndirectRoot(const Path & path) override; - - void syncWithGC() override; - - Roots findRoots(bool censor) override; - - void collectGarbage(const GCOptions & options, GCResults & results) override; - - void optimiseStore() override; - - bool verifyStore(bool checkContents, RepairFlag repair) override; - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override; - - void connect() override; - - unsigned int getProtocol() override; - - void flushBadConnections(); - - struct Connection - { - AutoCloseFD fd; - FdSink to; - FdSource from; - unsigned int daemonVersion; - std::chrono::time_point startTime; - - virtual ~Connection(); - - std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); - }; - - ref openConnectionWrapper(); - -protected: - - virtual ref openConnection() = 0; - - void initConnection(Connection & conn); - - ref> connections; - - virtual void setOptions(Connection & conn); - - ConnectionHandle getConnection(); - - friend struct ConnectionHandle; - - virtual ref getFSAccessor() override; - - virtual void narFromPath(const StorePath & path, Sink & sink) override; - - ref readValidPathInfo(ConnectionHandle & conn, const StorePath & path); - -private: - - std::atomic_bool failed{false}; - -}; - struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { UDSRemoteStoreConfig(const Store::Params & params) @@ -200,5 +49,4 @@ private: std::optional path; }; - }